[Notes] [Git][BuildStream/buildstream][tiagogomes/issue-287] 3 commits: tests: avoid setting max-jobs

[Tiago Gomes <gitlab mg gitlab com>]

Title: GitLab

Tiago Gomes pushed to branch tiagogomes/issue-287 at BuildStream / buildstream

Commits:

• 0bf76299
  by Tiago Gomes at 2018-09-07T13:52:59Z

  tests: avoid setting max-jobs
  
  Setting "max-jobs" won't be allowed anymore in a following commit.
  

• ff033825
  by Tiago Gomes at 2018-09-07T13:52:59Z

  Add validation of configuration variables
  
  Closes #287.
  

• d415bde5
  by Tiago Gomes at 2018-09-07T13:52:59Z

  Add tests for validating configuration variables
  

8 changed files:

• buildstream/_loader/loadelement.py
• buildstream/_project.py
• buildstream/data/projectconfig.yaml
• doc/source/format_declaring.rst
• tests/integration/manual.py
• + tests/loader/variables.py
• + tests/loader/variables/simple/foo.txt
• + tests/loader/variables/simple/project.conf

Changes:

buildstream/_loader/loadelement.py

---

...	...	@@ -74,6 +74,9 @@ class LoadElement():
74	74	'build-depends', 'runtime-depends',
75	75	])
76	76
	77	+ # Validate configuration variables
	78	+ _yaml.node_get_configuration_variables(self.node)
	79	+
77	80	# Extract the Dependencies
78	81	self.deps = _extract_depends_from_node(self.node)
79	82

buildstream/_project.py

---

...	...	@@ -566,6 +566,13 @@ class Project():
566	566	output.source_overrides = _yaml.node_get(config, Mapping, 'sources', default_value={})
567	567	config.pop('elements', None)
568	568	config.pop('sources', None)
	569	+
	570	+ for _, val in _yaml.node_items(output.element_overrides):
	571	+ _yaml.node_get_configuration_variables(val)
	572	+
	573	+ for _, val in _yaml.node_items(output.source_overrides):
	574	+ _yaml.node_get_configuration_variables(val)
	575	+
569	576	_yaml.node_final_assertions(config)
570	577
571	578	self._load_plugin_factories(config, output)
...	...	@@ -595,7 +602,7 @@ class Project():
595	602	output.options.process_node(config)
596	603
597	604	# Load base variables
598		- output.base_variables = _yaml.node_get(config, Mapping, 'variables')
	605	+ output.base_variables = _yaml.node_get_configuration_variables(config)
599	606
600	607	# Add the project name as a default variable
601	608	output.base_variables['project-name'] = self.name

buildstream/data/projectconfig.yaml

---

...	...	@@ -16,21 +16,7 @@ ref-storage: inline
16	16	# Variable Configuration
17	17	#
18	18	variables:
19		-
20		- # Maximum number of parallel build processes within a given
21		- # build, support for this is conditional on the element type
22		- # and the build system used (any element using 'make' can
23		- # implement this).
24		- #
25		- # Note: this value defaults to the number of cores available
26		- max-jobs: 4
27		-
28		- # Note: These variables are defined later on in element.py and _project.py
29		- element-name: ""
30		- project-name: ""
31		-
32	19	# Path configuration, to be used in build instructions.
33		- #
34	20	prefix: "/usr"
35	21	exec_prefix: "%{prefix}"
36	22	bindir: "%{exec_prefix}/bin"
...	...	@@ -89,7 +75,6 @@ variables:
89	75	find "%{install-root}" -name '*.pyc' -exec \
90	76	dd if=/dev/zero of={} bs=1 count=4 seek=4 conv=notrunc ';'
91	77
92		-
93	78	# Base sandbox environment, can be overridden by plugins
94	79	environment:
95	80	PATH: /usr/bin:/bin:/usr/sbin:/sbin

doc/source/format_declaring.rst

---

...	...	@@ -484,3 +484,26 @@ dependency and that all referenced variables are declared, the following is fine
484	484	install-commands:
485	485	- |
486	486	%{make-install} RELEASE_TEXT="%{release-text}"
	487	+
	488	+
	489	+Variables declared by BuildStream
	490	+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	491	+
	492	+BuildStream declares a set of :ref:`builtin <project_builtin_defaults>`
	493	+variables that may be overriden. In addition, the following
	494	+read-only variables are also dynamically declared by BuildStream:
	495	+
	496	+* ``element-name``
	497	+
	498	+ The name of the element being processed (e.g base/alpine.bst).
	499	+
	500	+* ``project-name``
	501	+
	502	+ The name of project where BuildStream is being used.
	503	+
	504	+* ``max-jobs``
	505	+
	506	+ Maximum number of parallel build processes within a given
	507	+ build, support for this is conditional on the element type
	508	+ and the build system used (any element using 'make' can
	509	+ implement this).

tests/integration/manual.py

---

...	...	@@ -64,7 +64,7 @@ strip
64	64
65	65
66	66	@pytest.mark.datafiles(DATA_DIR)
67		-def test_manual_element_noparallel(cli, tmpdir, datafiles):
	67	+def test_manual_element_environment(cli, tmpdir, datafiles):
68	68	project = os.path.join(datafiles.dirname, datafiles.basename)
69	69	checkout = os.path.join(cli.directory, 'checkout')
70	70	element_path = os.path.join(project, 'elements')
...	...	@@ -72,15 +72,11 @@ def test_manual_element_noparallel(cli, tmpdir, datafiles):
72	72
73	73	create_manual_element(element_name, element_path, {
74	74	'install-commands': [
75		- "echo $MAKEFLAGS >> test",
76	75	"echo $V >> test",
77	76	"cp test %{install-root}"
78	77	]
79	78	}, {
80		- 'max-jobs': 2,
81		- 'notparallel': True
82	79	}, {
83		- 'MAKEFLAGS': '-j%{max-jobs} -Wall',
84	80	'V': 2
85	81	})
86	82
...	...	@@ -93,13 +89,11 @@ def test_manual_element_noparallel(cli, tmpdir, datafiles):
93	89	with open(os.path.join(checkout, 'test')) as f:
94	90	text = f.read()
95	91
96		- assert text == """-j1 -Wall
97		-2
98		-"""
	92	+ assert text == "2\n"
99	93
100	94
101	95	@pytest.mark.datafiles(DATA_DIR)
102		-def test_manual_element_environment(cli, tmpdir, datafiles):
	96	+def test_manual_element_noparallel(cli, tmpdir, datafiles):
103	97	project = os.path.join(datafiles.dirname, datafiles.basename)
104	98	checkout = os.path.join(cli.directory, 'checkout')
105	99	element_path = os.path.join(project, 'elements')
...	...	@@ -112,7 +106,7 @@ def test_manual_element_environment(cli, tmpdir, datafiles):
112	106	"cp test %{install-root}"
113	107	]
114	108	}, {
115		- 'max-jobs': 2
	109	+ 'notparallel': True
116	110	}, {
117	111	'MAKEFLAGS': '-j%{max-jobs} -Wall',
118	112	'V': 2
...	...	@@ -127,6 +121,6 @@ def test_manual_element_environment(cli, tmpdir, datafiles):
127	121	with open(os.path.join(checkout, 'test')) as f:
128	122	text = f.read()
129	123
130		- assert text == """-j2 -Wall
	124	+ assert text == """-j1 -Wall
131	125	2
132	126	"""

tests/loader/variables.py

---

	1	+import os
	2	+import pytest
	3	+
	4	+from buildstream import _yaml
	5	+from buildstream._exceptions import ErrorDomain, LoadErrorReason
	6	+from tests.testutils import cli
	7	+
	8	+DATA_DIR = os.path.join(
	9	+ os.path.dirname(os.path.realpath(__file__)),
	10	+ 'variables',
	11	+)
	12	+
	13	+PROTECTED_VARIABLES = [('project-name'), ('element-name'), ('max-jobs')]
	14	+
	15	+
	16	+@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
	17	+@pytest.mark.datafiles(DATA_DIR)
	18	+def test_use_of_protected_var_in_element(cli, tmpdir, datafiles, protected_var):
	19	+ project = os.path.join(str(datafiles), 'simple')
	20	+
	21	+ element = {
	22	+ 'kind': 'import',
	23	+ 'sources': [
	24	+ {
	25	+ 'kind': 'local',
	26	+ 'path': 'foo.txt'
	27	+ }
	28	+ ],
	29	+ 'variables': {
	30	+ protected_var: 'some-value'
	31	+ }
	32	+ }
	33	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	34	+
	35	+ result = cli.run(project=project, args=['build', 'target.bst'])
	36	+ result.assert_main_error(ErrorDomain.LOAD,
	37	+ LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
	38	+
	39	+
	40	+@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
	41	+@pytest.mark.datafiles(DATA_DIR)
	42	+def test_use_of_protected_var_project_conf(cli, tmpdir, datafiles, protected_var):
	43	+ project = os.path.join(str(datafiles), 'simple')
	44	+
	45	+ conf = {
	46	+ 'name': 'test',
	47	+ 'variables': {
	48	+ protected_var: 'some-value'
	49	+ }
	50	+ }
	51	+ _yaml.dump(conf, os.path.join(project, 'project.conf'))
	52	+
	53	+ result = cli.run(project=project, args=['show'])
	54	+ result.assert_main_error(ErrorDomain.LOAD,
	55	+ LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
	56	+
	57	+
	58	+@pytest.mark.parametrize('plugin_type', [
	59	+ ('sources'),
	60	+ ('elements')
	61	+])
	62	+@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
	63	+@pytest.mark.datafiles(DATA_DIR)
	64	+def test_use_of_protected_var_element_overrides(cli, tmpdir, datafiles, plugin_type, protected_var):
	65	+ project = os.path.join(str(datafiles), 'simple')
	66	+
	67	+ conf = {
	68	+ 'name': 'test',
	69	+ plugin_type: {
	70	+ 'some-plugin-name': {
	71	+ 'variables': {
	72	+ protected_var: 'some-value'
	73	+ }
	74	+ }
	75	+ }
	76	+ }
	77	+ _yaml.dump(conf, os.path.join(project, 'project.conf'))
	78	+
	79	+ result = cli.run(project=project, args=['show'])
	80	+ result.assert_main_error(ErrorDomain.LOAD,
	81	+ LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)

tests/loader/variables/simple/foo.txt

---

1

+foo

tests/loader/variables/simple/project.conf

---

1

+name: foo

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.