[Notes] [Git][BuildStream/buildstream][bst-1.2] 5 commits: tests: avoid setting max-jobs

[Tiago Gomes <gitlab mg gitlab com>]

Title: GitLab

Tiago Gomes pushed to branch bst-1.2 at BuildStream / buildstream

Commits:

• 4aa0dd0b
  by Tiago Gomes at 2018-09-14T15:43:47Z

  tests: avoid setting max-jobs
  
  Setting "max-jobs" won't be allowed anymore in a following commit.
  

• 06001fef
  by Tiago Gomes at 2018-09-14T15:43:47Z

  docs: document protected variables
  
  And remove then from the defaults as they are dynamically set by
  BuildStream.
  

• 5e3587cd
  by Tiago Gomes at 2018-09-14T15:43:47Z

  element: validate configuration variables
  
  Ensure that protected variables are not being redefined by the user.
  

• 3cbcec43
  by Tiago Gomes at 2018-09-14T15:43:47Z

  Add tests for validating configuration variables
  

• dd513fcc
  by Tiago Gomes at 2018-09-14T16:10:28Z

  Merge branch 'tiagogomes/issue-287-backport' into 'bst-1.2'
  
  Backport of !678 (Add validation of configuration variables) to 1.2 branch.
  
  See merge request BuildStream/buildstream!789

8 changed files:

• buildstream/_exceptions.py
• buildstream/data/projectconfig.yaml
• buildstream/element.py
• doc/source/format_declaring.rst
• tests/integration/manual.py
• + tests/loader/variables.py
• + tests/loader/variables/simple/foo.txt
• + tests/loader/variables/simple/project.conf

Changes:

buildstream/_exceptions.py

---

...	...	@@ -219,6 +219,9 @@ class LoadErrorReason(Enum):
219	219	# A recursive variable has been encountered
220	220	RECURSIVE_VARIABLE = 22
221	221
	222	+ # An attempt so set the value of a protected variable
	223	+ PROTECTED_VARIABLE_REDEFINED = 23
	224	+
222	225
223	226	# LoadError
224	227	#

buildstream/data/projectconfig.yaml

---

...	...	@@ -20,21 +20,7 @@ fail-on-overlap: False
20	20	# Variable Configuration
21	21	#
22	22	variables:
23		-
24		- # Maximum number of parallel build processes within a given
25		- # build, support for this is conditional on the element type
26		- # and the build system used (any element using 'make' can
27		- # implement this).
28		- #
29		- # Note: this value defaults to the number of cores available
30		- max-jobs: 4
31		-
32		- # Note: These variables are defined later on in element.py and _project.py
33		- element-name: ""
34		- project-name: ""
35		-
36	23	# Path configuration, to be used in build instructions.
37		- #
38	24	prefix: "/usr"
39	25	exec_prefix: "%{prefix}"
40	26	bindir: "%{exec_prefix}/bin"
...	...	@@ -93,7 +79,6 @@ variables:
93	79	find "%{install-root}" -name '*.pyc' -exec \
94	80	dd if=/dev/zero of={} bs=1 count=4 seek=4 conv=notrunc ';'
95	81
96		-
97	82	# Base sandbox environment, can be overridden by plugins
98	83	environment:
99	84	PATH: /usr/bin:/bin:/usr/sbin:/sbin

buildstream/element.py

---

...	...	@@ -2166,7 +2166,8 @@ class Element(Plugin):
2166	2166	# substituting command strings to be run in the sandbox
2167	2167	#
2168	2168	def __extract_variables(self, meta):
2169		- default_vars = _yaml.node_get(self.__defaults, Mapping, 'variables', default_value={})
	2169	+ default_vars = _yaml.node_get(self.__defaults, Mapping, 'variables',
	2170	+ default_value={})
2170	2171
2171	2172	project = self._get_project()
2172	2173	if self.__is_junction:
...	...	@@ -2179,6 +2180,13 @@ class Element(Plugin):
2179	2180	_yaml.composite(variables, meta.variables)
2180	2181	_yaml.node_final_assertions(variables)
2181	2182
	2183	+ for var in ('project-name', 'element-name', 'max-jobs'):
	2184	+ provenance = _yaml.node_get_provenance(variables, var)
	2185	+ if provenance and provenance.filename != '':
	2186	+ raise LoadError(LoadErrorReason.PROTECTED_VARIABLE_REDEFINED,
	2187	+ "{}: invalid redefinition of protected variable '{}'"
	2188	+ .format(provenance, var))
	2189	+
2182	2190	return variables
2183	2191
2184	2192	# This will resolve the final configuration to be handed

doc/source/format_declaring.rst

---

...	...	@@ -420,3 +420,25 @@ dependency and that all referenced variables are declared, the following is fine
420	420	install-commands:
421	421	- |
422	422	%{make-install} RELEASE_TEXT="%{release-text}"
	423	+
	424	+
	425	+Variables declared by BuildStream
	426	+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	427	+BuildStream declares a set of :ref:`builtin <project_builtin_defaults>`
	428	+variables that may be overridden. In addition, the following
	429	+read-only variables are also dynamically declared by BuildStream:
	430	+
	431	+* ``element-name``
	432	+
	433	+ The name of the element being processed (e.g base/alpine.bst).
	434	+
	435	+* ``project-name``
	436	+
	437	+ The name of project where BuildStream is being used.
	438	+
	439	+* ``max-jobs``
	440	+
	441	+ Maximum number of parallel build processes within a given
	442	+ build, support for this is conditional on the element type
	443	+ and the build system used (any element using 'make' can
	444	+ implement this).

tests/integration/manual.py

---

...	...	@@ -64,7 +64,7 @@ strip
64	64
65	65
66	66	@pytest.mark.datafiles(DATA_DIR)
67		-def test_manual_element_noparallel(cli, tmpdir, datafiles):
	67	+def test_manual_element_environment(cli, tmpdir, datafiles):
68	68	project = os.path.join(datafiles.dirname, datafiles.basename)
69	69	checkout = os.path.join(cli.directory, 'checkout')
70	70	element_path = os.path.join(project, 'elements')
...	...	@@ -72,15 +72,11 @@ def test_manual_element_noparallel(cli, tmpdir, datafiles):
72	72
73	73	create_manual_element(element_name, element_path, {
74	74	'install-commands': [
75		- "echo $MAKEFLAGS >> test",
76	75	"echo $V >> test",
77	76	"cp test %{install-root}"
78	77	]
79	78	}, {
80		- 'max-jobs': 2,
81		- 'notparallel': True
82	79	}, {
83		- 'MAKEFLAGS': '-j%{max-jobs} -Wall',
84	80	'V': 2
85	81	})
86	82
...	...	@@ -93,13 +89,11 @@ def test_manual_element_noparallel(cli, tmpdir, datafiles):
93	89	with open(os.path.join(checkout, 'test')) as f:
94	90	text = f.read()
95	91
96		- assert text == """-j1 -Wall
97		-2
98		-"""
	92	+ assert text == "2\n"
99	93
100	94
101	95	@pytest.mark.datafiles(DATA_DIR)
102		-def test_manual_element_environment(cli, tmpdir, datafiles):
	96	+def test_manual_element_noparallel(cli, tmpdir, datafiles):
103	97	project = os.path.join(datafiles.dirname, datafiles.basename)
104	98	checkout = os.path.join(cli.directory, 'checkout')
105	99	element_path = os.path.join(project, 'elements')
...	...	@@ -112,7 +106,7 @@ def test_manual_element_environment(cli, tmpdir, datafiles):
112	106	"cp test %{install-root}"
113	107	]
114	108	}, {
115		- 'max-jobs': 2
	109	+ 'notparallel': True
116	110	}, {
117	111	'MAKEFLAGS': '-j%{max-jobs} -Wall',
118	112	'V': 2
...	...	@@ -127,6 +121,6 @@ def test_manual_element_environment(cli, tmpdir, datafiles):
127	121	with open(os.path.join(checkout, 'test')) as f:
128	122	text = f.read()
129	123
130		- assert text == """-j2 -Wall
	124	+ assert text == """-j1 -Wall
131	125	2
132	126	"""

tests/loader/variables.py

---

	1	+import os
	2	+import pytest
	3	+
	4	+from buildstream import _yaml
	5	+from buildstream._exceptions import ErrorDomain, LoadErrorReason
	6	+from tests.testutils import cli
	7	+
	8	+DATA_DIR = os.path.join(
	9	+ os.path.dirname(os.path.realpath(__file__)),
	10	+ 'variables',
	11	+)
	12	+
	13	+PROTECTED_VARIABLES = [('project-name'), ('element-name'), ('max-jobs')]
	14	+
	15	+
	16	+@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
	17	+@pytest.mark.datafiles(DATA_DIR)
	18	+def test_use_of_protected_var_project_conf(cli, tmpdir, datafiles, protected_var):
	19	+ project = os.path.join(str(datafiles), 'simple')
	20	+
	21	+ conf = {
	22	+ 'name': 'test',
	23	+ 'variables': {
	24	+ protected_var: 'some-value'
	25	+ }
	26	+ }
	27	+ _yaml.dump(conf, os.path.join(project, 'project.conf'))
	28	+
	29	+ element = {
	30	+ 'kind': 'import',
	31	+ 'sources': [
	32	+ {
	33	+ 'kind': 'local',
	34	+ 'path': 'foo.txt'
	35	+ }
	36	+ ],
	37	+ }
	38	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	39	+
	40	+ result = cli.run(project=project, args=['build', 'target.bst'])
	41	+ result.assert_main_error(ErrorDomain.LOAD,
	42	+ LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
	43	+
	44	+
	45	+@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
	46	+@pytest.mark.datafiles(DATA_DIR)
	47	+def test_use_of_protected_var_element_overrides(cli, tmpdir, datafiles, protected_var):
	48	+ project = os.path.join(str(datafiles), 'simple')
	49	+
	50	+ conf = {
	51	+ 'name': 'test',
	52	+ 'elements': {
	53	+ 'manual': {
	54	+ 'variables': {
	55	+ protected_var: 'some-value'
	56	+ }
	57	+ }
	58	+ }
	59	+ }
	60	+ _yaml.dump(conf, os.path.join(project, 'project.conf'))
	61	+
	62	+ element = {
	63	+ 'kind': 'manual',
	64	+ 'sources': [
	65	+ {
	66	+ 'kind': 'local',
	67	+ 'path': 'foo.txt'
	68	+ }
	69	+ ],
	70	+ }
	71	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	72	+
	73	+ result = cli.run(project=project, args=['build', 'target.bst'])
	74	+ result.assert_main_error(ErrorDomain.LOAD,
	75	+ LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
	76	+
	77	+
	78	+@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
	79	+@pytest.mark.datafiles(DATA_DIR)
	80	+def test_use_of_protected_var_in_element(cli, tmpdir, datafiles, protected_var):
	81	+ project = os.path.join(str(datafiles), 'simple')
	82	+
	83	+ element = {
	84	+ 'kind': 'import',
	85	+ 'sources': [
	86	+ {
	87	+ 'kind': 'local',
	88	+ 'path': 'foo.txt'
	89	+ }
	90	+ ],
	91	+ 'variables': {
	92	+ protected_var: 'some-value'
	93	+ }
	94	+ }
	95	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	96	+
	97	+ result = cli.run(project=project, args=['build', 'target.bst'])
	98	+ result.assert_main_error(ErrorDomain.LOAD,
	99	+ LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)

tests/loader/variables/simple/foo.txt

---

1

+foo

tests/loader/variables/simple/project.conf

---

1

+name: foo

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.