[Notes] [Git][BuildStream/buildstream][master] 5 commits: tests: avoid setting max-jobs

[Tiago Gomes <gitlab mg gitlab com>]

Title: GitLab

Tiago Gomes pushed to branch master at BuildStream / buildstream

Commits:

• ef66daf5
  by Tiago Gomes at 2018-09-10T14:57:41Z

  tests: avoid setting max-jobs
  
  Setting "max-jobs" won't be allowed anymore in a following commit.
  

• a37bc6ce
  by Tiago Gomes at 2018-09-10T14:57:41Z

  docs: document protected variables
  
  And remove then from the defaults as they are dynamically set by
  BuildStream.
  

• af74a3f8
  by Tiago Gomes at 2018-09-10T14:57:41Z

  element: validate configuration variables
  
  Ensure that protected variables are not being redefined by the user.
  

• 75c55633
  by Tiago Gomes at 2018-09-10T14:57:41Z

  Add tests for validating configuration variables
  

• e0bb71b2
  by Tiago Gomes at 2018-09-10T15:27:53Z

  Merge branch 'tiagogomes/issue-287' into 'master'
  
  Add validation of configuration variables
  
  See merge request BuildStream/buildstream!678

8 changed files:

• buildstream/_exceptions.py
• buildstream/data/projectconfig.yaml
• buildstream/element.py
• doc/source/format_declaring.rst
• tests/integration/manual.py
• + tests/loader/variables.py
• + tests/loader/variables/simple/foo.txt
• + tests/loader/variables/simple/project.conf

Changes:

buildstream/_exceptions.py

---

...	...	@@ -220,6 +220,9 @@ class LoadErrorReason(Enum):
220	220	# A recursive variable has been encountered
221	221	RECURSIVE_VARIABLE = 22
222	222
	223	+ # An attempt so set the value of a protected variable
	224	+ PROTECTED_VARIABLE_REDEFINED = 23
	225	+
223	226
224	227	# LoadError
225	228	#

buildstream/data/projectconfig.yaml

---

...	...	@@ -16,21 +16,7 @@ ref-storage: inline
16	16	# Variable Configuration
17	17	#
18	18	variables:
19		-
20		- # Maximum number of parallel build processes within a given
21		- # build, support for this is conditional on the element type
22		- # and the build system used (any element using 'make' can
23		- # implement this).
24		- #
25		- # Note: this value defaults to the number of cores available
26		- max-jobs: 4
27		-
28		- # Note: These variables are defined later on in element.py and _project.py
29		- element-name: ""
30		- project-name: ""
31		-
32	19	# Path configuration, to be used in build instructions.
33		- #
34	20	prefix: "/usr"
35	21	exec_prefix: "%{prefix}"
36	22	bindir: "%{exec_prefix}/bin"
...	...	@@ -89,7 +75,6 @@ variables:
89	75	find "%{install-root}" -name '*.pyc' -exec \
90	76	dd if=/dev/zero of={} bs=1 count=4 seek=4 conv=notrunc ';'
91	77
92		-
93	78	# Base sandbox environment, can be overridden by plugins
94	79	environment:
95	80	PATH: /usr/bin:/bin:/usr/sbin:/sbin

buildstream/element.py

---

...	...	@@ -2283,7 +2283,8 @@ class Element(Plugin):
2283	2283	# substituting command strings to be run in the sandbox
2284	2284	#
2285	2285	def __extract_variables(self, meta):
2286		- default_vars = _yaml.node_get(self.__defaults, Mapping, 'variables', default_value={})
	2286	+ default_vars = _yaml.node_get(self.__defaults, Mapping, 'variables',
	2287	+ default_value={})
2287	2288
2288	2289	project = self._get_project()
2289	2290	if self.__is_junction:
...	...	@@ -2296,6 +2297,13 @@ class Element(Plugin):
2296	2297	_yaml.composite(variables, meta.variables)
2297	2298	_yaml.node_final_assertions(variables)
2298	2299
	2300	+ for var in ('project-name', 'element-name', 'max-jobs'):
	2301	+ provenance = _yaml.node_get_provenance(variables, var)
	2302	+ if provenance and provenance.filename != '':
	2303	+ raise LoadError(LoadErrorReason.PROTECTED_VARIABLE_REDEFINED,
	2304	+ "{}: invalid redefinition of protected variable '{}'"
	2305	+ .format(provenance, var))
	2306	+
2299	2307	return variables
2300	2308
2301	2309	# This will resolve the final configuration to be handed

doc/source/format_declaring.rst

---

...	...	@@ -484,3 +484,25 @@ dependency and that all referenced variables are declared, the following is fine
484	484	install-commands:
485	485	- |
486	486	%{make-install} RELEASE_TEXT="%{release-text}"
	487	+
	488	+
	489	+Variables declared by BuildStream
	490	+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
	491	+BuildStream declares a set of :ref:`builtin <project_builtin_defaults>`
	492	+variables that may be overridden. In addition, the following
	493	+read-only variables are also dynamically declared by BuildStream:
	494	+
	495	+* ``element-name``
	496	+
	497	+ The name of the element being processed (e.g base/alpine.bst).
	498	+
	499	+* ``project-name``
	500	+
	501	+ The name of project where BuildStream is being used.
	502	+
	503	+* ``max-jobs``
	504	+
	505	+ Maximum number of parallel build processes within a given
	506	+ build, support for this is conditional on the element type
	507	+ and the build system used (any element using 'make' can
	508	+ implement this).

tests/integration/manual.py

---

...	...	@@ -64,7 +64,7 @@ strip
64	64
65	65
66	66	@pytest.mark.datafiles(DATA_DIR)
67		-def test_manual_element_noparallel(cli, tmpdir, datafiles):
	67	+def test_manual_element_environment(cli, tmpdir, datafiles):
68	68	project = os.path.join(datafiles.dirname, datafiles.basename)
69	69	checkout = os.path.join(cli.directory, 'checkout')
70	70	element_path = os.path.join(project, 'elements')
...	...	@@ -72,15 +72,11 @@ def test_manual_element_noparallel(cli, tmpdir, datafiles):
72	72
73	73	create_manual_element(element_name, element_path, {
74	74	'install-commands': [
75		- "echo $MAKEFLAGS >> test",
76	75	"echo $V >> test",
77	76	"cp test %{install-root}"
78	77	]
79	78	}, {
80		- 'max-jobs': 2,
81		- 'notparallel': True
82	79	}, {
83		- 'MAKEFLAGS': '-j%{max-jobs} -Wall',
84	80	'V': 2
85	81	})
86	82
...	...	@@ -93,13 +89,11 @@ def test_manual_element_noparallel(cli, tmpdir, datafiles):
93	89	with open(os.path.join(checkout, 'test')) as f:
94	90	text = f.read()
95	91
96		- assert text == """-j1 -Wall
97		-2
98		-"""
	92	+ assert text == "2\n"
99	93
100	94
101	95	@pytest.mark.datafiles(DATA_DIR)
102		-def test_manual_element_environment(cli, tmpdir, datafiles):
	96	+def test_manual_element_noparallel(cli, tmpdir, datafiles):
103	97	project = os.path.join(datafiles.dirname, datafiles.basename)
104	98	checkout = os.path.join(cli.directory, 'checkout')
105	99	element_path = os.path.join(project, 'elements')
...	...	@@ -112,7 +106,7 @@ def test_manual_element_environment(cli, tmpdir, datafiles):
112	106	"cp test %{install-root}"
113	107	]
114	108	}, {
115		- 'max-jobs': 2
	109	+ 'notparallel': True
116	110	}, {
117	111	'MAKEFLAGS': '-j%{max-jobs} -Wall',
118	112	'V': 2
...	...	@@ -127,6 +121,6 @@ def test_manual_element_environment(cli, tmpdir, datafiles):
127	121	with open(os.path.join(checkout, 'test')) as f:
128	122	text = f.read()
129	123
130		- assert text == """-j2 -Wall
	124	+ assert text == """-j1 -Wall
131	125	2
132	126	"""

tests/loader/variables.py

---

	1	+import os
	2	+import pytest
	3	+
	4	+from buildstream import _yaml
	5	+from buildstream._exceptions import ErrorDomain, LoadErrorReason
	6	+from tests.testutils import cli
	7	+
	8	+DATA_DIR = os.path.join(
	9	+ os.path.dirname(os.path.realpath(__file__)),
	10	+ 'variables',
	11	+)
	12	+
	13	+PROTECTED_VARIABLES = [('project-name'), ('element-name'), ('max-jobs')]
	14	+
	15	+
	16	+@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
	17	+@pytest.mark.datafiles(DATA_DIR)
	18	+def test_use_of_protected_var_project_conf(cli, tmpdir, datafiles, protected_var):
	19	+ project = os.path.join(str(datafiles), 'simple')
	20	+
	21	+ conf = {
	22	+ 'name': 'test',
	23	+ 'variables': {
	24	+ protected_var: 'some-value'
	25	+ }
	26	+ }
	27	+ _yaml.dump(conf, os.path.join(project, 'project.conf'))
	28	+
	29	+ element = {
	30	+ 'kind': 'import',
	31	+ 'sources': [
	32	+ {
	33	+ 'kind': 'local',
	34	+ 'path': 'foo.txt'
	35	+ }
	36	+ ],
	37	+ }
	38	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	39	+
	40	+ result = cli.run(project=project, args=['build', 'target.bst'])
	41	+ result.assert_main_error(ErrorDomain.LOAD,
	42	+ LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
	43	+
	44	+
	45	+@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
	46	+@pytest.mark.datafiles(DATA_DIR)
	47	+def test_use_of_protected_var_element_overrides(cli, tmpdir, datafiles, protected_var):
	48	+ project = os.path.join(str(datafiles), 'simple')
	49	+
	50	+ conf = {
	51	+ 'name': 'test',
	52	+ 'elements': {
	53	+ 'manual': {
	54	+ 'variables': {
	55	+ protected_var: 'some-value'
	56	+ }
	57	+ }
	58	+ }
	59	+ }
	60	+ _yaml.dump(conf, os.path.join(project, 'project.conf'))
	61	+
	62	+ element = {
	63	+ 'kind': 'manual',
	64	+ 'sources': [
	65	+ {
	66	+ 'kind': 'local',
	67	+ 'path': 'foo.txt'
	68	+ }
	69	+ ],
	70	+ }
	71	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	72	+
	73	+ result = cli.run(project=project, args=['build', 'target.bst'])
	74	+ result.assert_main_error(ErrorDomain.LOAD,
	75	+ LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)
	76	+
	77	+
	78	+@pytest.mark.parametrize('protected_var', PROTECTED_VARIABLES)
	79	+@pytest.mark.datafiles(DATA_DIR)
	80	+def test_use_of_protected_var_in_element(cli, tmpdir, datafiles, protected_var):
	81	+ project = os.path.join(str(datafiles), 'simple')
	82	+
	83	+ element = {
	84	+ 'kind': 'import',
	85	+ 'sources': [
	86	+ {
	87	+ 'kind': 'local',
	88	+ 'path': 'foo.txt'
	89	+ }
	90	+ ],
	91	+ 'variables': {
	92	+ protected_var: 'some-value'
	93	+ }
	94	+ }
	95	+ _yaml.dump(element, os.path.join(project, 'target.bst'))
	96	+
	97	+ result = cli.run(project=project, args=['build', 'target.bst'])
	98	+ result.assert_main_error(ErrorDomain.LOAD,
	99	+ LoadErrorReason.PROTECTED_VARIABLE_REDEFINED)

tests/loader/variables/simple/foo.txt

---

1

+foo

tests/loader/variables/simple/project.conf

---

1

+name: foo

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.