Phil Dawson pushed to branch phil/848-plugin-deprecation-warnings at BuildStream / buildstream
Commits:
-
c8cd24b9
by Jonathan Maw at 2019-02-18T17:37:00Z
-
de70f8c0
by Jonathan Maw at 2019-02-18T18:57:45Z
-
00b86b19
by Chandan Singh at 2019-02-18T18:59:31Z
-
067a0c6b
by Chandan Singh at 2019-02-18T20:31:10Z
-
fca37d0d
by Chandan Singh at 2019-02-19T06:37:04Z
-
5f0571d1
by Jürg Billeter at 2019-02-19T07:37:21Z
-
e2074693
by Gökçen Nurlu at 2019-02-19T07:44:54Z
-
cfbe409d
by Jürg Billeter at 2019-02-19T09:00:31Z
-
1ae17968
by Valentin David at 2019-02-19T09:01:48Z
-
afe823e8
by Jürg Billeter at 2019-02-19T10:33:59Z
-
d70bfc38
by Adam Jones at 2019-02-19T10:34:37Z
-
857e7414
by Tom Pollard at 2019-02-19T11:37:15Z
-
9bc389a8
by Jürg Billeter at 2019-02-19T11:39:44Z
-
4a002bee
by Jürg Billeter at 2019-02-19T13:17:00Z
-
1cd27edf
by Jürg Billeter at 2019-02-19T13:19:43Z
-
69675d22
by Jürg Billeter at 2019-02-19T13:27:31Z
-
99764715
by Jürg Billeter at 2019-02-19T15:03:58Z
-
2421027f
by Phil Dawson at 2019-02-19T15:29:32Z
20 changed files:
- .gitlab-ci.yml
- .gitlab/issue_templates/bst_bug.md
- .gitlab/issue_templates/bst_task.md
- NEWS
- buildstream/_project.py
- buildstream/_yaml.py
- buildstream/element.py
- buildstream/plugin.py
- doc/badges.py
- + tests/frontend/artifact.py
- tests/frontend/overlaps/a-whitelisted.bst
- tests/frontend/overlaps/b-whitelisted.bst
- tests/frontend/overlaps/c-whitelisted.bst
- tests/integration/artifact.py
- tests/integration/shell.py
- + tests/plugins/deprecationwarnings/deprecationwarnings.py
- + tests/plugins/deprecationwarnings/project/elements/deprecated.bst
- + tests/plugins/deprecationwarnings/project/plugins/elements/deprecated_plugin.py
- + tests/plugins/deprecationwarnings/project/plugins/elements/deprecated_plugin.yaml
- + tests/plugins/deprecationwarnings/project/project.conf
Changes:
... | ... | @@ -160,7 +160,7 @@ tests-wsl: |
160 | 160 |
|
161 | 161 |
script:
|
162 | 162 |
- "${TEST_COMMAND}"
|
163 |
- when: manual
|
|
163 |
+ allow_failure: true
|
|
164 | 164 |
|
165 | 165 |
# Automatically build documentation for every commit, we want to know
|
166 | 166 |
# if building documentation fails even if we're not deploying it.
|
... | ... | @@ -33,4 +33,6 @@ |
33 | 33 |
* BuildStream version affected: /milestone %BuildStream_v1.x
|
34 | 34 |
|
35 | 35 |
----
|
36 |
+[//]: # (To review information about possible relevant labels for this issue please view the list of labels: https://gitlab.com/BuildStream/buildstream/labels)
|
|
37 |
+ |
|
36 | 38 |
/label ~bug
|
... | ... | @@ -15,3 +15,5 @@ |
15 | 15 |
[//]: # (Acceptance criteria should follow the S.M.A.R.T. principle https://en.wikipedia.org/wiki/SMART_criteria )
|
16 | 16 |
|
17 | 17 |
----
|
18 |
+[//]: # (To review information about possible relevant labels for this issue please view the list of labels: https://gitlab.com/BuildStream/buildstream/labels)
|
|
19 |
+ |
... | ... | @@ -138,6 +138,9 @@ buildstream 1.3.1 |
138 | 138 |
o BREAKING CHANGE: Symlinks are no longer resolved during staging and absolute
|
139 | 139 |
symlinks are now preserved instead of being converted to relative symlinks.
|
140 | 140 |
|
141 |
+ o BREAKING CHANGE: Overlap whitelists now require absolute paths. This allows
|
|
142 |
+ use of variables such as %{prefix} and matches the documentation.
|
|
143 |
+ |
|
141 | 144 |
|
142 | 145 |
=================
|
143 | 146 |
buildstream 1.1.5
|
... | ... | @@ -236,7 +236,8 @@ class Project(): |
236 | 236 |
'artifacts', 'options',
|
237 | 237 |
'fail-on-overlap', 'shell', 'fatal-warnings',
|
238 | 238 |
'ref-storage', 'sandbox', 'mirrors', 'remote-execution',
|
239 |
- 'sources', '(@)'
|
|
239 |
+ 'sources', '(@)',
|
|
240 |
+ 'supress-deprecation-warnings'
|
|
240 | 241 |
])
|
241 | 242 |
|
242 | 243 |
# create_element()
|
... | ... | @@ -483,6 +484,10 @@ class Project(): |
483 | 484 |
|
484 | 485 |
self._validate_node(pre_config_node)
|
485 | 486 |
|
487 |
+ # Load plugin deprecation warning supression config
|
|
488 |
+ self.supressed_deprecation_warnings = _yaml.node_get(
|
|
489 |
+ pre_config_node, list, 'supress-deprecation-warnings', default_value=[])
|
|
490 |
+ |
|
486 | 491 |
# FIXME:
|
487 | 492 |
#
|
488 | 493 |
# Performing this check manually in the absense
|
... | ... | @@ -940,7 +940,7 @@ def node_sanitize(node): |
940 | 940 |
return [node_sanitize(elt) for elt in node]
|
941 | 941 |
|
942 | 942 |
# Finally ChainMap and dict, and other Mappings need special handling
|
943 |
- if node_type in (dict, ChainMap) or isinstance(node, collections.Mapping):
|
|
943 |
+ if node_type in (dict, ChainMap) or isinstance(node, collections.abc.Mapping):
|
|
944 | 944 |
result = SanitizedDict()
|
945 | 945 |
|
946 | 946 |
key_list = [key for key, _ in node_items(node)]
|
... | ... | @@ -1620,12 +1620,12 @@ class Element(Plugin): |
1620 | 1620 |
self.__dynamic_public = _yaml.node_copy(self.__public)
|
1621 | 1621 |
|
1622 | 1622 |
# Call the abstract plugin methods
|
1623 |
- try:
|
|
1624 |
- # Step 1 - Configure
|
|
1625 |
- self.__configure_sandbox(sandbox)
|
|
1626 |
- # Step 2 - Stage
|
|
1627 |
- self.stage(sandbox)
|
|
1628 | 1623 |
|
1624 |
+ # Step 1 - Configure
|
|
1625 |
+ self.__configure_sandbox(sandbox)
|
|
1626 |
+ # Step 2 - Stage
|
|
1627 |
+ self.stage(sandbox)
|
|
1628 |
+ try:
|
|
1629 | 1629 |
if self.__batch_prepare_assemble:
|
1630 | 1630 |
cm = sandbox.batch(self.__batch_prepare_assemble_flags,
|
1631 | 1631 |
collect=self.__batch_prepare_assemble_collect)
|
... | ... | @@ -2609,7 +2609,7 @@ class Element(Plugin): |
2609 | 2609 |
if include_file and not exclude_file:
|
2610 | 2610 |
yield filename.lstrip(os.sep)
|
2611 | 2611 |
|
2612 |
- def __file_is_whitelisted(self, pattern):
|
|
2612 |
+ def __file_is_whitelisted(self, path):
|
|
2613 | 2613 |
# Considered storing the whitelist regex for re-use, but public data
|
2614 | 2614 |
# can be altered mid-build.
|
2615 | 2615 |
# Public data is not guaranteed to stay the same for the duration of
|
... | ... | @@ -2621,7 +2621,7 @@ class Element(Plugin): |
2621 | 2621 |
whitelist_expressions = [utils._glob2re(self.__variables.subst(exp.strip())) for exp in whitelist]
|
2622 | 2622 |
_expression_ = ('^(?:' + '|'.join(whitelist_expressions) + ')$')
|
2623 | 2623 |
self.__whitelist_regex = re.compile(_expression_)
|
2624 |
- return self.__whitelist_regex.match(pattern)
|
|
2624 |
+ return self.__whitelist_regex.match(os.path.join(os.sep, path))
|
|
2625 | 2625 |
|
2626 | 2626 |
# __extract():
|
2627 | 2627 |
#
|
... | ... | @@ -164,6 +164,23 @@ class Plugin(): |
164 | 164 |
core format version :ref:`core format version <project_format_version>`.
|
165 | 165 |
"""
|
166 | 166 |
|
167 |
+ BST_PLUGIN_DEPRECATED = False
|
|
168 |
+ """True if this element plugin has been deprecated.
|
|
169 |
+ |
|
170 |
+ If this is set to true, the plugin will call self.emit_deprecation_warning()
|
|
171 |
+ on instantiation. Plugin authors may override this method to provide
|
|
172 |
+ custom deprecation warnings
|
|
173 |
+ |
|
174 |
+ """
|
|
175 |
+ |
|
176 |
+ BST_PLUGIN_DEPRECATION_MESSAGE = ""
|
|
177 |
+ """ The message printed when this element shows a deprecation warning.
|
|
178 |
+ |
|
179 |
+ This should be set if BST_PLUGIN_DEPRECATED is True and should direct the user
|
|
180 |
+ to the deprecated plug-in's replacement.
|
|
181 |
+ |
|
182 |
+ """
|
|
183 |
+ |
|
167 | 184 |
def __init__(self, name, context, project, provenance, type_tag):
|
168 | 185 |
|
169 | 186 |
self.name = name
|
... | ... | @@ -188,6 +205,12 @@ class Plugin(): |
188 | 205 |
self.__kind = modulename.split('.')[-1]
|
189 | 206 |
self.debug("Created: {}".format(self))
|
190 | 207 |
|
208 |
+ # If this plugin has been deprecated, emit a warning.
|
|
209 |
+ if self.BST_PLUGIN_DEPRECATED and not self.__deprecation_warning_silcenced():
|
|
210 |
+ detail = "Using deprecated plugin {}: {}".format(self.__kind,
|
|
211 |
+ self.BST_PLUGIN_DEPRECATION_MESSAGE)
|
|
212 |
+ self.__message(MessageType.WARN, detail)
|
|
213 |
+ |
|
191 | 214 |
def __del__(self):
|
192 | 215 |
# Dont send anything through the Message() pipeline at destruction time,
|
193 | 216 |
# any subsequent lookup of plugin by unique id would raise KeyError.
|
... | ... | @@ -767,6 +790,19 @@ class Plugin(): |
767 | 790 |
else:
|
768 | 791 |
return self.name
|
769 | 792 |
|
793 |
+ def __deprecation_warning_silcenced(self):
|
|
794 |
+ if not self.BST_PLUGIN_DEPRECATED:
|
|
795 |
+ return False
|
|
796 |
+ else:
|
|
797 |
+ silenced_warnings = set()
|
|
798 |
+ project= self.__project
|
|
799 |
+ plugin_overrides = {**project.element_overrides, **project.source_overrides}
|
|
800 |
+ |
|
801 |
+ for key, value in self.node_items(plugin_overrides):
|
|
802 |
+ if value.get('supress-deprecation-warnings', False):
|
|
803 |
+ silenced_warnings.add(key)
|
|
804 |
+ |
|
805 |
+ return self.get_kind() in silenced_warnings
|
|
770 | 806 |
|
771 | 807 |
# Hold on to a lookup table by counter of all instantiated plugins.
|
772 | 808 |
# We use this to send the id back from child processes so we can lookup
|
... | ... | @@ -96,7 +96,7 @@ def parse_tag(tag): |
96 | 96 |
def guess_version(release):
|
97 | 97 |
try:
|
98 | 98 |
tags_output = subprocess.check_output(['git', 'tag'])
|
99 |
- except CalledProcessError:
|
|
99 |
+ except subprocess.CalledProcessError:
|
|
100 | 100 |
return (0, 0, 0)
|
101 | 101 |
|
102 | 102 |
# Parse the `git tag` output into a list of integer tuples
|
1 |
+#
|
|
2 |
+# Copyright (C) 2018 Codethink Limited
|
|
3 |
+# Copyright (C) 2018 Bloomberg Finance LP
|
|
4 |
+#
|
|
5 |
+# This program is free software; you can redistribute it and/or
|
|
6 |
+# modify it under the terms of the GNU Lesser General Public
|
|
7 |
+# License as published by the Free Software Foundation; either
|
|
8 |
+# version 2 of the License, or (at your option) any later version.
|
|
9 |
+#
|
|
10 |
+# This library is distributed in the hope that it will be useful,
|
|
11 |
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
12 |
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
13 |
+# Lesser General Public License for more details.
|
|
14 |
+#
|
|
15 |
+# You should have received a copy of the GNU Lesser General Public
|
|
16 |
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
|
|
17 |
+#
|
|
18 |
+# Authors: Richard Maw <richard maw codethink co uk>
|
|
19 |
+#
|
|
20 |
+ |
|
21 |
+import os
|
|
22 |
+import pytest
|
|
23 |
+ |
|
24 |
+from buildstream.plugintestutils import cli
|
|
25 |
+ |
|
26 |
+ |
|
27 |
+# Project directory
|
|
28 |
+DATA_DIR = os.path.join(
|
|
29 |
+ os.path.dirname(os.path.realpath(__file__)),
|
|
30 |
+ "project",
|
|
31 |
+)
|
|
32 |
+ |
|
33 |
+ |
|
34 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
35 |
+def test_artifact_log(cli, tmpdir, datafiles):
|
|
36 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
37 |
+ |
|
38 |
+ # Get the cache key of our test element
|
|
39 |
+ result = cli.run(project=project, silent=True, args=[
|
|
40 |
+ '--no-colors',
|
|
41 |
+ 'show', '--deps', 'none', '--format', '%{full-key}',
|
|
42 |
+ 'target.bst'
|
|
43 |
+ ])
|
|
44 |
+ key = result.output.strip()
|
|
45 |
+ |
|
46 |
+ # Ensure we have an artifact to read
|
|
47 |
+ result = cli.run(project=project, args=['build', 'target.bst'])
|
|
48 |
+ assert result.exit_code == 0
|
|
49 |
+ |
|
50 |
+ # Read the log via the element name
|
|
51 |
+ result = cli.run(project=project, args=['artifact', 'log', 'target.bst'])
|
|
52 |
+ assert result.exit_code == 0
|
|
53 |
+ log = result.output
|
|
54 |
+ |
|
55 |
+ # Read the log via the key
|
|
56 |
+ result = cli.run(project=project, args=['artifact', 'log', 'test/target/' + key])
|
|
57 |
+ assert result.exit_code == 0
|
|
58 |
+ assert log == result.output
|
|
59 |
+ |
|
60 |
+ # Read the log via glob
|
|
61 |
+ result = cli.run(project=project, args=['artifact', 'log', 'test/target/*'])
|
|
62 |
+ assert result.exit_code == 0
|
|
63 |
+ # The artifact is cached under both a strong key and a weak key
|
|
64 |
+ assert (log + log) == result.output
|
... | ... | @@ -10,4 +10,4 @@ sources: |
10 | 10 |
public:
|
11 | 11 |
bst:
|
12 | 12 |
overlap-whitelist:
|
13 |
- - "file*"
|
|
13 |
+ - "/file*"
|
... | ... | @@ -8,9 +8,9 @@ sources: |
8 | 8 |
- kind: local
|
9 | 9 |
path: "b"
|
10 | 10 |
variables:
|
11 |
- FILE: file
|
|
11 |
+ FILE: /file
|
|
12 | 12 |
public:
|
13 | 13 |
bst:
|
14 | 14 |
overlap-whitelist:
|
15 |
- - file2
|
|
15 |
+ - /file2
|
|
16 | 16 |
- "%{FILE}3"
|
... | ... | @@ -8,4 +8,4 @@ sources: |
8 | 8 |
public:
|
9 | 9 |
bst:
|
10 | 10 |
overlap-whitelist:
|
11 |
- - "file*"
|
|
11 |
+ - "/file*"
|
... | ... | @@ -37,40 +37,6 @@ DATA_DIR = os.path.join( |
37 | 37 |
)
|
38 | 38 |
|
39 | 39 |
|
40 |
-@pytest.mark.integration
|
|
41 |
-@pytest.mark.datafiles(DATA_DIR)
|
|
42 |
-def test_artifact_log(cli, tmpdir, datafiles):
|
|
43 |
- project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
44 |
- |
|
45 |
- # Get the cache key of our test element
|
|
46 |
- result = cli.run(project=project, silent=True, args=[
|
|
47 |
- '--no-colors',
|
|
48 |
- 'show', '--deps', 'none', '--format', '%{full-key}',
|
|
49 |
- 'base.bst'
|
|
50 |
- ])
|
|
51 |
- key = result.output.strip()
|
|
52 |
- |
|
53 |
- # Ensure we have an artifact to read
|
|
54 |
- result = cli.run(project=project, args=['build', 'base.bst'])
|
|
55 |
- assert result.exit_code == 0
|
|
56 |
- |
|
57 |
- # Read the log via the element name
|
|
58 |
- result = cli.run(project=project, args=['artifact', 'log', 'base.bst'])
|
|
59 |
- assert result.exit_code == 0
|
|
60 |
- log = result.output
|
|
61 |
- |
|
62 |
- # Read the log via the key
|
|
63 |
- result = cli.run(project=project, args=['artifact', 'log', 'test/base/' + key])
|
|
64 |
- assert result.exit_code == 0
|
|
65 |
- assert log == result.output
|
|
66 |
- |
|
67 |
- # Read the log via glob
|
|
68 |
- result = cli.run(project=project, args=['artifact', 'log', 'test/base/*'])
|
|
69 |
- assert result.exit_code == 0
|
|
70 |
- # The artifact is cached under both a strong key and a weak key
|
|
71 |
- assert (log + log) == result.output
|
|
72 |
- |
|
73 |
- |
|
74 | 40 |
# A test to capture the integration of the cachebuildtrees
|
75 | 41 |
# behaviour, which by default is to include the buildtree
|
76 | 42 |
# content of an element on caching.
|
... | ... | @@ -212,6 +212,7 @@ def test_host_files_expand_environ(cli, tmpdir, datafiles, path): |
212 | 212 |
# Test that bind mounts defined in project.conf dont mount in isolation
|
213 | 213 |
@pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
|
214 | 214 |
@pytest.mark.datafiles(DATA_DIR)
|
215 |
+@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
|
|
215 | 216 |
def test_isolated_no_mount(cli, tmpdir, datafiles, path):
|
216 | 217 |
project = os.path.join(datafiles.dirname, datafiles.basename)
|
217 | 218 |
ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
|
... | ... | @@ -226,6 +227,8 @@ def test_isolated_no_mount(cli, tmpdir, datafiles, path): |
226 | 227 |
}
|
227 | 228 |
})
|
228 | 229 |
assert result.exit_code != 0
|
230 |
+ assert path in result.stderr
|
|
231 |
+ assert 'No such file or directory' in result.stderr
|
|
229 | 232 |
|
230 | 233 |
|
231 | 234 |
# Test that we warn about non-existing files on the host if the mount is not
|
1 |
+import pytest
|
|
2 |
+import tempfile
|
|
3 |
+import os
|
|
4 |
+from buildstream.plugintestutils import cli
|
|
5 |
+from buildstream import _yaml
|
|
6 |
+import buildstream.plugins.elements.manual
|
|
7 |
+ |
|
8 |
+ |
|
9 |
+DATA_DIR = os.path.join(
|
|
10 |
+ os.path.dirname(os.path.realpath(__file__)),
|
|
11 |
+ "project"
|
|
12 |
+)
|
|
13 |
+ |
|
14 |
+_DEPRECATION_MESSAGE = "Here is some detail."
|
|
15 |
+_DEPRECATION_WARNING = "Using deprecated plugin deprecated_plugin: {}".format(_DEPRECATION_MESSAGE)
|
|
16 |
+ |
|
17 |
+ |
|
18 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
19 |
+def test_deprecation_warning_present(cli, datafiles):
|
|
20 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
21 |
+ result = cli.run(project=project, args=['show', 'deprecated.bst'])
|
|
22 |
+ result.assert_success()
|
|
23 |
+ assert _DEPRECATION_WARNING in result.stderr
|
|
24 |
+ |
|
25 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
26 |
+def test_supress_deprecation_warning(cli, datafiles):
|
|
27 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
28 |
+ result = cli.run(project=project, args=['show', 'manual.bst'])
|
|
29 |
+ |
|
30 |
+ element_overrides = "elements:\n" \
|
|
31 |
+ " deprecated_plugin:\n" \
|
|
32 |
+ " supress-deprecation-warnings : True\n"
|
|
33 |
+ |
|
34 |
+ project_conf = os.path.join(project, 'project.conf')
|
|
35 |
+ with open(project_conf, 'a') as f:
|
|
36 |
+ f.write(element_overrides)
|
|
37 |
+ |
|
38 |
+ result = cli.run(project=project, args=['show', 'deprecated.bst'])
|
|
39 |
+ result.assert_success()
|
|
40 |
+ assert _DEPRECATION_WARNING not in result.stderr
|
1 |
+kind: deprecated_plugin
|
|
\ No newline at end of file |
1 |
+from buildstream import BuildElement, SandboxFlags
|
|
2 |
+ |
|
3 |
+ |
|
4 |
+class DeprecatedPlugin(BuildElement):
|
|
5 |
+ BST_PLUGIN_DEPRECATED = True
|
|
6 |
+ BST_PLUGIN_DEPRECATION_MESSAGE = "Here is some detail."
|
|
7 |
+ |
|
8 |
+ |
|
9 |
+# Plugin entry point
|
|
10 |
+def setup():
|
|
11 |
+ return DeprecatedPlugin
|
1 |
+# Deprecated-plugin build element does not provide any default
|
|
2 |
+# build commands
|
|
3 |
+config:
|
|
4 |
+ |
|
5 |
+ # Commands for configuring the software
|
|
6 |
+ #
|
|
7 |
+ configure-commands: []
|
|
8 |
+ |
|
9 |
+ # Commands for building the software
|
|
10 |
+ #
|
|
11 |
+ build-commands: []
|
|
12 |
+ |
|
13 |
+ # Commands for installing the software into a
|
|
14 |
+ # destination folder
|
|
15 |
+ #
|
|
16 |
+ install-commands: []
|
|
17 |
+ |
|
18 |
+ # Commands for stripping installed binaries
|
|
19 |
+ #
|
|
20 |
+ strip-commands:
|
|
21 |
+ - |
|
|
22 |
+ %{strip-binaries}
|
|
\ No newline at end of file |
1 |
+# Unique project name
|
|
2 |
+name: deprecation-warnings
|
|
3 |
+ |
|
4 |
+# Required BuildStream format version
|
|
5 |
+format-version: 20
|
|
6 |
+ |
|
7 |
+# Subdirectory where elements are stored
|
|
8 |
+element-path: elements
|
|
9 |
+ |
|
10 |
+plugins:
|
|
11 |
+ |
|
12 |
+- origin: local
|
|
13 |
+ path: plugins/elements
|
|
14 |
+ elements:
|
|
15 |
+ deprecated_plugin: 0
|