Chandan Singh pushed to branch chandan/junction-dependency-format at BuildStream / buildstream
Commits:
-
ad3a85b8
by Chandan Singh at 2019-02-18T18:25:57Z
-
ab6ab8fc
by Chandan Singh at 2019-02-18T18:25:57Z
-
50027bbb
by Chandan Singh at 2019-02-18T18:25:57Z
7 changed files:
- NEWS
- buildstream/_loader/loadelement.py
- buildstream/_loader/types.py
- buildstream/_versions.py
- buildstream/plugins/elements/junction.py
- doc/source/format_declaring.rst
- tests/frontend/buildcheckout.py
Changes:
... | ... | @@ -70,6 +70,9 @@ buildstream 1.3.1 |
70 | 70 |
to avoid having to specify the dependency type for every entry in
|
71 | 71 |
'depends'.
|
72 | 72 |
|
73 |
+ o Elements may now specify cross-junction dependencies as simple strings
|
|
74 |
+ using the format '{junction-name}:{element-name}'.
|
|
75 |
+ |
|
73 | 76 |
o Source plugins may now request access access to previous during track and
|
74 | 77 |
fetch by setting `BST_REQUIRES_PREVIOUS_SOURCES_TRACK` and/or
|
75 | 78 |
`BST_REQUIRES_PREVIOUS_SOURCES_FETCH` attributes.
|
... | ... | @@ -18,7 +18,6 @@ |
18 | 18 |
# Tristan Van Berkom <tristan vanberkom codethink co uk>
|
19 | 19 |
|
20 | 20 |
# System imports
|
21 |
-from collections.abc import Mapping
|
|
22 | 21 |
from itertools import count
|
23 | 22 |
|
24 | 23 |
from pyroaring import BitMap, FrozenBitMap # pylint: disable=no-name-in-module
|
... | ... | @@ -174,38 +173,7 @@ def _extract_depends_from_node(node, *, key=None): |
174 | 173 |
|
175 | 174 |
for index, dep in enumerate(depends):
|
176 | 175 |
dep_provenance = _yaml.node_get_provenance(node, key=key, indices=[index])
|
177 |
- |
|
178 |
- if isinstance(dep, str):
|
|
179 |
- dependency = Dependency(dep, provenance=dep_provenance, dep_type=default_dep_type)
|
|
180 |
- |
|
181 |
- elif isinstance(dep, Mapping):
|
|
182 |
- if default_dep_type:
|
|
183 |
- _yaml.node_validate(dep, ['filename', 'junction'])
|
|
184 |
- dep_type = default_dep_type
|
|
185 |
- else:
|
|
186 |
- _yaml.node_validate(dep, ['filename', 'type', 'junction'])
|
|
187 |
- |
|
188 |
- # Make type optional, for this we set it to None
|
|
189 |
- dep_type = _yaml.node_get(dep, str, Symbol.TYPE, default_value=None)
|
|
190 |
- if dep_type is None or dep_type == Symbol.ALL:
|
|
191 |
- dep_type = None
|
|
192 |
- elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]:
|
|
193 |
- provenance = _yaml.node_get_provenance(dep, key=Symbol.TYPE)
|
|
194 |
- raise LoadError(LoadErrorReason.INVALID_DATA,
|
|
195 |
- "{}: Dependency type '{}' is not 'build', 'runtime' or 'all'"
|
|
196 |
- .format(provenance, dep_type))
|
|
197 |
- |
|
198 |
- filename = _yaml.node_get(dep, str, Symbol.FILENAME)
|
|
199 |
- junction = _yaml.node_get(dep, str, Symbol.JUNCTION, default_value=None)
|
|
200 |
- dependency = Dependency(filename,
|
|
201 |
- dep_type=dep_type,
|
|
202 |
- junction=junction,
|
|
203 |
- provenance=dep_provenance)
|
|
204 |
- |
|
205 |
- else:
|
|
206 |
- raise LoadError(LoadErrorReason.INVALID_DATA,
|
|
207 |
- "{}: Dependency is not specified as a string or a dictionary".format(dep_provenance))
|
|
208 |
- |
|
176 |
+ dependency = Dependency(dep, dep_provenance, default_dep_type=default_dep_type)
|
|
209 | 177 |
output_deps.append(dependency)
|
210 | 178 |
|
211 | 179 |
# Now delete the field, we dont want it anymore
|
... | ... | @@ -17,6 +17,11 @@ |
17 | 17 |
# Authors:
|
18 | 18 |
# Tristan Van Berkom <tristan vanberkom codethink co uk>
|
19 | 19 |
|
20 |
+from collections.abc import Mapping
|
|
21 |
+ |
|
22 |
+from .._exceptions import LoadError, LoadErrorReason
|
|
23 |
+from .. import _yaml
|
|
24 |
+ |
|
20 | 25 |
|
21 | 26 |
# Symbol():
|
22 | 27 |
#
|
... | ... | @@ -56,9 +61,56 @@ class Symbol(): |
56 | 61 |
# dependency was declared
|
57 | 62 |
#
|
58 | 63 |
class Dependency():
|
59 |
- def __init__(self, name,
|
|
60 |
- dep_type=None, junction=None, provenance=None):
|
|
61 |
- self.name = name
|
|
62 |
- self.dep_type = dep_type
|
|
63 |
- self.junction = junction
|
|
64 |
+ def __init__(self, dep, provenance, default_dep_type=None):
|
|
64 | 65 |
self.provenance = provenance
|
66 |
+ |
|
67 |
+ if isinstance(dep, str):
|
|
68 |
+ self.name = dep
|
|
69 |
+ self.dep_type = default_dep_type
|
|
70 |
+ self.junction = None
|
|
71 |
+ |
|
72 |
+ elif isinstance(dep, Mapping):
|
|
73 |
+ if default_dep_type:
|
|
74 |
+ _yaml.node_validate(dep, ['filename', 'junction'])
|
|
75 |
+ dep_type = default_dep_type
|
|
76 |
+ else:
|
|
77 |
+ _yaml.node_validate(dep, ['filename', 'type', 'junction'])
|
|
78 |
+ |
|
79 |
+ # Make type optional, for this we set it to None
|
|
80 |
+ dep_type = _yaml.node_get(dep, str, Symbol.TYPE, default_value=None)
|
|
81 |
+ if dep_type is None or dep_type == Symbol.ALL:
|
|
82 |
+ dep_type = None
|
|
83 |
+ elif dep_type not in [Symbol.BUILD, Symbol.RUNTIME]:
|
|
84 |
+ provenance = _yaml.node_get_provenance(dep, key=Symbol.TYPE)
|
|
85 |
+ raise LoadError(LoadErrorReason.INVALID_DATA,
|
|
86 |
+ "{}: Dependency type '{}' is not 'build', 'runtime' or 'all'"
|
|
87 |
+ .format(provenance, dep_type))
|
|
88 |
+ |
|
89 |
+ self.name = _yaml.node_get(dep, str, Symbol.FILENAME)
|
|
90 |
+ self.dep_type = dep_type
|
|
91 |
+ self.junction = _yaml.node_get(dep, str, Symbol.JUNCTION, default_value=None)
|
|
92 |
+ |
|
93 |
+ else:
|
|
94 |
+ raise LoadError(LoadErrorReason.INVALID_DATA,
|
|
95 |
+ "{}: Dependency is not specified as a string or a dictionary".format(provenance))
|
|
96 |
+ |
|
97 |
+ # `:` characters are not allowed in filename if a junction was
|
|
98 |
+ # explicitly specified
|
|
99 |
+ if self.junction and ':' in self.name:
|
|
100 |
+ raise LoadError(LoadErrorReason.INVALID_DATA,
|
|
101 |
+ "{}: Dependency {} contains `:` in its name. "
|
|
102 |
+ "`:` characters are not allowed in filename when "
|
|
103 |
+ "junction attribute is specified.".format(self.provenance, self.name))
|
|
104 |
+ |
|
105 |
+ # Name of the element should never contain more than one `:` characters
|
|
106 |
+ if self.name.count(':') > 1:
|
|
107 |
+ raise LoadError(LoadErrorReason.INVALID_DATA,
|
|
108 |
+ "{}: Dependency {} contains multiple `:` in its name. "
|
|
109 |
+ "Recursive lookups for cross-junction elements is not "
|
|
110 |
+ "allowed.".format(self.provenance, self.name))
|
|
111 |
+ |
|
112 |
+ # Attempt to split name if no junction was specified explicitly
|
|
113 |
+ if not self.junction:
|
|
114 |
+ junction_path = self.name.rsplit(":", 1)
|
|
115 |
+ self.name = junction_path[-1]
|
|
116 |
+ self.junction = None if len(junction_path) == 1 else junction_path[-2]
|
... | ... | @@ -23,7 +23,7 @@ |
23 | 23 |
# This version is bumped whenever enhancements are made
|
24 | 24 |
# to the `project.conf` format or the core element format.
|
25 | 25 |
#
|
26 |
-BST_FORMAT_VERSION = 21
|
|
26 |
+BST_FORMAT_VERSION = 22
|
|
27 | 27 |
|
28 | 28 |
|
29 | 29 |
# The base BuildStream artifact version
|
... | ... | @@ -109,6 +109,8 @@ Junctions can configure options of the linked project. Options are never |
109 | 109 |
implicitly inherited across junctions, however, variables can be used to
|
110 | 110 |
explicitly assign the same value to a subproject option.
|
111 | 111 |
|
112 |
+.. _core_junction_nested:
|
|
113 |
+ |
|
112 | 114 |
Nested Junctions
|
113 | 115 |
----------------
|
114 | 116 |
Junctions can be nested. That is, subprojects are allowed to have junctions on
|
... | ... | @@ -401,6 +401,41 @@ Attributes: |
401 | 401 |
The ``junction`` attribute is available since :ref:`format version 1 <project_format_version>`
|
402 | 402 |
|
403 | 403 |
|
404 |
+Cross-junction dependencies
|
|
405 |
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
406 |
+As mentioned above, cross-junction dependencies can be specified using the
|
|
407 |
+``junction`` attribute. They can also be expressed as simple strings as a
|
|
408 |
+convenience shorthand. You can refer to cross-junction elements using the
|
|
409 |
+syntax ``{junction-name}:{element-name}``.
|
|
410 |
+ |
|
411 |
+For example, the following is logically same as the example above:
|
|
412 |
+ |
|
413 |
+.. code:: yaml
|
|
414 |
+ |
|
415 |
+ build-depends:
|
|
416 |
+ - baseproject.bst:foo.bst
|
|
417 |
+ |
|
418 |
+Similarly, you can also refer to cross-junction elements via the ``filename``
|
|
419 |
+attribute, like so:
|
|
420 |
+ |
|
421 |
+.. code:: yaml
|
|
422 |
+ |
|
423 |
+ depends:
|
|
424 |
+ - filename: baseproject.bst:foo.bst
|
|
425 |
+ type: build
|
|
426 |
+ |
|
427 |
+.. note::
|
|
428 |
+ |
|
429 |
+ BuildStream does not allow recursice lookups for junction elements. If a
|
|
430 |
+ filename contains more than one ``:`` (colon) character, an error will be
|
|
431 |
+ raised. See :ref:`nested junctions <core_junction_nested>` for more details
|
|
432 |
+ on nested junctions.
|
|
433 |
+ |
|
434 |
+.. note::
|
|
435 |
+ |
|
436 |
+ This shorthand is available since :ref:`format version 22 <project_format_version>`
|
|
437 |
+ |
|
438 |
+ |
|
404 | 439 |
.. _format_dependencies_types:
|
405 | 440 |
|
406 | 441 |
Dependency types
|
... | ... | @@ -729,3 +729,139 @@ def test_build_checkout_cross_junction(datafiles, cli, tmpdir): |
729 | 729 |
|
730 | 730 |
filename = os.path.join(checkout, 'etc', 'animal.conf')
|
731 | 731 |
assert os.path.exists(filename)
|
732 |
+ |
|
733 |
+ |
|
734 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
735 |
+def test_build_junction_short_notation(cli, tmpdir, datafiles):
|
|
736 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
737 |
+ subproject_path = os.path.join(project, 'files', 'sub-project')
|
|
738 |
+ junction_path = os.path.join(project, 'elements', 'junction.bst')
|
|
739 |
+ element_path = os.path.join(project, 'elements', 'junction-dep.bst')
|
|
740 |
+ workspace = os.path.join(cli.directory, 'workspace')
|
|
741 |
+ checkout = os.path.join(cli.directory, 'checkout')
|
|
742 |
+ |
|
743 |
+ # Create a repo to hold the subproject and generate a junction element for it
|
|
744 |
+ ref = generate_junction(tmpdir, subproject_path, junction_path)
|
|
745 |
+ |
|
746 |
+ # Create a stack element to depend on a cross junction element, using
|
|
747 |
+ # colon (:) as the separator
|
|
748 |
+ element = {
|
|
749 |
+ 'kind': 'stack',
|
|
750 |
+ 'depends': ['junction.bst:import-etc.bst']
|
|
751 |
+ }
|
|
752 |
+ _yaml.dump(element, element_path)
|
|
753 |
+ |
|
754 |
+ # Now try to build it, this should automatically result in fetching
|
|
755 |
+ # the junction itself at load time.
|
|
756 |
+ result = cli.run(project=project, args=['build', 'junction-dep.bst'])
|
|
757 |
+ result.assert_success()
|
|
758 |
+ |
|
759 |
+ # Assert that it's cached now
|
|
760 |
+ assert cli.get_element_state(project, 'junction-dep.bst') == 'cached'
|
|
761 |
+ |
|
762 |
+ # Now check it out
|
|
763 |
+ result = cli.run(project=project, args=[
|
|
764 |
+ 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout
|
|
765 |
+ ])
|
|
766 |
+ result.assert_success()
|
|
767 |
+ |
|
768 |
+ # Assert the content of /etc/animal.conf
|
|
769 |
+ filename = os.path.join(checkout, 'etc', 'animal.conf')
|
|
770 |
+ assert os.path.exists(filename)
|
|
771 |
+ with open(filename, 'r') as f:
|
|
772 |
+ contents = f.read()
|
|
773 |
+ assert contents == 'animal=Pony\n'
|
|
774 |
+ |
|
775 |
+ |
|
776 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
777 |
+def test_build_junction_short_notation_filename(cli, tmpdir, datafiles):
|
|
778 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
779 |
+ subproject_path = os.path.join(project, 'files', 'sub-project')
|
|
780 |
+ junction_path = os.path.join(project, 'elements', 'junction.bst')
|
|
781 |
+ element_path = os.path.join(project, 'elements', 'junction-dep.bst')
|
|
782 |
+ checkout = os.path.join(cli.directory, 'checkout')
|
|
783 |
+ |
|
784 |
+ # Create a repo to hold the subproject and generate a junction element for it
|
|
785 |
+ ref = generate_junction(tmpdir, subproject_path, junction_path)
|
|
786 |
+ |
|
787 |
+ # Create a stack element to depend on a cross junction element, using
|
|
788 |
+ # colon (:) as the separator
|
|
789 |
+ element = {
|
|
790 |
+ 'kind': 'stack',
|
|
791 |
+ 'depends': [{'filename': 'junction.bst:import-etc.bst'}]
|
|
792 |
+ }
|
|
793 |
+ _yaml.dump(element, element_path)
|
|
794 |
+ |
|
795 |
+ # Now try to build it, this should automatically result in fetching
|
|
796 |
+ # the junction itself at load time.
|
|
797 |
+ result = cli.run(project=project, args=['build', 'junction-dep.bst'])
|
|
798 |
+ result.assert_success()
|
|
799 |
+ |
|
800 |
+ # Assert that it's cached now
|
|
801 |
+ assert cli.get_element_state(project, 'junction-dep.bst') == 'cached'
|
|
802 |
+ |
|
803 |
+ # Now check it out
|
|
804 |
+ result = cli.run(project=project, args=[
|
|
805 |
+ 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout
|
|
806 |
+ ])
|
|
807 |
+ result.assert_success()
|
|
808 |
+ |
|
809 |
+ # Assert the content of /etc/animal.conf
|
|
810 |
+ filename = os.path.join(checkout, 'etc', 'animal.conf')
|
|
811 |
+ assert os.path.exists(filename)
|
|
812 |
+ with open(filename, 'r') as f:
|
|
813 |
+ contents = f.read()
|
|
814 |
+ assert contents == 'animal=Pony\n'
|
|
815 |
+ |
|
816 |
+ |
|
817 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
818 |
+def test_build_junction_short_notation_with_junction(cli, tmpdir, datafiles):
|
|
819 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
820 |
+ subproject_path = os.path.join(project, 'files', 'sub-project')
|
|
821 |
+ junction_path = os.path.join(project, 'elements', 'junction.bst')
|
|
822 |
+ element_path = os.path.join(project, 'elements', 'junction-dep.bst')
|
|
823 |
+ checkout = os.path.join(cli.directory, 'checkout')
|
|
824 |
+ |
|
825 |
+ # Create a repo to hold the subproject and generate a junction element for it
|
|
826 |
+ ref = generate_junction(tmpdir, subproject_path, junction_path)
|
|
827 |
+ |
|
828 |
+ # Create a stack element to depend on a cross junction element, using
|
|
829 |
+ # colon (:) as the separator
|
|
830 |
+ element = {
|
|
831 |
+ 'kind': 'stack',
|
|
832 |
+ 'depends': [{
|
|
833 |
+ 'filename': 'junction.bst:import-etc.bst',
|
|
834 |
+ 'junction': 'junction.bst',
|
|
835 |
+ }]
|
|
836 |
+ }
|
|
837 |
+ _yaml.dump(element, element_path)
|
|
838 |
+ |
|
839 |
+ # Now try to build it, this should fail as filenames should not contain
|
|
840 |
+ # `:` when junction is explicity specified
|
|
841 |
+ result = cli.run(project=project, args=['build', 'junction-dep.bst'])
|
|
842 |
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
|
|
843 |
+ |
|
844 |
+ |
|
845 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
846 |
+def test_build_junction_short_notation_with_junction(cli, tmpdir, datafiles):
|
|
847 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
848 |
+ subproject_path = os.path.join(project, 'files', 'sub-project')
|
|
849 |
+ junction_path = os.path.join(project, 'elements', 'junction.bst')
|
|
850 |
+ element_path = os.path.join(project, 'elements', 'junction-dep.bst')
|
|
851 |
+ checkout = os.path.join(cli.directory, 'checkout')
|
|
852 |
+ |
|
853 |
+ # Create a repo to hold the subproject and generate a junction element for it
|
|
854 |
+ ref = generate_junction(tmpdir, subproject_path, junction_path)
|
|
855 |
+ |
|
856 |
+ # Create a stack element to depend on a cross junction element, using
|
|
857 |
+ # colon (:) as the separator
|
|
858 |
+ element = {
|
|
859 |
+ 'kind': 'stack',
|
|
860 |
+ 'depends': ['junction.bst:import-etc.bst:foo.bst']
|
|
861 |
+ }
|
|
862 |
+ _yaml.dump(element, element_path)
|
|
863 |
+ |
|
864 |
+ # Now try to build it, this should fail as recursive lookups for
|
|
865 |
+ # cross-junction elements is not allowed.
|
|
866 |
+ result = cli.run(project=project, args=['build', 'junction-dep.bst'])
|
|
867 |
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
|