Chandan Singh pushed to branch chandan/junction-dependency-format at BuildStream / buildstream
Commits:
-
1b02c550
by Chandan Singh at 2019-02-12T12:05:14Z
-
43c98533
by Chandan Singh at 2019-02-12T14:34:35Z
-
beeab6ea
by Chandan Singh at 2019-02-12T14:34:35Z
5 changed files:
- NEWS
- buildstream/_loader/types.py
- buildstream/_versions.py
- doc/source/format_declaring.rst
- tests/frontend/buildcheckout.py
Changes:
... | ... | @@ -70,6 +70,9 @@ buildstream 1.3.1 |
70 | 70 |
to avoid having to specify the dependency type for every entry in
|
71 | 71 |
'depends'.
|
72 | 72 |
|
73 |
+ o Elements may now specify cross-junction dependencies as simple strings
|
|
74 |
+ using the format '{junction-name}:{element-name}'.
|
|
75 |
+ |
|
73 | 76 |
o Source plugins may now request access access to previous during track and
|
74 | 77 |
fetch by setting `BST_REQUIRES_PREVIOUS_SOURCES_TRACK` and/or
|
75 | 78 |
`BST_REQUIRES_PREVIOUS_SOURCES_FETCH` attributes.
|
... | ... | @@ -17,6 +17,8 @@ |
17 | 17 |
# Authors:
|
18 | 18 |
# Tristan Van Berkom <tristan vanberkom codethink co uk>
|
19 | 19 |
|
20 |
+from .._exceptions import LoadError, LoadErrorReason
|
|
21 |
+ |
|
20 | 22 |
|
21 | 23 |
# Symbol():
|
22 | 24 |
#
|
... | ... | @@ -56,8 +58,22 @@ class Symbol(): |
56 | 58 |
# dependency was declared
|
57 | 59 |
#
|
58 | 60 |
class Dependency():
|
59 |
- def __init__(self, name,
|
|
61 |
+ def __init__(self, filename,
|
|
60 | 62 |
dep_type=None, junction=None, provenance=None):
|
63 |
+ # Attempt to split name if no junction was specified explicitly
|
|
64 |
+ if not junction:
|
|
65 |
+ junction_path = filename.rsplit(":", 1)
|
|
66 |
+ name = junction_path[-1]
|
|
67 |
+ junction = None if len(junction_path) == 1 else junction_path[-2]
|
|
68 |
+ else:
|
|
69 |
+ name = filename
|
|
70 |
+ |
|
71 |
+ # Name of the element should not contain any `:` characters at this stage
|
|
72 |
+ if ':' in name:
|
|
73 |
+ raise LoadError(LoadErrorReason.INVALID_DATA,
|
|
74 |
+ "Element {} contains `:` in its name. "
|
|
75 |
+ "Recursive lookups for cross-junction elements is not allowed.".format(filename))
|
|
76 |
+ |
|
61 | 77 |
self.name = name
|
62 | 78 |
self.dep_type = dep_type
|
63 | 79 |
self.junction = junction
|
... | ... | @@ -23,7 +23,7 @@ |
23 | 23 |
# This version is bumped whenever enhancements are made
|
24 | 24 |
# to the `project.conf` format or the core element format.
|
25 | 25 |
#
|
26 |
-BST_FORMAT_VERSION = 21
|
|
26 |
+BST_FORMAT_VERSION = 22
|
|
27 | 27 |
|
28 | 28 |
|
29 | 29 |
# The base BuildStream artifact version
|
... | ... | @@ -401,6 +401,40 @@ Attributes: |
401 | 401 |
The ``junction`` attribute is available since :ref:`format version 1 <project_format_version>`
|
402 | 402 |
|
403 | 403 |
|
404 |
+Cross-junction dependencies
|
|
405 |
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
406 |
+As mentioned above, cross-junction dependencies can be specified using the
|
|
407 |
+``junction`` attribute. They can also be expressed as simple strings as a
|
|
408 |
+convenience shorthand. You can refer to cross-junction elements using the
|
|
409 |
+syntax ``{junction-name}:{element-name}``.
|
|
410 |
+ |
|
411 |
+For example, the following is logically same as the example above:
|
|
412 |
+ |
|
413 |
+.. code:: yaml
|
|
414 |
+ |
|
415 |
+ build-depends:
|
|
416 |
+ - baseproject.bst:foo.bst
|
|
417 |
+ |
|
418 |
+Similarly, you can also refer to cross-junction elements via the ``filename``
|
|
419 |
+attribute, like so:
|
|
420 |
+ |
|
421 |
+.. code:: yaml
|
|
422 |
+ |
|
423 |
+ depends:
|
|
424 |
+ - filename: baseproject.bst:foo.bst
|
|
425 |
+ type: build
|
|
426 |
+ |
|
427 |
+.. note::
|
|
428 |
+ |
|
429 |
+ Recursive lookups are not allowed using this syntax. If a filename
|
|
430 |
+ contains more than one ``:`` (colon) character, BuildStream will raise
|
|
431 |
+ an error.
|
|
432 |
+ |
|
433 |
+.. note::
|
|
434 |
+ |
|
435 |
+ This shorthand is available since :ref:`format version 22 <project_format_version>`
|
|
436 |
+ |
|
437 |
+ |
|
404 | 438 |
.. _format_dependencies_types:
|
405 | 439 |
|
406 | 440 |
Dependency types
|
... | ... | @@ -709,3 +709,114 @@ def test_build_checkout_cross_junction(datafiles, cli, tmpdir): |
709 | 709 |
|
710 | 710 |
filename = os.path.join(checkout, 'etc', 'animal.conf')
|
711 | 711 |
assert os.path.exists(filename)
|
712 |
+ |
|
713 |
+ |
|
714 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
715 |
+def test_build_junction_short_notation(cli, tmpdir, datafiles):
|
|
716 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
717 |
+ subproject_path = os.path.join(project, 'files', 'sub-project')
|
|
718 |
+ junction_path = os.path.join(project, 'elements', 'junction.bst')
|
|
719 |
+ element_path = os.path.join(project, 'elements', 'junction-dep.bst')
|
|
720 |
+ workspace = os.path.join(cli.directory, 'workspace')
|
|
721 |
+ checkout = os.path.join(cli.directory, 'checkout')
|
|
722 |
+ |
|
723 |
+ # Create a repo to hold the subproject and generate a junction element for it
|
|
724 |
+ ref = generate_junction(tmpdir, subproject_path, junction_path)
|
|
725 |
+ |
|
726 |
+ # Create a stack element to depend on a cross junction element, using
|
|
727 |
+ # colon (:) as the separator
|
|
728 |
+ element = {
|
|
729 |
+ 'kind': 'stack',
|
|
730 |
+ 'depends': ['junction.bst:import-etc.bst']
|
|
731 |
+ }
|
|
732 |
+ _yaml.dump(element, element_path)
|
|
733 |
+ |
|
734 |
+ # Now try to build it, this should automatically result in fetching
|
|
735 |
+ # the junction itself at load time.
|
|
736 |
+ result = cli.run(project=project, args=['build', 'junction-dep.bst'])
|
|
737 |
+ result.assert_success()
|
|
738 |
+ |
|
739 |
+ # Assert that it's cached now
|
|
740 |
+ assert cli.get_element_state(project, 'junction-dep.bst') == 'cached'
|
|
741 |
+ |
|
742 |
+ # Now check it out
|
|
743 |
+ result = cli.run(project=project, args=[
|
|
744 |
+ 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout
|
|
745 |
+ ])
|
|
746 |
+ result.assert_success()
|
|
747 |
+ |
|
748 |
+ # Assert the content of /etc/animal.conf
|
|
749 |
+ filename = os.path.join(checkout, 'etc', 'animal.conf')
|
|
750 |
+ assert os.path.exists(filename)
|
|
751 |
+ with open(filename, 'r') as f:
|
|
752 |
+ contents = f.read()
|
|
753 |
+ assert contents == 'animal=Pony\n'
|
|
754 |
+ |
|
755 |
+ |
|
756 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
757 |
+def test_build_junction_short_notation_filename(cli, tmpdir, datafiles):
|
|
758 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
759 |
+ subproject_path = os.path.join(project, 'files', 'sub-project')
|
|
760 |
+ junction_path = os.path.join(project, 'elements', 'junction.bst')
|
|
761 |
+ element_path = os.path.join(project, 'elements', 'junction-dep.bst')
|
|
762 |
+ checkout = os.path.join(cli.directory, 'checkout')
|
|
763 |
+ |
|
764 |
+ # Create a repo to hold the subproject and generate a junction element for it
|
|
765 |
+ ref = generate_junction(tmpdir, subproject_path, junction_path)
|
|
766 |
+ |
|
767 |
+ # Create a stack element to depend on a cross junction element, using
|
|
768 |
+ # colon (:) as the separator
|
|
769 |
+ element = {
|
|
770 |
+ 'kind': 'stack',
|
|
771 |
+ 'depends': [{'filename': 'junction.bst:import-etc.bst'}]
|
|
772 |
+ }
|
|
773 |
+ _yaml.dump(element, element_path)
|
|
774 |
+ |
|
775 |
+ # Now try to build it, this should automatically result in fetching
|
|
776 |
+ # the junction itself at load time.
|
|
777 |
+ result = cli.run(project=project, args=['build', 'junction-dep.bst'])
|
|
778 |
+ result.assert_success()
|
|
779 |
+ |
|
780 |
+ # Assert that it's cached now
|
|
781 |
+ assert cli.get_element_state(project, 'junction-dep.bst') == 'cached'
|
|
782 |
+ |
|
783 |
+ # Now check it out
|
|
784 |
+ result = cli.run(project=project, args=[
|
|
785 |
+ 'artifact', 'checkout', 'junction-dep.bst', '--directory', checkout
|
|
786 |
+ ])
|
|
787 |
+ result.assert_success()
|
|
788 |
+ |
|
789 |
+ # Assert the content of /etc/animal.conf
|
|
790 |
+ filename = os.path.join(checkout, 'etc', 'animal.conf')
|
|
791 |
+ assert os.path.exists(filename)
|
|
792 |
+ with open(filename, 'r') as f:
|
|
793 |
+ contents = f.read()
|
|
794 |
+ assert contents == 'animal=Pony\n'
|
|
795 |
+ |
|
796 |
+ |
|
797 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
798 |
+def test_build_junction_short_notation_no_recurse(cli, tmpdir, datafiles):
|
|
799 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
800 |
+ subproject_path = os.path.join(project, 'files', 'sub-project')
|
|
801 |
+ junction_path = os.path.join(project, 'elements', 'junction.bst')
|
|
802 |
+ element_path = os.path.join(project, 'elements', 'junction-dep.bst')
|
|
803 |
+ checkout = os.path.join(cli.directory, 'checkout')
|
|
804 |
+ |
|
805 |
+ # Create a repo to hold the subproject and generate a junction element for it
|
|
806 |
+ ref = generate_junction(tmpdir, subproject_path, junction_path)
|
|
807 |
+ |
|
808 |
+ # Create a stack element to depend on a cross junction element, using
|
|
809 |
+ # colon (:) as the separator
|
|
810 |
+ element = {
|
|
811 |
+ 'kind': 'stack',
|
|
812 |
+ 'depends': [{
|
|
813 |
+ 'filename': 'junction.bst:import-etc.bst',
|
|
814 |
+ 'junction': 'junction.bst',
|
|
815 |
+ }]
|
|
816 |
+ }
|
|
817 |
+ _yaml.dump(element, element_path)
|
|
818 |
+ |
|
819 |
+ # Now try to build it, this should fail as recursive lookups for
|
|
820 |
+ # cross-junction elements is not allowed.
|
|
821 |
+ result = cli.run(project=project, args=['build', 'junction-dep.bst'])
|
|
822 |
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
|