James Ennis pushed to branch jennis/filter-docs at BuildStream / buildstream
Commits:
-
5df4105a
by Angelos Evripiotis at 2019-01-28T10:13:40Z
-
9c981eff
by Angelos Evripiotis at 2019-01-28T10:17:57Z
-
bef80291
by Angelos Evripiotis at 2019-01-28T10:19:17Z
-
564cb245
by Angelos Evripiotis at 2019-01-28T11:36:16Z
-
0ead1f1c
by James Ennis at 2019-01-28T15:07:36Z
-
3c860c34
by James Ennis at 2019-01-28T15:09:27Z
15 changed files:
- NEWS
- buildstream/_gitsourcebase.py
- buildstream/_loader/loader.py
- buildstream/_pipeline.py
- buildstream/plugins/elements/filter.py
- buildstream/plugins/elements/filter.yaml
- buildstream/plugins/elements/junction.py
- buildstream/plugins/sources/bzr.py
- buildstream/plugins/sources/deb.py
- buildstream/plugins/sources/git.py
- buildstream/plugins/sources/remote.py
- buildstream/plugins/sources/tar.py
- buildstream/plugins/sources/zip.py
- doc/source/format_project_refs.rst
- tests/frontend/completions.py
Changes:
... | ... | @@ -20,7 +20,7 @@ buildstream 1.3.1 |
20 | 20 |
an element's sources and generated build scripts you can do the command
|
21 | 21 |
`bst source-checkout --include-build-scripts --tar foo.bst some-file.tar`
|
22 | 22 |
|
23 |
- o BREAKING CHANGE: `bst track` and `bst fetch` commands are now osbolete.
|
|
23 |
+ o BREAKING CHANGE: `bst track` and `bst fetch` commands are now obsolete.
|
|
24 | 24 |
Their functionality is provided by `bst source track` and
|
25 | 25 |
`bst source fetch` respectively.
|
26 | 26 |
|
... | ... | @@ -605,7 +605,7 @@ class _GitSourceBase(Source): |
605 | 605 |
detail = "The ref provided for the element does not exist locally " + \
|
606 | 606 |
"in the provided track branch / tag '{}'.\n".format(self.tracking) + \
|
607 | 607 |
"You may wish to track the element to update the ref from '{}' ".format(self.tracking) + \
|
608 |
- "with `bst track`,\n" + \
|
|
608 |
+ "with `bst source track`,\n" + \
|
|
609 | 609 |
"or examine the upstream at '{}' for the specific ref.".format(self.mirror.url)
|
610 | 610 |
|
611 | 611 |
self.warn("{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n"
|
... | ... | @@ -557,7 +557,7 @@ class Loader(): |
557 | 557 |
ticker(filename, 'Fetching subproject from {} source'.format(source.get_kind()))
|
558 | 558 |
source._fetch(sources[0:idx])
|
559 | 559 |
else:
|
560 |
- detail = "Try fetching the project with `bst fetch {}`".format(filename)
|
|
560 |
+ detail = "Try fetching the project with `bst source fetch {}`".format(filename)
|
|
561 | 561 |
raise LoadError(LoadErrorReason.SUBPROJECT_FETCH_NEEDED,
|
562 | 562 |
"Subproject fetch needed for junction: {}".format(filename),
|
563 | 563 |
detail=detail)
|
... | ... | @@ -565,7 +565,7 @@ class Loader(): |
565 | 565 |
# Handle the case where a subproject has no ref
|
566 | 566 |
#
|
567 | 567 |
elif source.get_consistency() == Consistency.INCONSISTENT:
|
568 |
- detail = "Try tracking the junction element with `bst track {}`".format(filename)
|
|
568 |
+ detail = "Try tracking the junction element with `bst source track {}`".format(filename)
|
|
569 | 569 |
raise LoadError(LoadErrorReason.SUBPROJECT_INCONSISTENT,
|
570 | 570 |
"Subproject has no ref for junction: {}".format(filename),
|
571 | 571 |
detail=detail)
|
... | ... | @@ -373,7 +373,7 @@ class Pipeline(): |
373 | 373 |
if source._get_consistency() == Consistency.INCONSISTENT:
|
374 | 374 |
detail += " {} is missing ref\n".format(source)
|
375 | 375 |
detail += '\n'
|
376 |
- detail += "Try tracking these elements first with `bst track`\n"
|
|
376 |
+ detail += "Try tracking these elements first with `bst source track`\n"
|
|
377 | 377 |
|
378 | 378 |
raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline")
|
379 | 379 |
|
... | ... | @@ -406,7 +406,7 @@ class Pipeline(): |
406 | 406 |
if source._get_consistency() != Consistency.CACHED:
|
407 | 407 |
detail += " {}\n".format(source)
|
408 | 408 |
detail += '\n'
|
409 |
- detail += "Try fetching these elements first with `bst fetch`,\n" + \
|
|
409 |
+ detail += "Try fetching these elements first with `bst source fetch`,\n" + \
|
|
410 | 410 |
"or run this command with `--fetch` option\n"
|
411 | 411 |
|
412 | 412 |
raise PipelineError("Uncached sources", detail=detail, reason="uncached-sources")
|
... | ... | @@ -20,25 +20,127 @@ |
20 | 20 |
"""
|
21 | 21 |
filter - Extract a subset of files from another element
|
22 | 22 |
=======================================================
|
23 |
-This filters another element by producing an output that is a subset of
|
|
24 |
-the filtered element.
|
|
23 |
+Filter another element by producing an output that is a subset of
|
|
24 |
+the parent element's output. Subsets are defined by the parent element's
|
|
25 |
+:ref:`split rules <public_split_rules>`.
|
|
25 | 26 |
|
26 |
-To specify the element to filter, specify it as the one and only build
|
|
27 |
-dependency to filter. See :ref:`Dependencies <format_dependencies>`
|
|
28 |
-for what dependencies are and how to specify them.
|
|
27 |
+Overview
|
|
28 |
+--------
|
|
29 |
+A filter element must have exactly one *build* dependency, where said
|
|
30 |
+dependency is the 'parent' element which we would like to filter.
|
|
31 |
+Runtime dependencies may also be specified, which can be useful to propagate
|
|
32 |
+forward from this filter element onto its reverse dependencies.
|
|
33 |
+See :ref:`Dependencies <format_dependencies>` to see how we specify dependencies.
|
|
29 | 34 |
|
30 |
-Dependencies aside from the filtered element may be specified, but
|
|
31 |
-they must be runtime dependencies only. This can be useful to propagate
|
|
32 |
-runtime dependencies forward from this filter element onto its reverse
|
|
33 |
-dependencies.
|
|
35 |
+When workspaces are opened, closed or reset on a filter element, or this
|
|
36 |
+element is tracked, the filter element will transparently pass on the command
|
|
37 |
+to its parent element (the sole build-dependency).
|
|
34 | 38 |
|
35 |
-When workspaces are opened, closed or reset on this element, or this
|
|
36 |
-element is tracked, instead of erroring due to a lack of sources, this
|
|
37 |
-element will transparently pass on the command to its sole build-dependency.
|
|
39 |
+Example
|
|
40 |
+-------
|
|
41 |
+Consider a simple import element, ``import.bst`` which imports the local files
|
|
42 |
+'foo', 'bar' and 'baz' (each stored in ``files/``, relative to the project's root):
|
|
38 | 43 |
|
39 |
-The default configuration and possible options are as such:
|
|
40 |
- .. literalinclude:: ../../../buildstream/plugins/elements/filter.yaml
|
|
41 |
- :language: yaml
|
|
44 |
+.. code:: yaml
|
|
45 |
+ |
|
46 |
+ kind: import
|
|
47 |
+ |
|
48 |
+ # Specify sources to import
|
|
49 |
+ sources:
|
|
50 |
+ - kind: local
|
|
51 |
+ path: files
|
|
52 |
+ |
|
53 |
+ # Specify public domain data, visible to other elements
|
|
54 |
+ public:
|
|
55 |
+ bst:
|
|
56 |
+ split-rules:
|
|
57 |
+ foo:
|
|
58 |
+ - /foo
|
|
59 |
+ bar:
|
|
60 |
+ - /bar
|
|
61 |
+ |
|
62 |
+.. note::
|
|
63 |
+ |
|
64 |
+ We can make an element's metadata visible to all reverse dependencies by making use
|
|
65 |
+ of the ``public:`` field. See the :ref:`public data documentation <format_public>`
|
|
66 |
+ for more information.
|
|
67 |
+ |
|
68 |
+In this example, ``import.bst`` will serve as the 'parent' of the filter element, thus
|
|
69 |
+its output will be filtered. It is important to understand that the artifact of the
|
|
70 |
+above element will contain the files: 'foo', 'bar' and 'baz'.
|
|
71 |
+ |
|
72 |
+Now, to produce an element whose artifact contains the file 'foo', and exlusively 'foo',
|
|
73 |
+we can define the following filter, ``filter-foo.bst``:
|
|
74 |
+ |
|
75 |
+.. code:: yaml
|
|
76 |
+ |
|
77 |
+ kind: filter
|
|
78 |
+ |
|
79 |
+ # Declare the sole build-dependency of the filter element
|
|
80 |
+ depends:
|
|
81 |
+ - filename: import.bst
|
|
82 |
+ type: build
|
|
83 |
+ |
|
84 |
+ # Declare a list of domains to include in the filter's artifact
|
|
85 |
+ config:
|
|
86 |
+ include:
|
|
87 |
+ - foo
|
|
88 |
+ |
|
89 |
+.. note::
|
|
90 |
+ |
|
91 |
+ We can also specify build-dependencies with a 'build-depends' field which has been
|
|
92 |
+ available since :ref:`format version 14 <project_format_version>`. See the
|
|
93 |
+ :ref:`Build-Depends documentation <format_build_depends>` for more detail.
|
|
94 |
+ |
|
95 |
+It should be noted that an 'empty' ``include:`` list would, by default, include all
|
|
96 |
+split-rules specified in the parent element, which, in this example, would be the
|
|
97 |
+files 'foo' and 'bar' (the file 'baz' was not covered by any split rules).
|
|
98 |
+ |
|
99 |
+Equally, we can use the ``exclude:`` statement to create the same artifact (which
|
|
100 |
+only contains the file 'foo') by declaring the following element, ``exclude-bar.bst``:
|
|
101 |
+ |
|
102 |
+.. code:: yaml
|
|
103 |
+ |
|
104 |
+ kind: filter
|
|
105 |
+ |
|
106 |
+ # Declare the sole build-dependency of the filter element
|
|
107 |
+ depends:
|
|
108 |
+ - filename: import.bst
|
|
109 |
+ type: build
|
|
110 |
+ |
|
111 |
+ # Declare a list of domains to exclude in the filter's artifact
|
|
112 |
+ config:
|
|
113 |
+ exclude:
|
|
114 |
+ - bar
|
|
115 |
+ |
|
116 |
+In addition to the ``include:`` and ``exclude:`` fields, there exists an ``include-orphans:``
|
|
117 |
+(Boolean) field, which defaults to ``False``. This will determine whether to include files
|
|
118 |
+which are not present in the 'split-rules'. For example, if we wanted to filter out all files
|
|
119 |
+which are not included as split rules we can define the following element, ``filter-misc.bst``:
|
|
120 |
+ |
|
121 |
+.. code:: yaml
|
|
122 |
+ |
|
123 |
+ kind: filter
|
|
124 |
+ |
|
125 |
+ # Declare the sole build-dependency of the filter element
|
|
126 |
+ depends:
|
|
127 |
+ - filename: import.bst
|
|
128 |
+ type: build
|
|
129 |
+ |
|
130 |
+ # Filter out all files which are not declared as split rules
|
|
131 |
+ config:
|
|
132 |
+ exclude:
|
|
133 |
+ - foo
|
|
134 |
+ - bar
|
|
135 |
+ include-orphans: True
|
|
136 |
+ |
|
137 |
+The artifact of ``filter-misc.bst`` will only contain the file 'baz'.
|
|
138 |
+ |
|
139 |
+Below is more information regarding the the default configurations and possible options
|
|
140 |
+of the filter element:
|
|
141 |
+ |
|
142 |
+.. literalinclude:: ../../../buildstream/plugins/elements/filter.yaml
|
|
143 |
+ :language: yaml
|
|
42 | 144 |
"""
|
43 | 145 |
|
44 | 146 |
from buildstream import Element, ElementError, Scope
|
... | ... | @@ -2,20 +2,21 @@ |
2 | 2 |
# Filter element configuration
|
3 | 3 |
config:
|
4 | 4 |
|
5 |
- # A list of domains to include from each artifact, as
|
|
6 |
- # they were defined in the element's 'split-rules'.
|
|
5 |
+ # A list of domains to include in each artifact, as
|
|
6 |
+ # they were defined as public data in the parent
|
|
7 |
+ # element's 'split-rules'.
|
|
7 | 8 |
#
|
8 | 9 |
# Since domains can be added, it is not an error to
|
9 | 10 |
# specify domains which may not exist for all of the
|
10 | 11 |
# elements in this composition.
|
11 | 12 |
#
|
12 | 13 |
# The default empty list indicates that all domains
|
13 |
- # from each dependency should be included.
|
|
14 |
+ # of the parent's artifact should be included.
|
|
14 | 15 |
#
|
15 | 16 |
include: []
|
16 | 17 |
|
17 | 18 |
# A list of domains to exclude from each artifact, as
|
18 |
- # they were defined in the element's 'split-rules'.
|
|
19 |
+ # they were defined in the parent element's 'split-rules'.
|
|
19 | 20 |
#
|
20 | 21 |
# In the case that a file is spoken for by a domain
|
21 | 22 |
# in the 'include' list and another in the 'exclude'
|
... | ... | @@ -23,7 +24,7 @@ config: |
23 | 24 |
exclude: []
|
24 | 25 |
|
25 | 26 |
# Whether to include orphan files which are not
|
26 |
- # included by any of the 'split-rules' present on
|
|
27 |
- # a given element.
|
|
27 |
+ # included by any of the 'split-rules' present in
|
|
28 |
+ # the parent element.
|
|
28 | 29 |
#
|
29 | 30 |
include-orphans: False
|
... | ... | @@ -79,7 +79,7 @@ depend on a junction element itself. |
79 | 79 |
|
80 | 80 |
Therefore, if you require the most up-to-date version of a subproject,
|
81 | 81 |
you must explicitly track the junction element by invoking:
|
82 |
- `bst track JUNCTION_ELEMENT`.
|
|
82 |
+ `bst source track JUNCTION_ELEMENT`.
|
|
83 | 83 |
|
84 | 84 |
Furthermore, elements within the subproject are also not tracked by default.
|
85 | 85 |
For this, we must specify the `--track-cross-junctions` option. This option
|
... | ... | @@ -93,7 +93,7 @@ cached yet. However, they can be fetched explicitly: |
93 | 93 |
|
94 | 94 |
.. code::
|
95 | 95 |
|
96 |
- bst fetch junction.bst
|
|
96 |
+ bst source fetch junction.bst
|
|
97 | 97 |
|
98 | 98 |
Other commands such as ``bst build`` implicitly fetch junction sources.
|
99 | 99 |
|
... | ... | @@ -146,7 +146,7 @@ class JunctionElement(Element): |
146 | 146 |
|
147 | 147 |
def get_unique_key(self):
|
148 | 148 |
# Junctions do not produce artifacts. get_unique_key() implementation
|
149 |
- # is still required for `bst fetch`.
|
|
149 |
+ # is still required for `bst source fetch`.
|
|
150 | 150 |
return 1
|
151 | 151 |
|
152 | 152 |
def configure_sandbox(self, sandbox):
|
... | ... | @@ -46,7 +46,7 @@ bzr - stage files from a bazaar repository |
46 | 46 |
# but revisions on a branch are of the form
|
47 | 47 |
# <revision-branched-from>.<branch-number>.<revision-since-branching>
|
48 | 48 |
# e.g. 6622.1.6.
|
49 |
- # The ref must be specified to build, and 'bst track' will update the
|
|
49 |
+ # The ref must be specified to build, and 'bst source track' will update the
|
|
50 | 50 |
# revision number to the one on the tip of the branch specified in 'track'.
|
51 | 51 |
ref: 6622
|
52 | 52 |
|
... | ... | @@ -34,7 +34,7 @@ deb - stage files from .deb packages |
34 | 34 |
kind: deb
|
35 | 35 |
|
36 | 36 |
# Specify the deb url. Using an alias defined in your project
|
37 |
- # configuration is encouraged. 'bst track' will update the
|
|
37 |
+ # configuration is encouraged. 'bst source track' will update the
|
|
38 | 38 |
# sha256sum in 'ref' to the downloaded file's sha256sum.
|
39 | 39 |
url: upstream:foo.deb
|
40 | 40 |
|
... | ... | @@ -112,7 +112,7 @@ git - stage files from a git repository |
112 | 112 |
# o Enable `track-tags` feature
|
113 | 113 |
# o Set the `track` parameter to the desired commit sha which
|
114 | 114 |
# the current `ref` points to
|
115 |
- # o Run `bst track` for these elements, this will result in
|
|
115 |
+ # o Run `bst source track` for these elements, this will result in
|
|
116 | 116 |
# populating the `tags` portion of the refs without changing
|
117 | 117 |
# the refs
|
118 | 118 |
# o Restore the `track` parameter to the branches which you have
|
... | ... | @@ -37,7 +37,7 @@ remote - stage files from remote urls |
37 | 37 |
# executable: true
|
38 | 38 |
|
39 | 39 |
# Specify the url. Using an alias defined in your project
|
40 |
- # configuration is encouraged. 'bst track' will update the
|
|
40 |
+ # configuration is encouraged. 'bst source track' will update the
|
|
41 | 41 |
# sha256sum in 'ref' to the downloaded file's sha256sum.
|
42 | 42 |
url: upstream:foo
|
43 | 43 |
|
... | ... | @@ -33,7 +33,7 @@ tar - stage files from tar archives |
33 | 33 |
kind: tar
|
34 | 34 |
|
35 | 35 |
# Specify the tar url. Using an alias defined in your project
|
36 |
- # configuration is encouraged. 'bst track' will update the
|
|
36 |
+ # configuration is encouraged. 'bst source track' will update the
|
|
37 | 37 |
# sha256sum in 'ref' to the downloaded file's sha256sum.
|
38 | 38 |
url: upstream:foo.tar
|
39 | 39 |
|
... | ... | @@ -29,7 +29,7 @@ zip - stage files from zip archives |
29 | 29 |
kind: zip
|
30 | 30 |
|
31 | 31 |
# Specify the zip url. Using an alias defined in your project
|
32 |
- # configuration is encouraged. 'bst track' will update the
|
|
32 |
+ # configuration is encouraged. 'bst source track' will update the
|
|
33 | 33 |
# sha256sum in 'ref' to the downloaded file's sha256sum.
|
34 | 34 |
url: upstream:foo.zip
|
35 | 35 |
|
... | ... | @@ -21,9 +21,9 @@ When a ``project.refs`` file is in use, any source references found |
21 | 21 |
in the :ref:`inline source declarations <format_sources>` are considered
|
22 | 22 |
invalid and will be ignored, and a warning will be emitted for them.
|
23 | 23 |
|
24 |
-When ``bst track`` is run for your project, the ``project.refs`` file
|
|
24 |
+When ``bst source track`` is run for your project, the ``project.refs`` file
|
|
25 | 25 |
will be updated instead of the inline source declarations. In the absence
|
26 |
-of a ``project.refs`` file, ``bst track`` will create one automatically
|
|
26 |
+of a ``project.refs`` file, ``bst source track`` will create one automatically
|
|
27 | 27 |
with the tracking results.
|
28 | 28 |
|
29 | 29 |
An interesting property of ``project.refs`` is that it allows for
|
... | ... | @@ -160,7 +160,7 @@ def test_options(cli, cmd, word_idx, expected): |
160 | 160 |
('bst show --deps b', 3, ['build ']),
|
161 | 161 |
('bst show --deps=b', 2, ['build ']),
|
162 | 162 |
('bst show --deps r', 3, ['run ']),
|
163 |
- ('bst track --deps ', 3, ['all ', 'none ']),
|
|
163 |
+ ('bst source track --deps ', 4, ['all ', 'none ']),
|
|
164 | 164 |
])
|
165 | 165 |
def test_option_choice(cli, cmd, word_idx, expected):
|
166 | 166 |
assert_completion(cli, cmd, word_idx, expected)
|