Chandan Singh pushed to branch chandan/dot-graph at BuildStream / buildstream
Commits:
-
118644b2
by Tom Pollard at 2019-02-13T13:37:32Z
-
b546bac1
by Tom Pollard at 2019-02-13T13:37:32Z
-
d2029444
by Tom Pollard at 2019-02-13T13:37:32Z
-
54ec032a
by Jürg Billeter at 2019-02-13T14:50:15Z
-
fe119c4b
by Chandan Singh at 2019-02-13T16:12:59Z
10 changed files:
- NEWS
- buildstream/_context.py
- buildstream/_frontend/app.py
- buildstream/_frontend/cli.py
- buildstream/data/userconfig.yaml
- buildstream/element.py
- + contrib/bst-graph
- tests/frontend/completions.py
- tests/integration/artifact.py
- tests/integration/build-tree.py → tests/integration/shellbuildtrees.py
Changes:
... | ... | @@ -126,6 +126,14 @@ buildstream 1.3.1 |
126 | 126 |
Providing a remote will limit build's pull/push remote actions to the given
|
127 | 127 |
remote specifically, ignoring those defined via user or project configuration.
|
128 | 128 |
|
129 |
+ o Artifacts can now be cached explicitly with an empty `build tree` when built.
|
|
130 |
+ Element types without a build-root were already cached with an empty build tree
|
|
131 |
+ directory, this can now be extended to all or successful artifacts to save on cache
|
|
132 |
+ overheads. The cli main option '--cache-buildtrees' or the user configuration cache
|
|
133 |
+ group option 'cache-buildtrees' can be set as 'always', 'failure' or 'never', with
|
|
134 |
+ the default being always. Note, as the cache-key for the artifact is independant of
|
|
135 |
+ the cached build tree input it will remain unaltered, however the availbility of the
|
|
136 |
+ build tree content may differ.
|
|
129 | 137 |
|
130 | 138 |
=================
|
131 | 139 |
buildstream 1.1.5
|
... | ... | @@ -121,6 +121,9 @@ class Context(): |
121 | 121 |
# Whether or not to attempt to pull build trees globally
|
122 | 122 |
self.pull_buildtrees = None
|
123 | 123 |
|
124 |
+ # Whether or not to cache build trees on artifact creation
|
|
125 |
+ self.cache_buildtrees = None
|
|
126 |
+ |
|
124 | 127 |
# Boolean, whether we double-check with the user that they meant to
|
125 | 128 |
# close the workspace when they're using it to access the project.
|
126 | 129 |
self.prompt_workspace_close_project_inaccessible = None
|
... | ... | @@ -201,7 +204,7 @@ class Context(): |
201 | 204 |
# our artifactdir - the artifactdir may not have been created
|
202 | 205 |
# yet.
|
203 | 206 |
cache = _yaml.node_get(defaults, Mapping, 'cache')
|
204 |
- _yaml.node_validate(cache, ['quota', 'pull-buildtrees'])
|
|
207 |
+ _yaml.node_validate(cache, ['quota', 'pull-buildtrees', 'cache-buildtrees'])
|
|
205 | 208 |
|
206 | 209 |
self.config_cache_quota = _yaml.node_get(cache, str, 'quota')
|
207 | 210 |
|
... | ... | @@ -213,6 +216,10 @@ class Context(): |
213 | 216 |
# Load pull build trees configuration
|
214 | 217 |
self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
|
215 | 218 |
|
219 |
+ # Load cache build trees configuration
|
|
220 |
+ self.cache_buildtrees = _node_get_option_str(
|
|
221 |
+ cache, 'cache-buildtrees', ['always', 'failure', 'never'])
|
|
222 |
+ |
|
216 | 223 |
# Load logging config
|
217 | 224 |
logging = _yaml.node_get(defaults, Mapping, 'logging')
|
218 | 225 |
_yaml.node_validate(logging, [
|
... | ... | @@ -183,7 +183,8 @@ class App(): |
183 | 183 |
'builders': 'sched_builders',
|
184 | 184 |
'pushers': 'sched_pushers',
|
185 | 185 |
'network_retries': 'sched_network_retries',
|
186 |
- 'pull_buildtrees': 'pull_buildtrees'
|
|
186 |
+ 'pull_buildtrees': 'pull_buildtrees',
|
|
187 |
+ 'cache_buildtrees': 'cache_buildtrees'
|
|
187 | 188 |
}
|
188 | 189 |
for cli_option, context_attr in override_map.items():
|
189 | 190 |
option_value = self._main_options.get(cli_option)
|
... | ... | @@ -251,6 +251,9 @@ def print_version(ctx, param, value): |
251 | 251 |
help="The mirror to fetch from first, before attempting other mirrors")
|
252 | 252 |
@click.option('--pull-buildtrees', is_flag=True, default=None,
|
253 | 253 |
help="Include an element's build tree when pulling remote element artifacts")
|
254 |
+@click.option('--cache-buildtrees', default=None,
|
|
255 |
+ type=click.Choice(['always', 'failure', 'never']),
|
|
256 |
+ help="Cache artifact build tree content on creation")
|
|
254 | 257 |
@click.pass_context
|
255 | 258 |
def cli(context, **kwargs):
|
256 | 259 |
"""Build and manipulate BuildStream projects
|
... | ... | @@ -572,7 +575,8 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, command) |
572 | 575 |
if choice != "never":
|
573 | 576 |
use_buildtree = choice
|
574 | 577 |
|
575 |
- if use_buildtree and not element._cached_success():
|
|
578 |
+ # Raise warning if the element is cached in a failed state
|
|
579 |
+ if use_buildtree and element._cached_failure():
|
|
576 | 580 |
click.echo("WARNING: using a buildtree from a failed build.", err=True)
|
577 | 581 |
|
578 | 582 |
try:
|
... | ... | @@ -41,6 +41,15 @@ cache: |
41 | 41 |
# Whether to pull build trees when downloading element artifacts
|
42 | 42 |
pull-buildtrees: False
|
43 | 43 |
|
44 |
+ # Whether to cache build trees on artifact creation:
|
|
45 |
+ #
|
|
46 |
+ # always - Always cache artifact build tree content
|
|
47 |
+ # failure - Only cache build trees of failed builds
|
|
48 |
+ # never - Don't cache artifact build tree content
|
|
49 |
+ #
|
|
50 |
+ cache-buildtrees: always
|
|
51 |
+ |
|
52 |
+ |
|
44 | 53 |
#
|
45 | 54 |
# Scheduler
|
46 | 55 |
#
|
... | ... | @@ -1457,6 +1457,9 @@ class Element(Plugin): |
1457 | 1457 |
elif usebuildtree:
|
1458 | 1458 |
artifact_base, _ = self.__extract()
|
1459 | 1459 |
import_dir = os.path.join(artifact_base, 'buildtree')
|
1460 |
+ if not os.listdir(import_dir):
|
|
1461 |
+ detail = "Element type either does not expect a buildtree or it was explictily cached without one."
|
|
1462 |
+ self.warn("WARNING: {} Artifact contains an empty buildtree".format(self.name), detail=detail)
|
|
1460 | 1463 |
else:
|
1461 | 1464 |
# No workspace or cached buildtree, stage source directly
|
1462 | 1465 |
for source in self.sources():
|
... | ... | @@ -1663,6 +1666,8 @@ class Element(Plugin): |
1663 | 1666 |
# No collect directory existed
|
1664 | 1667 |
collectvdir = None
|
1665 | 1668 |
|
1669 |
+ context = self._get_context()
|
|
1670 |
+ |
|
1666 | 1671 |
# Create artifact directory structure
|
1667 | 1672 |
assembledir = os.path.join(rootdir, 'artifact')
|
1668 | 1673 |
filesdir = os.path.join(assembledir, 'files')
|
... | ... | @@ -1680,20 +1685,30 @@ class Element(Plugin): |
1680 | 1685 |
if collect is not None and collectvdir is not None:
|
1681 | 1686 |
collectvdir.export_files(filesdir, can_link=True)
|
1682 | 1687 |
|
1683 |
- try:
|
|
1684 |
- sandbox_vroot = sandbox.get_virtual_directory()
|
|
1685 |
- sandbox_build_dir = sandbox_vroot.descend(
|
|
1686 |
- self.get_variable('build-root').lstrip(os.sep).split(os.sep))
|
|
1687 |
- # Hard link files from build-root dir to buildtreedir directory
|
|
1688 |
- sandbox_build_dir.export_files(buildtreedir)
|
|
1689 |
- except VirtualDirectoryError:
|
|
1690 |
- # Directory could not be found. Pre-virtual
|
|
1691 |
- # directory behaviour was to continue silently
|
|
1692 |
- # if the directory could not be found.
|
|
1693 |
- pass
|
|
1688 |
+ cache_buildtrees = context.cache_buildtrees
|
|
1689 |
+ build_success = self.__build_result[0]
|
|
1690 |
+ |
|
1691 |
+ # cache_buildtrees defaults to 'always', as such the
|
|
1692 |
+ # default behaviour is to attempt to cache them. If only
|
|
1693 |
+ # caching failed artifact buildtrees, then query the build
|
|
1694 |
+ # result. Element types without a build-root dir will be cached
|
|
1695 |
+ # with an empty buildtreedir regardless of this configuration.
|
|
1696 |
+ |
|
1697 |
+ if cache_buildtrees == 'always' or (cache_buildtrees == 'failure' and not build_success):
|
|
1698 |
+ try:
|
|
1699 |
+ sandbox_vroot = sandbox.get_virtual_directory()
|
|
1700 |
+ sandbox_build_dir = sandbox_vroot.descend(
|
|
1701 |
+ self.get_variable('build-root').lstrip(os.sep).split(os.sep))
|
|
1702 |
+ # Hard link files from build-root dir to buildtreedir directory
|
|
1703 |
+ sandbox_build_dir.export_files(buildtreedir)
|
|
1704 |
+ except VirtualDirectoryError:
|
|
1705 |
+ # Directory could not be found. Pre-virtual
|
|
1706 |
+ # directory behaviour was to continue silently
|
|
1707 |
+ # if the directory could not be found.
|
|
1708 |
+ pass
|
|
1694 | 1709 |
|
1695 | 1710 |
# Copy build log
|
1696 |
- log_filename = self._get_context().get_log_filename()
|
|
1711 |
+ log_filename = context.get_log_filename()
|
|
1697 | 1712 |
self._build_log_path = os.path.join(logsdir, 'build.log')
|
1698 | 1713 |
if log_filename:
|
1699 | 1714 |
shutil.copyfile(log_filename, self._build_log_path)
|
... | ... | @@ -1834,7 +1849,7 @@ class Element(Plugin): |
1834 | 1849 |
return True
|
1835 | 1850 |
|
1836 | 1851 |
# Do not push elements that aren't cached, or that are cached with a dangling buildtree
|
1837 |
- # artifact unless element type is expected to have an an empty buildtree directory
|
|
1852 |
+ # ref unless element type is expected to have an an empty buildtree directory
|
|
1838 | 1853 |
if not self._cached_buildtree():
|
1839 | 1854 |
return True
|
1840 | 1855 |
|
... | ... | @@ -2036,6 +2051,8 @@ class Element(Plugin): |
2036 | 2051 |
# Returns:
|
2037 | 2052 |
# (bool): True if artifact cached with buildtree, False if
|
2038 | 2053 |
# element not cached or missing expected buildtree.
|
2054 |
+ # Note this only confirms if a buildtree is present,
|
|
2055 |
+ # not its contents.
|
|
2039 | 2056 |
#
|
2040 | 2057 |
def _cached_buildtree(self):
|
2041 | 2058 |
context = self._get_context()
|
1 |
+#!/usr/bin/env python3
|
|
2 |
+'''Print dependency graph of given element(s) in DOT format.
|
|
3 |
+ |
|
4 |
+This script must be run from the same directory where you would normally
|
|
5 |
+run `bst` commands.
|
|
6 |
+ |
|
7 |
+When `--format` option is specified, the output will also be rendered in the
|
|
8 |
+given format. A file with name `bst-graph.{format}` will be created in the same
|
|
9 |
+directory. To use this option, you must have the `graphviz` command line tool
|
|
10 |
+installed.
|
|
11 |
+'''
|
|
12 |
+ |
|
13 |
+import argparse
|
|
14 |
+import subprocess
|
|
15 |
+ |
|
16 |
+from graphviz import Digraph
|
|
17 |
+ |
|
18 |
+ |
|
19 |
+def parse_args():
|
|
20 |
+ '''Handle parsing of command line arguments.
|
|
21 |
+ |
|
22 |
+ Returns:
|
|
23 |
+ A argparse.Namespace object
|
|
24 |
+ '''
|
|
25 |
+ parser = argparse.ArgumentParser(description=__doc__)
|
|
26 |
+ parser.add_argument(
|
|
27 |
+ 'ELEMENT', nargs='*',
|
|
28 |
+ help='Name of the element'
|
|
29 |
+ )
|
|
30 |
+ parser.add_argument(
|
|
31 |
+ '--format',
|
|
32 |
+ help='Redner the graph in given format (`pdf`, `png`, `svg` etc)'
|
|
33 |
+ )
|
|
34 |
+ parser.add_argument(
|
|
35 |
+ '--view', action='store_true',
|
|
36 |
+ help='Open the rendered graph with the default application'
|
|
37 |
+ )
|
|
38 |
+ return parser.parse_args()
|
|
39 |
+ |
|
40 |
+ |
|
41 |
+def parse_graph(lines):
|
|
42 |
+ '''Return nodes and edges of the parsed grpah.
|
|
43 |
+ |
|
44 |
+ Args:
|
|
45 |
+ lines: List of lines in format 'NAME|BUILD-DEPS|RUNTIME-DEPS'
|
|
46 |
+ |
|
47 |
+ Returns:
|
|
48 |
+ Tuple of format (nodes,build_deps,runtime_deps)
|
|
49 |
+ Each member of build_deps and runtime_deps is also a tuple.
|
|
50 |
+ '''
|
|
51 |
+ nodes = set()
|
|
52 |
+ build_deps = set()
|
|
53 |
+ runtime_deps = set()
|
|
54 |
+ for line in lines:
|
|
55 |
+ # It is safe to split on '|' as it is not a valid character for
|
|
56 |
+ # element names.
|
|
57 |
+ name, build_dep, runtime_dep = line.split('|')
|
|
58 |
+ build_dep = build_dep.lstrip('[').rstrip(']').split(',')
|
|
59 |
+ runtime_dep = runtime_dep.lstrip('[').rstrip(']').split(',')
|
|
60 |
+ nodes.add(name)
|
|
61 |
+ [build_deps.add((name, dep)) for dep in build_dep if dep]
|
|
62 |
+ [runtime_deps.add((name, dep)) for dep in runtime_dep if dep]
|
|
63 |
+ |
|
64 |
+ return nodes, build_deps, runtime_deps
|
|
65 |
+ |
|
66 |
+ |
|
67 |
+def generate_graph(nodes, build_deps, runtime_deps):
|
|
68 |
+ '''Generate graph from given nodes and edges.
|
|
69 |
+ |
|
70 |
+ Args:
|
|
71 |
+ nodes: set of nodes
|
|
72 |
+ build_deps: set of tuples of build depdencies
|
|
73 |
+ runtime_deps: set of tuples of runtime depdencies
|
|
74 |
+ |
|
75 |
+ Returns:
|
|
76 |
+ A graphviz.Digraph object
|
|
77 |
+ '''
|
|
78 |
+ graph = Digraph()
|
|
79 |
+ for node in nodes:
|
|
80 |
+ graph.node(node)
|
|
81 |
+ for source, target in build_deps:
|
|
82 |
+ graph.edge(source, target, label='build-dep')
|
|
83 |
+ for source, target in runtime_deps:
|
|
84 |
+ graph.edge(source, target, label='runtime-dep')
|
|
85 |
+ return graph
|
|
86 |
+ |
|
87 |
+ |
|
88 |
+def main():
|
|
89 |
+ args = parse_args()
|
|
90 |
+ cmd = ['bst', 'show', '--format', '%{name}|%{build-deps}|%{runtime-deps}']
|
|
91 |
+ if 'element' in args:
|
|
92 |
+ cmd += args.element
|
|
93 |
+ graph_lines = subprocess.check_output(cmd, universal_newlines=True)
|
|
94 |
+ # NOTE: We generate nodes and edges before giving them to graphviz as
|
|
95 |
+ # the library does not de-deuplicate them.
|
|
96 |
+ nodes, build_deps, runtime_deps = parse_graph(graph_lines.splitlines())
|
|
97 |
+ graph = generate_graph(nodes, build_deps, runtime_deps)
|
|
98 |
+ print(graph.source)
|
|
99 |
+ if args.format:
|
|
100 |
+ graph.render(cleanup=True,
|
|
101 |
+ filename='bst-graph',
|
|
102 |
+ format=args.format,
|
|
103 |
+ view=args.view)
|
|
104 |
+ |
|
105 |
+ |
|
106 |
+if __name__ == '__main__':
|
|
107 |
+ main()
|
... | ... | @@ -23,6 +23,7 @@ MAIN_OPTIONS = [ |
23 | 23 |
"--builders ",
|
24 | 24 |
"-c ",
|
25 | 25 |
"-C ",
|
26 |
+ "--cache-buildtrees ",
|
|
26 | 27 |
"--colors ",
|
27 | 28 |
"--config ",
|
28 | 29 |
"--debug ",
|
... | ... | @@ -156,6 +157,7 @@ def test_options(cli, cmd, word_idx, expected): |
156 | 157 |
|
157 | 158 |
@pytest.mark.parametrize("cmd,word_idx,expected", [
|
158 | 159 |
('bst --on-error ', 2, ['continue ', 'quit ', 'terminate ']),
|
160 |
+ ('bst --cache-buildtrees ', 2, ['always ', 'failure ', 'never ']),
|
|
159 | 161 |
('bst show --deps ', 3, ['all ', 'build ', 'none ', 'plan ', 'run ']),
|
160 | 162 |
('bst show --deps=', 2, ['all ', 'build ', 'none ', 'plan ', 'run ']),
|
161 | 163 |
('bst show --deps b', 3, ['build ']),
|
... | ... | @@ -20,9 +20,12 @@ |
20 | 20 |
|
21 | 21 |
import os
|
22 | 22 |
import pytest
|
23 |
+import shutil
|
|
23 | 24 |
|
24 | 25 |
from buildstream.plugintestutils import cli_integration as cli
|
25 |
- |
|
26 |
+from tests.testutils import create_artifact_share
|
|
27 |
+from tests.testutils.site import HAVE_SANDBOX
|
|
28 |
+from buildstream._exceptions import ErrorDomain
|
|
26 | 29 |
|
27 | 30 |
pytestmark = pytest.mark.integration
|
28 | 31 |
|
... | ... | @@ -66,3 +69,106 @@ def test_artifact_log(cli, tmpdir, datafiles): |
66 | 69 |
assert result.exit_code == 0
|
67 | 70 |
# The artifact is cached under both a strong key and a weak key
|
68 | 71 |
assert (log + log) == result.output
|
72 |
+ |
|
73 |
+ |
|
74 |
+# A test to capture the integration of the cachebuildtrees
|
|
75 |
+# behaviour, which by default is to include the buildtree
|
|
76 |
+# content of an element on caching.
|
|
77 |
+@pytest.mark.integration
|
|
78 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
79 |
+@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
|
|
80 |
+def test_cache_buildtrees(cli, tmpdir, datafiles):
|
|
81 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
82 |
+ element_name = 'autotools/amhello.bst'
|
|
83 |
+ |
|
84 |
+ # Create artifact shares for pull & push testing
|
|
85 |
+ with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
|
|
86 |
+ create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\
|
|
87 |
+ create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3:
|
|
88 |
+ cli.configure({
|
|
89 |
+ 'artifacts': {'url': share1.repo, 'push': True},
|
|
90 |
+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
|
|
91 |
+ })
|
|
92 |
+ |
|
93 |
+ # Build autotools element with cache-buildtrees set via the
|
|
94 |
+ # cli. The artifact should be successfully pushed to the share1 remote
|
|
95 |
+ # and cached locally with an 'empty' buildtree digest, as it's not a
|
|
96 |
+ # dangling ref
|
|
97 |
+ result = cli.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])
|
|
98 |
+ assert result.exit_code == 0
|
|
99 |
+ assert cli.get_element_state(project, element_name) == 'cached'
|
|
100 |
+ assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
|
|
101 |
+ |
|
102 |
+ # The extracted buildtree dir should be empty, as we set the config
|
|
103 |
+ # to not cache buildtrees
|
|
104 |
+ cache_key = cli.get_element_key(project, element_name)
|
|
105 |
+ elementdigest = share1.has_artifact('test', element_name, cache_key)
|
|
106 |
+ buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
|
|
107 |
+ elementdigest.hash, 'buildtree')
|
|
108 |
+ assert os.path.isdir(buildtreedir)
|
|
109 |
+ assert not os.listdir(buildtreedir)
|
|
110 |
+ |
|
111 |
+ # Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees
|
|
112 |
+ # that is was cached in share1 as expected with an empty buildtree dir
|
|
113 |
+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
|
|
114 |
+ assert cli.get_element_state(project, element_name) != 'cached'
|
|
115 |
+ result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
|
|
116 |
+ assert element_name in result.get_pulled_elements()
|
|
117 |
+ assert os.path.isdir(buildtreedir)
|
|
118 |
+ assert not os.listdir(buildtreedir)
|
|
119 |
+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
|
|
120 |
+ |
|
121 |
+ # Assert that the default behaviour of pull to not include buildtrees on the artifact
|
|
122 |
+ # in share1 which was purposely cached with an empty one behaves as expected. As such the
|
|
123 |
+ # pulled artifact will have a dangling ref for the buildtree dir, regardless of content,
|
|
124 |
+ # leading to no buildtreedir being extracted
|
|
125 |
+ result = cli.run(project=project, args=['artifact', 'pull', element_name])
|
|
126 |
+ assert element_name in result.get_pulled_elements()
|
|
127 |
+ assert not os.path.isdir(buildtreedir)
|
|
128 |
+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
|
|
129 |
+ |
|
130 |
+ # Repeat building the artifacts, this time with the default behaviour of caching buildtrees,
|
|
131 |
+ # as such the buildtree dir should not be empty
|
|
132 |
+ cli.configure({
|
|
133 |
+ 'artifacts': {'url': share2.repo, 'push': True},
|
|
134 |
+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
|
|
135 |
+ })
|
|
136 |
+ result = cli.run(project=project, args=['build', element_name])
|
|
137 |
+ assert result.exit_code == 0
|
|
138 |
+ assert cli.get_element_state(project, element_name) == 'cached'
|
|
139 |
+ assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
|
|
140 |
+ |
|
141 |
+ # Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths
|
|
142 |
+ elementdigest = share2.has_artifact('test', element_name, cache_key)
|
|
143 |
+ buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
|
|
144 |
+ elementdigest.hash, 'buildtree')
|
|
145 |
+ assert os.path.isdir(buildtreedir)
|
|
146 |
+ assert os.listdir(buildtreedir) is not None
|
|
147 |
+ |
|
148 |
+ # Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees
|
|
149 |
+ # that it was cached in share2 as expected with a populated buildtree dir
|
|
150 |
+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
|
|
151 |
+ assert cli.get_element_state(project, element_name) != 'cached'
|
|
152 |
+ result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
|
|
153 |
+ assert element_name in result.get_pulled_elements()
|
|
154 |
+ assert os.path.isdir(buildtreedir)
|
|
155 |
+ assert os.listdir(buildtreedir) is not None
|
|
156 |
+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
|
|
157 |
+ |
|
158 |
+ # Clarify that the user config option for cache-buildtrees works as the cli
|
|
159 |
+ # main option does. Point to share3 which does not have the artifacts cached to force
|
|
160 |
+ # a build
|
|
161 |
+ cli.configure({
|
|
162 |
+ 'artifacts': {'url': share3.repo, 'push': True},
|
|
163 |
+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts'),
|
|
164 |
+ 'cache': {'cache-buildtrees': 'never'}
|
|
165 |
+ })
|
|
166 |
+ result = cli.run(project=project, args=['build', element_name])
|
|
167 |
+ assert result.exit_code == 0
|
|
168 |
+ assert cli.get_element_state(project, element_name) == 'cached'
|
|
169 |
+ cache_key = cli.get_element_key(project, element_name)
|
|
170 |
+ elementdigest = share3.has_artifact('test', element_name, cache_key)
|
|
171 |
+ buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
|
|
172 |
+ elementdigest.hash, 'buildtree')
|
|
173 |
+ assert os.path.isdir(buildtreedir)
|
|
174 |
+ assert not os.listdir(buildtreedir)
|
... | ... | @@ -52,6 +52,29 @@ def test_buildtree_staged_forced_true(cli_integration, tmpdir, datafiles): |
52 | 52 |
assert 'Hi' in res.output
|
53 | 53 |
|
54 | 54 |
|
55 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
56 |
+@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
|
|
57 |
+def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
|
|
58 |
+ # Test that if we stage a cached and empty buildtree, we warn the user.
|
|
59 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
60 |
+ element_name = 'build-shell/buildtree.bst'
|
|
61 |
+ |
|
62 |
+ # Switch to a temp artifact cache dir to ensure the artifact is rebuilt,
|
|
63 |
+ # caching an empty buildtree
|
|
64 |
+ cli_integration.configure({
|
|
65 |
+ 'artifactdir': os.path.join(os.path.join(str(tmpdir), 'artifacts'))
|
|
66 |
+ })
|
|
67 |
+ |
|
68 |
+ res = cli_integration.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])
|
|
69 |
+ res.assert_success()
|
|
70 |
+ |
|
71 |
+ res = cli_integration.run(project=project, args=[
|
|
72 |
+ 'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test'
|
|
73 |
+ ])
|
|
74 |
+ res.assert_shell_error()
|
|
75 |
+ assert "Artifact contains an empty buildtree" in res.stderr
|
|
76 |
+ |
|
77 |
+ |
|
55 | 78 |
@pytest.mark.datafiles(DATA_DIR)
|
56 | 79 |
@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
|
57 | 80 |
def test_buildtree_staged_if_available(cli_integration, tmpdir, datafiles):
|
... | ... | @@ -106,6 +129,54 @@ def test_buildtree_from_failure(cli_integration, tmpdir, datafiles): |
106 | 129 |
assert 'Hi' in res.output
|
107 | 130 |
|
108 | 131 |
|
132 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
133 |
+@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
|
|
134 |
+def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles):
|
|
135 |
+ |
|
136 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
137 |
+ element_name = 'build-shell/buildtree-fail.bst'
|
|
138 |
+ |
|
139 |
+ # Switch to a temp artifact cache dir to ensure the artifact is rebuilt,
|
|
140 |
+ # caching an empty buildtree
|
|
141 |
+ cli_integration.configure({
|
|
142 |
+ 'artifactdir': os.path.join(os.path.join(str(tmpdir), 'artifacts'))
|
|
143 |
+ })
|
|
144 |
+ |
|
145 |
+ res = cli_integration.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])
|
|
146 |
+ res.assert_main_error(ErrorDomain.STREAM, None)
|
|
147 |
+ |
|
148 |
+ res = cli_integration.run(project=project, args=[
|
|
149 |
+ 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
|
|
150 |
+ ])
|
|
151 |
+ res.assert_shell_error()
|
|
152 |
+ assert "Artifact contains an empty buildtree" in res.stderr
|
|
153 |
+ |
|
154 |
+ |
|
155 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
156 |
+@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
|
|
157 |
+def test_buildtree_from_failure_option_failure(cli_integration, tmpdir, datafiles):
|
|
158 |
+ |
|
159 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
160 |
+ element_name = 'build-shell/buildtree-fail.bst'
|
|
161 |
+ |
|
162 |
+ # build with --cache-buildtrees set to 'failure', behaviour should match
|
|
163 |
+ # default behaviour (which is always) as the buildtree will explicitly have been
|
|
164 |
+ # cached with content.
|
|
165 |
+ cli_integration.configure({
|
|
166 |
+ 'artifactdir': os.path.join(os.path.join(str(tmpdir), 'artifacts'))
|
|
167 |
+ })
|
|
168 |
+ |
|
169 |
+ res = cli_integration.run(project=project, args=['--cache-buildtrees', 'failure', 'build', element_name])
|
|
170 |
+ res.assert_main_error(ErrorDomain.STREAM, None)
|
|
171 |
+ |
|
172 |
+ res = cli_integration.run(project=project, args=[
|
|
173 |
+ 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
|
|
174 |
+ ])
|
|
175 |
+ res.assert_success()
|
|
176 |
+ assert "WARNING: using a buildtree from a failed build" in res.stderr
|
|
177 |
+ assert 'Hi' in res.output
|
|
178 |
+ |
|
179 |
+ |
|
109 | 180 |
# Check that build shells work when pulled from a remote cache
|
110 | 181 |
# This is to roughly simulate remote execution
|
111 | 182 |
@pytest.mark.datafiles(DATA_DIR)
|