[Notes] [Git][BuildStream/buildstream][chandan/dot-graph] 5 commits: Add cli main & user conf option for 'cache-buildtrees' context



Title: GitLab

Chandan Singh pushed to branch chandan/dot-graph at BuildStream / buildstream

Commits:

10 changed files:

Changes:

  • NEWS
    ... ... @@ -126,6 +126,14 @@ buildstream 1.3.1
    126 126
         Providing a remote will limit build's pull/push remote actions to the given
    
    127 127
         remote specifically, ignoring those defined via user or project configuration.
    
    128 128
     
    
    129
    +  o Artifacts can now be cached explicitly with an empty `build tree` when built.
    
    130
    +    Element types without a build-root were already cached with an empty build tree
    
    131
    +    directory, this can now be extended to all or successful artifacts to save on cache
    
    132
    +    overheads. The cli main option '--cache-buildtrees' or the user configuration cache
    
    133
    +    group option 'cache-buildtrees' can be set as 'always', 'failure' or 'never', with
    
    134
    +    the default being always. Note, as the cache-key for the artifact is independant of
    
    135
    +    the cached build tree input it will remain unaltered, however the availbility of the
    
    136
    +    build tree content may differ.
    
    129 137
     
    
    130 138
     =================
    
    131 139
     buildstream 1.1.5
    

  • buildstream/_context.py
    ... ... @@ -121,6 +121,9 @@ class Context():
    121 121
             # Whether or not to attempt to pull build trees globally
    
    122 122
             self.pull_buildtrees = None
    
    123 123
     
    
    124
    +        # Whether or not to cache build trees on artifact creation
    
    125
    +        self.cache_buildtrees = None
    
    126
    +
    
    124 127
             # Boolean, whether we double-check with the user that they meant to
    
    125 128
             # close the workspace when they're using it to access the project.
    
    126 129
             self.prompt_workspace_close_project_inaccessible = None
    
    ... ... @@ -201,7 +204,7 @@ class Context():
    201 204
             # our artifactdir - the artifactdir may not have been created
    
    202 205
             # yet.
    
    203 206
             cache = _yaml.node_get(defaults, Mapping, 'cache')
    
    204
    -        _yaml.node_validate(cache, ['quota', 'pull-buildtrees'])
    
    207
    +        _yaml.node_validate(cache, ['quota', 'pull-buildtrees', 'cache-buildtrees'])
    
    205 208
     
    
    206 209
             self.config_cache_quota = _yaml.node_get(cache, str, 'quota')
    
    207 210
     
    
    ... ... @@ -213,6 +216,10 @@ class Context():
    213 216
             # Load pull build trees configuration
    
    214 217
             self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
    
    215 218
     
    
    219
    +        # Load cache build trees configuration
    
    220
    +        self.cache_buildtrees = _node_get_option_str(
    
    221
    +            cache, 'cache-buildtrees', ['always', 'failure', 'never'])
    
    222
    +
    
    216 223
             # Load logging config
    
    217 224
             logging = _yaml.node_get(defaults, Mapping, 'logging')
    
    218 225
             _yaml.node_validate(logging, [
    

  • buildstream/_frontend/app.py
    ... ... @@ -183,7 +183,8 @@ class App():
    183 183
                 'builders': 'sched_builders',
    
    184 184
                 'pushers': 'sched_pushers',
    
    185 185
                 'network_retries': 'sched_network_retries',
    
    186
    -            'pull_buildtrees': 'pull_buildtrees'
    
    186
    +            'pull_buildtrees': 'pull_buildtrees',
    
    187
    +            'cache_buildtrees': 'cache_buildtrees'
    
    187 188
             }
    
    188 189
             for cli_option, context_attr in override_map.items():
    
    189 190
                 option_value = self._main_options.get(cli_option)
    

  • buildstream/_frontend/cli.py
    ... ... @@ -251,6 +251,9 @@ def print_version(ctx, param, value):
    251 251
                   help="The mirror to fetch from first, before attempting other mirrors")
    
    252 252
     @click.option('--pull-buildtrees', is_flag=True, default=None,
    
    253 253
                   help="Include an element's build tree when pulling remote element artifacts")
    
    254
    +@click.option('--cache-buildtrees', default=None,
    
    255
    +              type=click.Choice(['always', 'failure', 'never']),
    
    256
    +              help="Cache artifact build tree content on creation")
    
    254 257
     @click.pass_context
    
    255 258
     def cli(context, **kwargs):
    
    256 259
         """Build and manipulate BuildStream projects
    
    ... ... @@ -572,7 +575,8 @@ def shell(app, element, sysroot, mount, isolate, build_, cli_buildtree, command)
    572 575
                         if choice != "never":
    
    573 576
                             use_buildtree = choice
    
    574 577
     
    
    575
    -        if use_buildtree and not element._cached_success():
    
    578
    +        # Raise warning if the element is cached in a failed state
    
    579
    +        if use_buildtree and element._cached_failure():
    
    576 580
                 click.echo("WARNING: using a buildtree from a failed build.", err=True)
    
    577 581
     
    
    578 582
             try:
    

  • buildstream/data/userconfig.yaml
    ... ... @@ -41,6 +41,15 @@ cache:
    41 41
       # Whether to pull build trees when downloading element artifacts
    
    42 42
       pull-buildtrees: False
    
    43 43
     
    
    44
    +  # Whether to cache build trees on artifact creation:
    
    45
    +  #
    
    46
    +  #  always - Always cache artifact build tree content
    
    47
    +  #  failure - Only cache build trees of failed builds
    
    48
    +  #  never - Don't cache artifact build tree content
    
    49
    +  #
    
    50
    +  cache-buildtrees: always
    
    51
    +
    
    52
    +
    
    44 53
     #
    
    45 54
     #    Scheduler
    
    46 55
     #
    

  • buildstream/element.py
    ... ... @@ -1457,6 +1457,9 @@ class Element(Plugin):
    1457 1457
                 elif usebuildtree:
    
    1458 1458
                     artifact_base, _ = self.__extract()
    
    1459 1459
                     import_dir = os.path.join(artifact_base, 'buildtree')
    
    1460
    +                if not os.listdir(import_dir):
    
    1461
    +                    detail = "Element type either does not expect a buildtree or it was explictily cached without one."
    
    1462
    +                    self.warn("WARNING: {} Artifact contains an empty buildtree".format(self.name), detail=detail)
    
    1460 1463
                 else:
    
    1461 1464
                     # No workspace or cached buildtree, stage source directly
    
    1462 1465
                     for source in self.sources():
    
    ... ... @@ -1663,6 +1666,8 @@ class Element(Plugin):
    1663 1666
                     # No collect directory existed
    
    1664 1667
                     collectvdir = None
    
    1665 1668
     
    
    1669
    +        context = self._get_context()
    
    1670
    +
    
    1666 1671
             # Create artifact directory structure
    
    1667 1672
             assembledir = os.path.join(rootdir, 'artifact')
    
    1668 1673
             filesdir = os.path.join(assembledir, 'files')
    
    ... ... @@ -1680,20 +1685,30 @@ class Element(Plugin):
    1680 1685
             if collect is not None and collectvdir is not None:
    
    1681 1686
                 collectvdir.export_files(filesdir, can_link=True)
    
    1682 1687
     
    
    1683
    -        try:
    
    1684
    -            sandbox_vroot = sandbox.get_virtual_directory()
    
    1685
    -            sandbox_build_dir = sandbox_vroot.descend(
    
    1686
    -                self.get_variable('build-root').lstrip(os.sep).split(os.sep))
    
    1687
    -            # Hard link files from build-root dir to buildtreedir directory
    
    1688
    -            sandbox_build_dir.export_files(buildtreedir)
    
    1689
    -        except VirtualDirectoryError:
    
    1690
    -            # Directory could not be found. Pre-virtual
    
    1691
    -            # directory behaviour was to continue silently
    
    1692
    -            # if the directory could not be found.
    
    1693
    -            pass
    
    1688
    +        cache_buildtrees = context.cache_buildtrees
    
    1689
    +        build_success = self.__build_result[0]
    
    1690
    +
    
    1691
    +        # cache_buildtrees defaults to 'always', as such the
    
    1692
    +        # default behaviour is to attempt to cache them. If only
    
    1693
    +        # caching failed artifact buildtrees, then query the build
    
    1694
    +        # result. Element types without a build-root dir will be cached
    
    1695
    +        # with an empty buildtreedir regardless of this configuration.
    
    1696
    +
    
    1697
    +        if cache_buildtrees == 'always' or (cache_buildtrees == 'failure' and not build_success):
    
    1698
    +            try:
    
    1699
    +                sandbox_vroot = sandbox.get_virtual_directory()
    
    1700
    +                sandbox_build_dir = sandbox_vroot.descend(
    
    1701
    +                    self.get_variable('build-root').lstrip(os.sep).split(os.sep))
    
    1702
    +                # Hard link files from build-root dir to buildtreedir directory
    
    1703
    +                sandbox_build_dir.export_files(buildtreedir)
    
    1704
    +            except VirtualDirectoryError:
    
    1705
    +                # Directory could not be found. Pre-virtual
    
    1706
    +                # directory behaviour was to continue silently
    
    1707
    +                # if the directory could not be found.
    
    1708
    +                pass
    
    1694 1709
     
    
    1695 1710
             # Copy build log
    
    1696
    -        log_filename = self._get_context().get_log_filename()
    
    1711
    +        log_filename = context.get_log_filename()
    
    1697 1712
             self._build_log_path = os.path.join(logsdir, 'build.log')
    
    1698 1713
             if log_filename:
    
    1699 1714
                 shutil.copyfile(log_filename, self._build_log_path)
    
    ... ... @@ -1834,7 +1849,7 @@ class Element(Plugin):
    1834 1849
                 return True
    
    1835 1850
     
    
    1836 1851
             # Do not push elements that aren't cached, or that are cached with a dangling buildtree
    
    1837
    -        # artifact unless element type is expected to have an an empty buildtree directory
    
    1852
    +        # ref unless element type is expected to have an an empty buildtree directory
    
    1838 1853
             if not self._cached_buildtree():
    
    1839 1854
                 return True
    
    1840 1855
     
    
    ... ... @@ -2036,6 +2051,8 @@ class Element(Plugin):
    2036 2051
         # Returns:
    
    2037 2052
         #     (bool): True if artifact cached with buildtree, False if
    
    2038 2053
         #             element not cached or missing expected buildtree.
    
    2054
    +    #             Note this only confirms if a buildtree is present,
    
    2055
    +    #             not its contents.
    
    2039 2056
         #
    
    2040 2057
         def _cached_buildtree(self):
    
    2041 2058
             context = self._get_context()
    

  • contrib/bst-graph
    1
    +#!/usr/bin/env python3
    
    2
    +'''Print dependency graph of given element(s) in DOT format.
    
    3
    +
    
    4
    +This script must be run from the same directory where you would normally
    
    5
    +run `bst` commands.
    
    6
    +
    
    7
    +When `--format` option is specified, the output will also be rendered in the
    
    8
    +given format. A file with name `bst-graph.{format}` will be created in the same
    
    9
    +directory. To use this option, you must have the `graphviz` command line tool
    
    10
    +installed.
    
    11
    +'''
    
    12
    +
    
    13
    +import argparse
    
    14
    +import subprocess
    
    15
    +
    
    16
    +from graphviz import Digraph
    
    17
    +
    
    18
    +
    
    19
    +def parse_args():
    
    20
    +    '''Handle parsing of command line arguments.
    
    21
    +
    
    22
    +    Returns:
    
    23
    +       A argparse.Namespace object
    
    24
    +    '''
    
    25
    +    parser = argparse.ArgumentParser(description=__doc__)
    
    26
    +    parser.add_argument(
    
    27
    +        'ELEMENT', nargs='*',
    
    28
    +        help='Name of the element'
    
    29
    +    )
    
    30
    +    parser.add_argument(
    
    31
    +        '--format',
    
    32
    +        help='Redner the graph in given format (`pdf`, `png`, `svg` etc)'
    
    33
    +    )
    
    34
    +    parser.add_argument(
    
    35
    +        '--view', action='store_true',
    
    36
    +        help='Open the rendered graph with the default application'
    
    37
    +    )
    
    38
    +    return parser.parse_args()
    
    39
    +
    
    40
    +
    
    41
    +def parse_graph(lines):
    
    42
    +    '''Return nodes and edges of the parsed grpah.
    
    43
    +
    
    44
    +    Args:
    
    45
    +       lines: List of lines in format 'NAME|BUILD-DEPS|RUNTIME-DEPS'
    
    46
    +
    
    47
    +    Returns:
    
    48
    +       Tuple of format (nodes,build_deps,runtime_deps)
    
    49
    +       Each member of build_deps and runtime_deps is also a tuple.
    
    50
    +    '''
    
    51
    +    nodes = set()
    
    52
    +    build_deps = set()
    
    53
    +    runtime_deps = set()
    
    54
    +    for line in lines:
    
    55
    +        # It is safe to split on '|' as it is not a valid character for
    
    56
    +        # element names.
    
    57
    +        name, build_dep, runtime_dep = line.split('|')
    
    58
    +        build_dep = build_dep.lstrip('[').rstrip(']').split(',')
    
    59
    +        runtime_dep = runtime_dep.lstrip('[').rstrip(']').split(',')
    
    60
    +        nodes.add(name)
    
    61
    +        [build_deps.add((name, dep)) for dep in build_dep if dep]
    
    62
    +        [runtime_deps.add((name, dep)) for dep in runtime_dep if dep]
    
    63
    +
    
    64
    +    return nodes, build_deps, runtime_deps
    
    65
    +
    
    66
    +
    
    67
    +def generate_graph(nodes, build_deps, runtime_deps):
    
    68
    +    '''Generate graph from given nodes and edges.
    
    69
    +
    
    70
    +    Args:
    
    71
    +       nodes: set of nodes
    
    72
    +       build_deps: set of tuples of build depdencies
    
    73
    +       runtime_deps: set of tuples of runtime depdencies
    
    74
    +
    
    75
    +    Returns:
    
    76
    +       A graphviz.Digraph object
    
    77
    +    '''
    
    78
    +    graph = Digraph()
    
    79
    +    for node in nodes:
    
    80
    +        graph.node(node)
    
    81
    +    for source, target in build_deps:
    
    82
    +        graph.edge(source, target, label='build-dep')
    
    83
    +    for source, target in runtime_deps:
    
    84
    +        graph.edge(source, target, label='runtime-dep')
    
    85
    +    return graph
    
    86
    +
    
    87
    +
    
    88
    +def main():
    
    89
    +    args = parse_args()
    
    90
    +    cmd = ['bst', 'show', '--format', '%{name}|%{build-deps}|%{runtime-deps}']
    
    91
    +    if 'element' in args:
    
    92
    +        cmd += args.element
    
    93
    +    graph_lines = subprocess.check_output(cmd, universal_newlines=True)
    
    94
    +    # NOTE: We generate nodes and edges before giving them to graphviz as
    
    95
    +    # the library does not de-deuplicate them.
    
    96
    +    nodes, build_deps, runtime_deps = parse_graph(graph_lines.splitlines())
    
    97
    +    graph = generate_graph(nodes, build_deps, runtime_deps)
    
    98
    +    print(graph.source)
    
    99
    +    if args.format:
    
    100
    +        graph.render(cleanup=True,
    
    101
    +                     filename='bst-graph',
    
    102
    +                     format=args.format,
    
    103
    +                     view=args.view)
    
    104
    +
    
    105
    +
    
    106
    +if __name__ == '__main__':
    
    107
    +    main()

  • tests/frontend/completions.py
    ... ... @@ -23,6 +23,7 @@ MAIN_OPTIONS = [
    23 23
         "--builders ",
    
    24 24
         "-c ",
    
    25 25
         "-C ",
    
    26
    +    "--cache-buildtrees ",
    
    26 27
         "--colors ",
    
    27 28
         "--config ",
    
    28 29
         "--debug ",
    
    ... ... @@ -156,6 +157,7 @@ def test_options(cli, cmd, word_idx, expected):
    156 157
     
    
    157 158
     @pytest.mark.parametrize("cmd,word_idx,expected", [
    
    158 159
         ('bst --on-error ', 2, ['continue ', 'quit ', 'terminate ']),
    
    160
    +    ('bst --cache-buildtrees ', 2, ['always ', 'failure ', 'never ']),
    
    159 161
         ('bst show --deps ', 3, ['all ', 'build ', 'none ', 'plan ', 'run ']),
    
    160 162
         ('bst show --deps=', 2, ['all ', 'build ', 'none ', 'plan ', 'run ']),
    
    161 163
         ('bst show --deps b', 3, ['build ']),
    

  • tests/integration/artifact.py
    ... ... @@ -20,9 +20,12 @@
    20 20
     
    
    21 21
     import os
    
    22 22
     import pytest
    
    23
    +import shutil
    
    23 24
     
    
    24 25
     from buildstream.plugintestutils import cli_integration as cli
    
    25
    -
    
    26
    +from tests.testutils import create_artifact_share
    
    27
    +from tests.testutils.site import HAVE_SANDBOX
    
    28
    +from buildstream._exceptions import ErrorDomain
    
    26 29
     
    
    27 30
     pytestmark = pytest.mark.integration
    
    28 31
     
    
    ... ... @@ -66,3 +69,106 @@ def test_artifact_log(cli, tmpdir, datafiles):
    66 69
         assert result.exit_code == 0
    
    67 70
         # The artifact is cached under both a strong key and a weak key
    
    68 71
         assert (log + log) == result.output
    
    72
    +
    
    73
    +
    
    74
    +# A test to capture the integration of the cachebuildtrees
    
    75
    +# behaviour, which by default is to include the buildtree
    
    76
    +# content of an element on caching.
    
    77
    +@pytest.mark.integration
    
    78
    +@pytest.mark.datafiles(DATA_DIR)
    
    79
    +@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
    
    80
    +def test_cache_buildtrees(cli, tmpdir, datafiles):
    
    81
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    82
    +    element_name = 'autotools/amhello.bst'
    
    83
    +
    
    84
    +    # Create artifact shares for pull & push testing
    
    85
    +    with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
    
    86
    +        create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\
    
    87
    +        create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3:
    
    88
    +        cli.configure({
    
    89
    +            'artifacts': {'url': share1.repo, 'push': True},
    
    90
    +            'artifactdir': os.path.join(str(tmpdir), 'artifacts')
    
    91
    +        })
    
    92
    +
    
    93
    +        # Build autotools element with cache-buildtrees set via the
    
    94
    +        # cli. The artifact should be successfully pushed to the share1 remote
    
    95
    +        # and cached locally with an 'empty' buildtree digest, as it's not a
    
    96
    +        # dangling ref
    
    97
    +        result = cli.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])
    
    98
    +        assert result.exit_code == 0
    
    99
    +        assert cli.get_element_state(project, element_name) == 'cached'
    
    100
    +        assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    
    101
    +
    
    102
    +        # The extracted buildtree dir should be empty, as we set the config
    
    103
    +        # to not cache buildtrees
    
    104
    +        cache_key = cli.get_element_key(project, element_name)
    
    105
    +        elementdigest = share1.has_artifact('test', element_name, cache_key)
    
    106
    +        buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
    
    107
    +                                    elementdigest.hash, 'buildtree')
    
    108
    +        assert os.path.isdir(buildtreedir)
    
    109
    +        assert not os.listdir(buildtreedir)
    
    110
    +
    
    111
    +        # Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees
    
    112
    +        # that is was cached in share1 as expected with an empty buildtree dir
    
    113
    +        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    
    114
    +        assert cli.get_element_state(project, element_name) != 'cached'
    
    115
    +        result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
    
    116
    +        assert element_name in result.get_pulled_elements()
    
    117
    +        assert os.path.isdir(buildtreedir)
    
    118
    +        assert not os.listdir(buildtreedir)
    
    119
    +        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    
    120
    +
    
    121
    +        # Assert that the default behaviour of pull to not include buildtrees on the artifact
    
    122
    +        # in share1 which was purposely cached with an empty one behaves as expected. As such the
    
    123
    +        # pulled artifact will have a dangling ref for the buildtree dir, regardless of content,
    
    124
    +        # leading to no buildtreedir being extracted
    
    125
    +        result = cli.run(project=project, args=['artifact', 'pull', element_name])
    
    126
    +        assert element_name in result.get_pulled_elements()
    
    127
    +        assert not os.path.isdir(buildtreedir)
    
    128
    +        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    
    129
    +
    
    130
    +        # Repeat building the artifacts, this time with the default behaviour of caching buildtrees,
    
    131
    +        # as such the buildtree dir should not be empty
    
    132
    +        cli.configure({
    
    133
    +            'artifacts': {'url': share2.repo, 'push': True},
    
    134
    +            'artifactdir': os.path.join(str(tmpdir), 'artifacts')
    
    135
    +        })
    
    136
    +        result = cli.run(project=project, args=['build', element_name])
    
    137
    +        assert result.exit_code == 0
    
    138
    +        assert cli.get_element_state(project, element_name) == 'cached'
    
    139
    +        assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    
    140
    +
    
    141
    +        # Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths
    
    142
    +        elementdigest = share2.has_artifact('test', element_name, cache_key)
    
    143
    +        buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
    
    144
    +                                    elementdigest.hash, 'buildtree')
    
    145
    +        assert os.path.isdir(buildtreedir)
    
    146
    +        assert os.listdir(buildtreedir) is not None
    
    147
    +
    
    148
    +        # Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees
    
    149
    +        # that it was cached in share2 as expected with a populated buildtree dir
    
    150
    +        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    
    151
    +        assert cli.get_element_state(project, element_name) != 'cached'
    
    152
    +        result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
    
    153
    +        assert element_name in result.get_pulled_elements()
    
    154
    +        assert os.path.isdir(buildtreedir)
    
    155
    +        assert os.listdir(buildtreedir) is not None
    
    156
    +        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    
    157
    +
    
    158
    +        # Clarify that the user config option for cache-buildtrees works as the cli
    
    159
    +        # main option does. Point to share3 which does not have the artifacts cached to force
    
    160
    +        # a build
    
    161
    +        cli.configure({
    
    162
    +            'artifacts': {'url': share3.repo, 'push': True},
    
    163
    +            'artifactdir': os.path.join(str(tmpdir), 'artifacts'),
    
    164
    +            'cache': {'cache-buildtrees': 'never'}
    
    165
    +        })
    
    166
    +        result = cli.run(project=project, args=['build', element_name])
    
    167
    +        assert result.exit_code == 0
    
    168
    +        assert cli.get_element_state(project, element_name) == 'cached'
    
    169
    +        cache_key = cli.get_element_key(project, element_name)
    
    170
    +        elementdigest = share3.has_artifact('test', element_name, cache_key)
    
    171
    +        buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
    
    172
    +                                    elementdigest.hash, 'buildtree')
    
    173
    +        assert os.path.isdir(buildtreedir)
    
    174
    +        assert not os.listdir(buildtreedir)

  • tests/integration/build-tree.pytests/integration/shellbuildtrees.py
    ... ... @@ -52,6 +52,29 @@ def test_buildtree_staged_forced_true(cli_integration, tmpdir, datafiles):
    52 52
         assert 'Hi' in res.output
    
    53 53
     
    
    54 54
     
    
    55
    +@pytest.mark.datafiles(DATA_DIR)
    
    56
    +@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
    
    57
    +def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
    
    58
    +    # Test that if we stage a cached and empty buildtree, we warn the user.
    
    59
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    60
    +    element_name = 'build-shell/buildtree.bst'
    
    61
    +
    
    62
    +    # Switch to a temp artifact cache dir to ensure the artifact is rebuilt,
    
    63
    +    # caching an empty buildtree
    
    64
    +    cli_integration.configure({
    
    65
    +        'artifactdir': os.path.join(os.path.join(str(tmpdir), 'artifacts'))
    
    66
    +    })
    
    67
    +
    
    68
    +    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])
    
    69
    +    res.assert_success()
    
    70
    +
    
    71
    +    res = cli_integration.run(project=project, args=[
    
    72
    +        'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test'
    
    73
    +    ])
    
    74
    +    res.assert_shell_error()
    
    75
    +    assert "Artifact contains an empty buildtree" in res.stderr
    
    76
    +
    
    77
    +
    
    55 78
     @pytest.mark.datafiles(DATA_DIR)
    
    56 79
     @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
    
    57 80
     def test_buildtree_staged_if_available(cli_integration, tmpdir, datafiles):
    
    ... ... @@ -106,6 +129,54 @@ def test_buildtree_from_failure(cli_integration, tmpdir, datafiles):
    106 129
         assert 'Hi' in res.output
    
    107 130
     
    
    108 131
     
    
    132
    +@pytest.mark.datafiles(DATA_DIR)
    
    133
    +@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
    
    134
    +def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles):
    
    135
    +
    
    136
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    137
    +    element_name = 'build-shell/buildtree-fail.bst'
    
    138
    +
    
    139
    +    # Switch to a temp artifact cache dir to ensure the artifact is rebuilt,
    
    140
    +    # caching an empty buildtree
    
    141
    +    cli_integration.configure({
    
    142
    +        'artifactdir': os.path.join(os.path.join(str(tmpdir), 'artifacts'))
    
    143
    +    })
    
    144
    +
    
    145
    +    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])
    
    146
    +    res.assert_main_error(ErrorDomain.STREAM, None)
    
    147
    +
    
    148
    +    res = cli_integration.run(project=project, args=[
    
    149
    +        'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
    
    150
    +    ])
    
    151
    +    res.assert_shell_error()
    
    152
    +    assert "Artifact contains an empty buildtree" in res.stderr
    
    153
    +
    
    154
    +
    
    155
    +@pytest.mark.datafiles(DATA_DIR)
    
    156
    +@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
    
    157
    +def test_buildtree_from_failure_option_failure(cli_integration, tmpdir, datafiles):
    
    158
    +
    
    159
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    160
    +    element_name = 'build-shell/buildtree-fail.bst'
    
    161
    +
    
    162
    +    # build with  --cache-buildtrees set to 'failure', behaviour should match
    
    163
    +    # default behaviour (which is always) as the buildtree will explicitly have been
    
    164
    +    # cached with content.
    
    165
    +    cli_integration.configure({
    
    166
    +        'artifactdir': os.path.join(os.path.join(str(tmpdir), 'artifacts'))
    
    167
    +    })
    
    168
    +
    
    169
    +    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'failure', 'build', element_name])
    
    170
    +    res.assert_main_error(ErrorDomain.STREAM, None)
    
    171
    +
    
    172
    +    res = cli_integration.run(project=project, args=[
    
    173
    +        'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
    
    174
    +    ])
    
    175
    +    res.assert_success()
    
    176
    +    assert "WARNING: using a buildtree from a failed build" in res.stderr
    
    177
    +    assert 'Hi' in res.output
    
    178
    +
    
    179
    +
    
    109 180
     # Check that build shells work when pulled from a remote cache
    
    110 181
     # This is to roughly simulate remote execution
    
    111 182
     @pytest.mark.datafiles(DATA_DIR)
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]