[Notes] [Git][BuildStream/buildstream][tpollard/896] 3 commits: contributing: snakeviz replaces pyflame+flamegraph



Title: GitLab

Tom Pollard pushed to branch tpollard/896 at BuildStream / buildstream

Commits:

10 changed files:

Changes:

  • CONTRIBUTING.rst
    ... ... @@ -1707,26 +1707,13 @@ You can then analyze the results interactively using the 'pstats' module:
    1707 1707
     For more detailed documentation of cProfile and 'pstats', see:
    
    1708 1708
     https://docs.python.org/3/library/profile.html.
    
    1709 1709
     
    
    1710
    -For a richer visualisation of the callstack you can try `Pyflame
    
    1711
    -<https://github.com/uber/pyflame>`_. Once you have followed the instructions in
    
    1712
    -Pyflame's README to install the tool, you can profile `bst` commands as in the
    
    1713
    -following example:
    
    1710
    +For a richer and interactive visualisation of the `.cprofile` files, you can
    
    1711
    +try `snakeviz <http://jiffyclub.github.io/snakeviz/#interpreting-results>`_.
    
    1712
    +You can install it with `pip install snakeviz`. Here is an example invocation:
    
    1714 1713
     
    
    1715
    -    pyflame --output bst.flame --trace bst --help
    
    1716
    -
    
    1717
    -You may see an `Unexpected ptrace(2) exception:` error. Note that the `bst`
    
    1718
    -operation will continue running in the background in this case, you will need
    
    1719
    -to wait for it to complete or kill it. Once this is done, rerun the above
    
    1720
    -command which appears to fix the issue.
    
    1721
    -
    
    1722
    -Once you have output from pyflame, you can use the ``flamegraph.pl`` script
    
    1723
    -from the `Flamegraph project <https://github.com/brendangregg/FlameGraph>`_
    
    1724
    -to generate an .svg image:
    
    1725
    -
    
    1726
    -    ./flamegraph.pl bst.flame > bst-flamegraph.svg
    
    1727
    -
    
    1728
    -The generated SVG file can then be viewed in your preferred web browser.
    
    1714
    +    snakeviz bst.cprofile
    
    1729 1715
     
    
    1716
    +It will then start a webserver and launch a browser to the relevant page.
    
    1730 1717
     
    
    1731 1718
     Profiling specific parts of BuildStream with BST_PROFILE
    
    1732 1719
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    

  • NEWS
    ... ... @@ -126,6 +126,13 @@ buildstream 1.3.1
    126 126
         Providing a remote will limit build's pull/push remote actions to the given
    
    127 127
         remote specifically, ignoring those defined via user or project configuration.
    
    128 128
     
    
    129
    +  o Artifacts can now be cached explicitly with an empty `build tree` when built.
    
    130
    +    Element types without a build-root were already cached with an empty build tree
    
    131
    +    directory, this can now be extended to all artifacts to save on cache overheads.
    
    132
    +    The cli main option 'no-cache-buildtrees' or the user configuration cache group
    
    133
    +    option 'cache-buildtrees = False' can be set to enable this behaviour. Note, as
    
    134
    +    the cache-key for the artifact is independant of the cached build tree input it will
    
    135
    +    remain unaltered, however the availbility of the build tree content may differ.
    
    129 136
     
    
    130 137
     =================
    
    131 138
     buildstream 1.1.5
    

  • buildstream/_context.py
    ... ... @@ -121,6 +121,9 @@ class Context():
    121 121
             # Whether or not to attempt to pull build trees globally
    
    122 122
             self.pull_buildtrees = None
    
    123 123
     
    
    124
    +        # Whether or not to cache build trees on artifact creation
    
    125
    +        self.cache_buildtrees = None
    
    126
    +
    
    124 127
             # Boolean, whether we double-check with the user that they meant to
    
    125 128
             # close the workspace when they're using it to access the project.
    
    126 129
             self.prompt_workspace_close_project_inaccessible = None
    
    ... ... @@ -201,7 +204,7 @@ class Context():
    201 204
             # our artifactdir - the artifactdir may not have been created
    
    202 205
             # yet.
    
    203 206
             cache = _yaml.node_get(defaults, Mapping, 'cache')
    
    204
    -        _yaml.node_validate(cache, ['quota', 'pull-buildtrees'])
    
    207
    +        _yaml.node_validate(cache, ['quota', 'pull-buildtrees', 'cache-buildtrees'])
    
    205 208
     
    
    206 209
             self.config_cache_quota = _yaml.node_get(cache, str, 'quota')
    
    207 210
     
    
    ... ... @@ -213,6 +216,9 @@ class Context():
    213 216
             # Load pull build trees configuration
    
    214 217
             self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
    
    215 218
     
    
    219
    +        # Load cache build trees configuration
    
    220
    +        self.cache_buildtrees = _yaml.node_get(cache, bool, 'cache-buildtrees')
    
    221
    +
    
    216 222
             # Load logging config
    
    217 223
             logging = _yaml.node_get(defaults, Mapping, 'logging')
    
    218 224
             _yaml.node_validate(logging, [
    

  • buildstream/_frontend/app.py
    ... ... @@ -183,7 +183,8 @@ class App():
    183 183
                 'builders': 'sched_builders',
    
    184 184
                 'pushers': 'sched_pushers',
    
    185 185
                 'network_retries': 'sched_network_retries',
    
    186
    -            'pull_buildtrees': 'pull_buildtrees'
    
    186
    +            'pull_buildtrees': 'pull_buildtrees',
    
    187
    +            'cache_buildtrees': 'cache_buildtrees'
    
    187 188
             }
    
    188 189
             for cli_option, context_attr in override_map.items():
    
    189 190
                 option_value = self._main_options.get(cli_option)
    

  • buildstream/_frontend/cli.py
    ... ... @@ -252,6 +252,8 @@ def print_version(ctx, param, value):
    252 252
                   help="The mirror to fetch from first, before attempting other mirrors")
    
    253 253
     @click.option('--pull-buildtrees', is_flag=True, default=None,
    
    254 254
                   help="Include an element's build tree when pulling remote element artifacts")
    
    255
    +@click.option('--cache-buildtrees/--no-cache-buildtrees', default=None,
    
    256
    +              help="Cache artifact build tree content on creation")
    
    255 257
     @click.pass_context
    
    256 258
     def cli(context, **kwargs):
    
    257 259
         """Build and manipulate BuildStream projects
    

  • buildstream/data/userconfig.yaml
    ... ... @@ -41,6 +41,9 @@ cache:
    41 41
       # Whether to pull build trees when downloading element artifacts
    
    42 42
       pull-buildtrees: False
    
    43 43
     
    
    44
    +  # Whether to cache build trees on artifact creation
    
    45
    +  cache-buildtrees: True
    
    46
    +
    
    44 47
     #
    
    45 48
     #    Scheduler
    
    46 49
     #
    

  • buildstream/element.py
    ... ... @@ -1425,6 +1425,9 @@ class Element(Plugin):
    1425 1425
                 elif usebuildtree:
    
    1426 1426
                     artifact_base, _ = self.__extract()
    
    1427 1427
                     import_dir = os.path.join(artifact_base, 'buildtree')
    
    1428
    +                if not os.listdir(import_dir):
    
    1429
    +                    detail = "Element type either does not expect a buildtree or it was explictily cached without one."
    
    1430
    +                    self.warn("WARNING: {} Artifact contains an empty buildtree".format(self.name), detail=detail)
    
    1428 1431
                 else:
    
    1429 1432
                     # No workspace or cached buildtree, stage source directly
    
    1430 1433
                     for source in self.sources():
    
    ... ... @@ -1631,6 +1634,8 @@ class Element(Plugin):
    1631 1634
                     # No collect directory existed
    
    1632 1635
                     collectvdir = None
    
    1633 1636
     
    
    1637
    +        context = self._get_context()
    
    1638
    +
    
    1634 1639
             # Create artifact directory structure
    
    1635 1640
             assembledir = os.path.join(rootdir, 'artifact')
    
    1636 1641
             filesdir = os.path.join(assembledir, 'files')
    
    ... ... @@ -1648,20 +1653,27 @@ class Element(Plugin):
    1648 1653
             if collect is not None and collectvdir is not None:
    
    1649 1654
                 collectvdir.export_files(filesdir, can_link=True)
    
    1650 1655
     
    
    1651
    -        try:
    
    1652
    -            sandbox_vroot = sandbox.get_virtual_directory()
    
    1653
    -            sandbox_build_dir = sandbox_vroot.descend(
    
    1654
    -                self.get_variable('build-root').lstrip(os.sep).split(os.sep))
    
    1655
    -            # Hard link files from build-root dir to buildtreedir directory
    
    1656
    -            sandbox_build_dir.export_files(buildtreedir)
    
    1657
    -        except VirtualDirectoryError:
    
    1658
    -            # Directory could not be found. Pre-virtual
    
    1659
    -            # directory behaviour was to continue silently
    
    1660
    -            # if the directory could not be found.
    
    1661
    -            pass
    
    1656
    +        cache_buildtrees = context.cache_buildtrees
    
    1657
    +
    
    1658
    +        # cache_buildtrees defaults to True, as such the
    
    1659
    +        # default behaviour is to attempt to cache them. Element
    
    1660
    +        # types without a build-root dir will be cached with an empty
    
    1661
    +        # buildtreedir regardless of this configuration.
    
    1662
    +        if cache_buildtrees:
    
    1663
    +            try:
    
    1664
    +                sandbox_vroot = sandbox.get_virtual_directory()
    
    1665
    +                sandbox_build_dir = sandbox_vroot.descend(
    
    1666
    +                    self.get_variable('build-root').lstrip(os.sep).split(os.sep))
    
    1667
    +                # Hard link files from build-root dir to buildtreedir directory
    
    1668
    +                sandbox_build_dir.export_files(buildtreedir)
    
    1669
    +            except VirtualDirectoryError:
    
    1670
    +                # Directory could not be found. Pre-virtual
    
    1671
    +                # directory behaviour was to continue silently
    
    1672
    +                # if the directory could not be found.
    
    1673
    +                pass
    
    1662 1674
     
    
    1663 1675
             # Copy build log
    
    1664
    -        log_filename = self._get_context().get_log_filename()
    
    1676
    +        log_filename = context.get_log_filename()
    
    1665 1677
             self._build_log_path = os.path.join(logsdir, 'build.log')
    
    1666 1678
             if log_filename:
    
    1667 1679
                 shutil.copyfile(log_filename, self._build_log_path)
    
    ... ... @@ -1802,7 +1814,7 @@ class Element(Plugin):
    1802 1814
                 return True
    
    1803 1815
     
    
    1804 1816
             # Do not push elements that aren't cached, or that are cached with a dangling buildtree
    
    1805
    -        # artifact unless element type is expected to have an an empty buildtree directory
    
    1817
    +        # ref unless element type is expected to have an an empty buildtree directory
    
    1806 1818
             if not self._cached_buildtree():
    
    1807 1819
                 return True
    
    1808 1820
     
    
    ... ... @@ -2004,6 +2016,8 @@ class Element(Plugin):
    2004 2016
         # Returns:
    
    2005 2017
         #     (bool): True if artifact cached with buildtree, False if
    
    2006 2018
         #             element not cached or missing expected buildtree.
    
    2019
    +    #             Note this only confirms if a buildtree is present,
    
    2020
    +    #             not it's contents.
    
    2007 2021
         #
    
    2008 2022
         def _cached_buildtree(self):
    
    2009 2023
             context = self._get_context()
    

  • tests/frontend/completions.py
    ... ... @@ -23,6 +23,7 @@ MAIN_OPTIONS = [
    23 23
         "--builders ",
    
    24 24
         "-c ",
    
    25 25
         "-C ",
    
    26
    +    "--cache-buildtrees ",
    
    26 27
         "--colors ",
    
    27 28
         "--config ",
    
    28 29
         "--debug ",
    
    ... ... @@ -33,6 +34,7 @@ MAIN_OPTIONS = [
    33 34
         "--log-file ",
    
    34 35
         "--message-lines ",
    
    35 36
         "--network-retries ",
    
    37
    +    "--no-cache-buildtrees ",
    
    36 38
         "--no-colors ",
    
    37 39
         "--no-debug ",
    
    38 40
         "--no-interactive ",
    

  • tests/integration/artifact.py
    ... ... @@ -20,9 +20,12 @@
    20 20
     
    
    21 21
     import os
    
    22 22
     import pytest
    
    23
    +import shutil
    
    23 24
     
    
    24 25
     from buildstream.plugintestutils import cli_integration as cli
    
    25
    -
    
    26
    +from tests.testutils import create_artifact_share
    
    27
    +from tests.testutils.site import HAVE_SANDBOX
    
    28
    +from buildstream._exceptions import ErrorDomain
    
    26 29
     
    
    27 30
     pytestmark = pytest.mark.integration
    
    28 31
     
    
    ... ... @@ -66,3 +69,106 @@ def test_artifact_log(cli, tmpdir, datafiles):
    66 69
         assert result.exit_code == 0
    
    67 70
         # The artifact is cached under both a strong key and a weak key
    
    68 71
         assert (log + log) == result.output
    
    72
    +
    
    73
    +
    
    74
    +# A test to capture the integration of the cachebuildtrees
    
    75
    +# behaviour, which by default is to include the buildtree
    
    76
    +# content of an element on caching.
    
    77
    +@pytest.mark.integration
    
    78
    +@pytest.mark.datafiles(DATA_DIR)
    
    79
    +@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
    
    80
    +def test_cache_buildtrees(cli, tmpdir, datafiles):
    
    81
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    82
    +    element_name = 'autotools/amhello.bst'
    
    83
    +
    
    84
    +    # Create artifact shares for pull & push testing
    
    85
    +    with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
    
    86
    +        create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\
    
    87
    +        create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3:
    
    88
    +        cli.configure({
    
    89
    +            'artifacts': {'url': share1.repo, 'push': True},
    
    90
    +            'artifactdir': os.path.join(str(tmpdir), 'artifacts')
    
    91
    +        })
    
    92
    +
    
    93
    +        # Build autotools element with cache-buildtrees set via the
    
    94
    +        # cli. The artifact should be successfully pushed to the share1 remote
    
    95
    +        # and cached locally with an 'empty' buildtree digest, as it's not a
    
    96
    +        # dangling ref
    
    97
    +        result = cli.run(project=project, args=['--no-cache-buildtrees', 'build', element_name])
    
    98
    +        assert result.exit_code == 0
    
    99
    +        assert cli.get_element_state(project, element_name) == 'cached'
    
    100
    +        assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    
    101
    +
    
    102
    +        # The extracted buildtree dir should be empty, as we set the config
    
    103
    +        # to not cache buildtrees
    
    104
    +        cache_key = cli.get_element_key(project, element_name)
    
    105
    +        elementdigest = share1.has_artifact('test', element_name, cache_key)
    
    106
    +        buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
    
    107
    +                                    elementdigest.hash, 'buildtree')
    
    108
    +        assert os.path.isdir(buildtreedir)
    
    109
    +        assert not os.listdir(buildtreedir)
    
    110
    +
    
    111
    +        # Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees
    
    112
    +        # that is was cached in share1 as expected with an empty buildtree dir
    
    113
    +        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    
    114
    +        assert cli.get_element_state(project, element_name) != 'cached'
    
    115
    +        result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
    
    116
    +        assert element_name in result.get_pulled_elements()
    
    117
    +        assert os.path.isdir(buildtreedir)
    
    118
    +        assert not os.listdir(buildtreedir)
    
    119
    +        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    
    120
    +
    
    121
    +        # Assert that the default behaviour of pull to not include buildtrees on the artifact
    
    122
    +        # in share1 which was purposely cached with an empty one behaves as expected. As such the
    
    123
    +        # pulled artifact will have a dangling ref for the buildtree dir, regardless of content,
    
    124
    +        # leading to no buildtreedir being extracted
    
    125
    +        result = cli.run(project=project, args=['artifact', 'pull', element_name])
    
    126
    +        assert element_name in result.get_pulled_elements()
    
    127
    +        assert not os.path.isdir(buildtreedir)
    
    128
    +        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    
    129
    +
    
    130
    +        # Repeat building the artifacts, this time with the default behaviour of caching buildtrees,
    
    131
    +        # as such the buildtree dir should not be empty
    
    132
    +        cli.configure({
    
    133
    +            'artifacts': {'url': share2.repo, 'push': True},
    
    134
    +            'artifactdir': os.path.join(str(tmpdir), 'artifacts')
    
    135
    +        })
    
    136
    +        result = cli.run(project=project, args=['build', element_name])
    
    137
    +        assert result.exit_code == 0
    
    138
    +        assert cli.get_element_state(project, element_name) == 'cached'
    
    139
    +        assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    
    140
    +
    
    141
    +        # Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths
    
    142
    +        elementdigest = share2.has_artifact('test', element_name, cache_key)
    
    143
    +        buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
    
    144
    +                                    elementdigest.hash, 'buildtree')
    
    145
    +        assert os.path.isdir(buildtreedir)
    
    146
    +        assert os.listdir(buildtreedir) is not None
    
    147
    +
    
    148
    +        # Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees
    
    149
    +        # that it was cached in share2 as expected with a populated buildtree dir
    
    150
    +        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    
    151
    +        assert cli.get_element_state(project, element_name) != 'cached'
    
    152
    +        result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
    
    153
    +        assert element_name in result.get_pulled_elements()
    
    154
    +        assert os.path.isdir(buildtreedir)
    
    155
    +        assert os.listdir(buildtreedir) is not None
    
    156
    +        shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    
    157
    +
    
    158
    +        # Clarify that the user config option for cache-buildtrees works as the cli
    
    159
    +        # main option does. Point to share3 which does not have the artifacts cached to force
    
    160
    +        # a build
    
    161
    +        cli.configure({
    
    162
    +            'artifacts': {'url': share3.repo, 'push': True},
    
    163
    +            'artifactdir': os.path.join(str(tmpdir), 'artifacts'),
    
    164
    +            'cache': {'cache-buildtrees': False}
    
    165
    +        })
    
    166
    +        result = cli.run(project=project, args=['build', element_name])
    
    167
    +        assert result.exit_code == 0
    
    168
    +        assert cli.get_element_state(project, element_name) == 'cached'
    
    169
    +        cache_key = cli.get_element_key(project, element_name)
    
    170
    +        elementdigest = share3.has_artifact('test', element_name, cache_key)
    
    171
    +        buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
    
    172
    +                                    elementdigest.hash, 'buildtree')
    
    173
    +        assert os.path.isdir(buildtreedir)
    
    174
    +        assert not os.listdir(buildtreedir)

  • tests/integration/build-tree.pytests/integration/shellbuildtrees.py
    ... ... @@ -52,6 +52,30 @@ def test_buildtree_staged_forced_true(cli_integration, tmpdir, datafiles):
    52 52
         assert 'Hi' in res.output
    
    53 53
     
    
    54 54
     
    
    55
    +@pytest.mark.datafiles(DATA_DIR)
    
    56
    +@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
    
    57
    +def test_buildtree_staged_warn_empty(cli, tmpdir, datafiles):
    
    58
    +    # Test that if we stage a cached and empty, we warn the user.
    
    59
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    60
    +    element_name = 'build-shell/buildtree.bst'
    
    61
    +
    
    62
    +    # Switch to a temp artifact cache dir to ensure the artifact is rebuilt,
    
    63
    +    # caching an empty buildtree
    
    64
    +    cli.configure({
    
    65
    +        'artifactdir': os.path.join(cli.directory, 'artifacts2')
    
    66
    +    })
    
    67
    +
    
    68
    +    res = cli.run(project=project, args=['--no-cache-buildtrees', 'build', element_name])
    
    69
    +    res.assert_success()
    
    70
    +
    
    71
    +    res = cli.run(project=project, args=[
    
    72
    +        'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test'
    
    73
    +    ])
    
    74
    +    res.assert_shell_error()
    
    75
    +    assert "Artifact contains an empty buildtree" in res.stderr
    
    76
    +    shutil.rmtree(os.path.join(cli.directory, 'artifacts2'))
    
    77
    +
    
    78
    +
    
    55 79
     @pytest.mark.datafiles(DATA_DIR)
    
    56 80
     @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
    
    57 81
     def test_buildtree_staged_if_available(cli_integration, tmpdir, datafiles):
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]