Benjamin Schubert pushed to branch bschubert/dont-cache-errors-from-host-tools at BuildStream / buildstream
Commits:
-
fe33e328
by James Ennis at 2018-11-08T17:54:18Z
-
09faf002
by James Ennis at 2018-11-08T17:54:18Z
-
d153453c
by Javier Jardón at 2018-11-08T18:22:54Z
-
d1c6ea97
by Benjamin Schubert at 2018-11-09T11:26:34Z
5 changed files:
- buildstream/_artifactcache/artifactcache.py
- buildstream/element.py
- doc/source/using_config.rst
- tests/integration/cachedfail.py
- tests/utils/misc.py
Changes:
... | ... | @@ -937,15 +937,22 @@ class ArtifactCache(): |
937 | 937 |
"Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
|
938 | 938 |
"BuildStream requires a minimum cache quota of 2G.")
|
939 | 939 |
elif cache_quota > cache_size + available_space: # Check maximum
|
940 |
+ if '%' in self.context.config_cache_quota:
|
|
941 |
+ available = (available_space / (stat.f_blocks * stat.f_bsize)) * 100
|
|
942 |
+ available = '{}% of total disk space'.format(round(available, 1))
|
|
943 |
+ else:
|
|
944 |
+ available = utils._pretty_size(available_space)
|
|
945 |
+ |
|
940 | 946 |
raise LoadError(LoadErrorReason.INVALID_DATA,
|
941 | 947 |
("Your system does not have enough available " +
|
942 | 948 |
"space to support the cache quota specified.\n" +
|
943 |
- "You currently have:\n" +
|
|
944 |
- "- {used} of cache in use at {local_cache_path}\n" +
|
|
945 |
- "- {available} of available system storage").format(
|
|
946 |
- used=utils._pretty_size(cache_size),
|
|
947 |
- local_cache_path=self.context.artifactdir,
|
|
948 |
- available=utils._pretty_size(available_space)))
|
|
949 |
+ "\nYou have specified a quota of {quota} total disk space.\n" +
|
|
950 |
+ "- The filesystem containing {local_cache_path} only " +
|
|
951 |
+ "has: {available_size} available.")
|
|
952 |
+ .format(
|
|
953 |
+ quota=self.context.config_cache_quota,
|
|
954 |
+ local_cache_path=self.context.artifactdir,
|
|
955 |
+ available_size=available))
|
|
949 | 956 |
|
950 | 957 |
# Place a slight headroom (2e9 (2GB) on the cache_quota) into
|
951 | 958 |
# cache_quota to try and avoid exceptions.
|
... | ... | @@ -85,7 +85,8 @@ import shutil |
85 | 85 |
from . import _yaml
|
86 | 86 |
from ._variables import Variables
|
87 | 87 |
from ._versions import BST_CORE_ARTIFACT_VERSION
|
88 |
-from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, ErrorDomain
|
|
88 |
+from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, \
|
|
89 |
+ ErrorDomain, SandboxError
|
|
89 | 90 |
from .utils import UtilError
|
90 | 91 |
from . import Plugin, Consistency, Scope
|
91 | 92 |
from . import SandboxFlags
|
... | ... | @@ -1554,6 +1555,8 @@ class Element(Plugin): |
1554 | 1555 |
|
1555 | 1556 |
# Call the abstract plugin methods
|
1556 | 1557 |
collect = None
|
1558 |
+ save_artifacts = True
|
|
1559 |
+ |
|
1557 | 1560 |
try:
|
1558 | 1561 |
# Step 1 - Configure
|
1559 | 1562 |
self.configure_sandbox(sandbox)
|
... | ... | @@ -1565,6 +1568,9 @@ class Element(Plugin): |
1565 | 1568 |
collect = self.assemble(sandbox) # pylint: disable=assignment-from-no-return
|
1566 | 1569 |
self.__set_build_result(success=True, description="succeeded")
|
1567 | 1570 |
except BstError as e:
|
1571 |
+ if isinstance(e, SandboxError):
|
|
1572 |
+ save_artifacts = False
|
|
1573 |
+ |
|
1568 | 1574 |
# Shelling into a sandbox is useful to debug this error
|
1569 | 1575 |
e.sandbox = True
|
1570 | 1576 |
|
... | ... | @@ -1592,100 +1598,108 @@ class Element(Plugin): |
1592 | 1598 |
self.__set_build_result(success=False, description=str(e), detail=e.detail)
|
1593 | 1599 |
raise
|
1594 | 1600 |
finally:
|
1595 |
- if collect is not None:
|
|
1596 |
- try:
|
|
1597 |
- sandbox_vroot = sandbox.get_virtual_directory()
|
|
1598 |
- collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
|
|
1599 |
- except VirtualDirectoryError:
|
|
1600 |
- # No collect directory existed
|
|
1601 |
- collectvdir = None
|
|
1602 |
- |
|
1603 |
- # Create artifact directory structure
|
|
1604 |
- assembledir = os.path.join(rootdir, 'artifact')
|
|
1605 |
- filesdir = os.path.join(assembledir, 'files')
|
|
1606 |
- logsdir = os.path.join(assembledir, 'logs')
|
|
1607 |
- metadir = os.path.join(assembledir, 'meta')
|
|
1608 |
- buildtreedir = os.path.join(assembledir, 'buildtree')
|
|
1609 |
- os.mkdir(assembledir)
|
|
1610 |
- if collect is not None and collectvdir is not None:
|
|
1611 |
- os.mkdir(filesdir)
|
|
1612 |
- os.mkdir(logsdir)
|
|
1613 |
- os.mkdir(metadir)
|
|
1614 |
- os.mkdir(buildtreedir)
|
|
1615 |
- |
|
1616 |
- # Hard link files from collect dir to files directory
|
|
1617 |
- if collect is not None and collectvdir is not None:
|
|
1618 |
- collectvdir.export_files(filesdir, can_link=True)
|
|
1619 |
- |
|
1620 |
- try:
|
|
1621 |
- sandbox_vroot = sandbox.get_virtual_directory()
|
|
1622 |
- sandbox_build_dir = sandbox_vroot.descend(
|
|
1623 |
- self.get_variable('build-root').lstrip(os.sep).split(os.sep))
|
|
1624 |
- # Hard link files from build-root dir to buildtreedir directory
|
|
1625 |
- sandbox_build_dir.export_files(buildtreedir)
|
|
1626 |
- except VirtualDirectoryError:
|
|
1627 |
- # Directory could not be found. Pre-virtual
|
|
1628 |
- # directory behaviour was to continue silently
|
|
1629 |
- # if the directory could not be found.
|
|
1630 |
- pass
|
|
1631 |
- |
|
1632 |
- # Copy build log
|
|
1633 |
- log_filename = context.get_log_filename()
|
|
1634 |
- self._build_log_path = os.path.join(logsdir, 'build.log')
|
|
1635 |
- if log_filename:
|
|
1636 |
- shutil.copyfile(log_filename, self._build_log_path)
|
|
1637 |
- |
|
1638 |
- # Store public data
|
|
1639 |
- _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
|
|
1640 |
- |
|
1641 |
- # Store result
|
|
1642 |
- build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
|
|
1643 |
- if self.__build_result[2] is not None:
|
|
1644 |
- build_result_dict["detail"] = self.__build_result[2]
|
|
1645 |
- _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
|
|
1646 |
- |
|
1647 |
- # ensure we have cache keys
|
|
1648 |
- self._assemble_done()
|
|
1649 |
- |
|
1650 |
- # Store keys.yaml
|
|
1651 |
- _yaml.dump(_yaml.node_sanitize({
|
|
1652 |
- 'strong': self._get_cache_key(),
|
|
1653 |
- 'weak': self._get_cache_key(_KeyStrength.WEAK),
|
|
1654 |
- }), os.path.join(metadir, 'keys.yaml'))
|
|
1655 |
- |
|
1656 |
- # Store dependencies.yaml
|
|
1657 |
- _yaml.dump(_yaml.node_sanitize({
|
|
1658 |
- e.name: e._get_cache_key() for e in self.dependencies(Scope.BUILD)
|
|
1659 |
- }), os.path.join(metadir, 'dependencies.yaml'))
|
|
1660 |
- |
|
1661 |
- # Store workspaced.yaml
|
|
1662 |
- _yaml.dump(_yaml.node_sanitize({
|
|
1663 |
- 'workspaced': True if self._get_workspace() else False
|
|
1664 |
- }), os.path.join(metadir, 'workspaced.yaml'))
|
|
1665 |
- |
|
1666 |
- # Store workspaced-dependencies.yaml
|
|
1667 |
- _yaml.dump(_yaml.node_sanitize({
|
|
1668 |
- 'workspaced-dependencies': [
|
|
1669 |
- e.name for e in self.dependencies(Scope.BUILD)
|
|
1670 |
- if e._get_workspace()
|
|
1671 |
- ]
|
|
1672 |
- }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
|
|
1673 |
- |
|
1674 |
- with self.timed_activity("Caching artifact"):
|
|
1675 |
- artifact_size = utils._get_dir_size(assembledir)
|
|
1676 |
- self.__artifacts.commit(self, assembledir, self.__get_cache_keys_for_commit())
|
|
1677 |
- |
|
1678 |
- if collect is not None and collectvdir is None:
|
|
1679 |
- raise ElementError(
|
|
1680 |
- "Directory '{}' was not found inside the sandbox, "
|
|
1681 |
- "unable to collect artifact contents"
|
|
1682 |
- .format(collect))
|
|
1601 |
+ if save_artifacts:
|
|
1602 |
+ artifact_size = self._cache_artifact(rootdir, sandbox, context, collect)
|
|
1603 |
+ else:
|
|
1604 |
+ artifact_size = None
|
|
1683 | 1605 |
|
1684 | 1606 |
# Finally cleanup the build dir
|
1685 | 1607 |
cleanup_rootdir()
|
1686 | 1608 |
|
1687 | 1609 |
return artifact_size
|
1688 | 1610 |
|
1611 |
+ def _cache_artifact(self, rootdir, sandbox, context, collect):
|
|
1612 |
+ if collect is not None:
|
|
1613 |
+ try:
|
|
1614 |
+ sandbox_vroot = sandbox.get_virtual_directory()
|
|
1615 |
+ collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
|
|
1616 |
+ except VirtualDirectoryError:
|
|
1617 |
+ # No collect directory existed
|
|
1618 |
+ collectvdir = None
|
|
1619 |
+ |
|
1620 |
+ # Create artifact directory structure
|
|
1621 |
+ assembledir = os.path.join(rootdir, 'artifact')
|
|
1622 |
+ filesdir = os.path.join(assembledir, 'files')
|
|
1623 |
+ logsdir = os.path.join(assembledir, 'logs')
|
|
1624 |
+ metadir = os.path.join(assembledir, 'meta')
|
|
1625 |
+ buildtreedir = os.path.join(assembledir, 'buildtree')
|
|
1626 |
+ os.mkdir(assembledir)
|
|
1627 |
+ if collect is not None and collectvdir is not None:
|
|
1628 |
+ os.mkdir(filesdir)
|
|
1629 |
+ os.mkdir(logsdir)
|
|
1630 |
+ os.mkdir(metadir)
|
|
1631 |
+ os.mkdir(buildtreedir)
|
|
1632 |
+ |
|
1633 |
+ # Hard link files from collect dir to files directory
|
|
1634 |
+ if collect is not None and collectvdir is not None:
|
|
1635 |
+ collectvdir.export_files(filesdir, can_link=True)
|
|
1636 |
+ |
|
1637 |
+ try:
|
|
1638 |
+ sandbox_vroot = sandbox.get_virtual_directory()
|
|
1639 |
+ sandbox_build_dir = sandbox_vroot.descend(
|
|
1640 |
+ self.get_variable('build-root').lstrip(os.sep).split(os.sep))
|
|
1641 |
+ # Hard link files from build-root dir to buildtreedir directory
|
|
1642 |
+ sandbox_build_dir.export_files(buildtreedir)
|
|
1643 |
+ except VirtualDirectoryError:
|
|
1644 |
+ # Directory could not be found. Pre-virtual
|
|
1645 |
+ # directory behaviour was to continue silently
|
|
1646 |
+ # if the directory could not be found.
|
|
1647 |
+ pass
|
|
1648 |
+ |
|
1649 |
+ # Copy build log
|
|
1650 |
+ log_filename = context.get_log_filename()
|
|
1651 |
+ self._build_log_path = os.path.join(logsdir, 'build.log')
|
|
1652 |
+ if log_filename:
|
|
1653 |
+ shutil.copyfile(log_filename, self._build_log_path)
|
|
1654 |
+ |
|
1655 |
+ # Store public data
|
|
1656 |
+ _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
|
|
1657 |
+ |
|
1658 |
+ # Store result
|
|
1659 |
+ build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
|
|
1660 |
+ if self.__build_result[2] is not None:
|
|
1661 |
+ build_result_dict["detail"] = self.__build_result[2]
|
|
1662 |
+ _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
|
|
1663 |
+ |
|
1664 |
+ # ensure we have cache keys
|
|
1665 |
+ self._assemble_done()
|
|
1666 |
+ |
|
1667 |
+ # Store keys.yaml
|
|
1668 |
+ _yaml.dump(_yaml.node_sanitize({
|
|
1669 |
+ 'strong': self._get_cache_key(),
|
|
1670 |
+ 'weak': self._get_cache_key(_KeyStrength.WEAK),
|
|
1671 |
+ }), os.path.join(metadir, 'keys.yaml'))
|
|
1672 |
+ |
|
1673 |
+ # Store dependencies.yaml
|
|
1674 |
+ _yaml.dump(_yaml.node_sanitize({
|
|
1675 |
+ e.name: e._get_cache_key() for e in self.dependencies(Scope.BUILD)
|
|
1676 |
+ }), os.path.join(metadir, 'dependencies.yaml'))
|
|
1677 |
+ |
|
1678 |
+ # Store workspaced.yaml
|
|
1679 |
+ _yaml.dump(_yaml.node_sanitize({
|
|
1680 |
+ 'workspaced': True if self._get_workspace() else False
|
|
1681 |
+ }), os.path.join(metadir, 'workspaced.yaml'))
|
|
1682 |
+ |
|
1683 |
+ # Store workspaced-dependencies.yaml
|
|
1684 |
+ _yaml.dump(_yaml.node_sanitize({
|
|
1685 |
+ 'workspaced-dependencies': [
|
|
1686 |
+ e.name for e in self.dependencies(Scope.BUILD)
|
|
1687 |
+ if e._get_workspace()
|
|
1688 |
+ ]
|
|
1689 |
+ }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
|
|
1690 |
+ |
|
1691 |
+ with self.timed_activity("Caching artifact"):
|
|
1692 |
+ artifact_size = utils._get_dir_size(assembledir)
|
|
1693 |
+ self.__artifacts.commit(self, assembledir, self.__get_cache_keys_for_commit())
|
|
1694 |
+ |
|
1695 |
+ if collect is not None and collectvdir is None:
|
|
1696 |
+ raise ElementError(
|
|
1697 |
+ "Directory '{}' was not found inside the sandbox, "
|
|
1698 |
+ "unable to collect artifact contents"
|
|
1699 |
+ .format(collect))
|
|
1700 |
+ |
|
1701 |
+ return artifact_size
|
|
1702 |
+ |
|
1689 | 1703 |
def _get_build_log(self):
|
1690 | 1704 |
return self._build_log_path
|
1691 | 1705 |
|
... | ... | @@ -147,6 +147,44 @@ The default mirror is defined by its name, e.g. |
147 | 147 |
``--default-mirror`` command-line option.
|
148 | 148 |
|
149 | 149 |
|
150 |
+Local cache expiry
|
|
151 |
+~~~~~~~~~~~~~~~~~~
|
|
152 |
+BuildStream locally caches artifacts, build trees, log files and sources within a
|
|
153 |
+cache located at ``~/.cache/buildstream`` (unless a $XDG_CACHE_HOME environment
|
|
154 |
+variable exists). When building large projects, this cache can get very large,
|
|
155 |
+thus BuildStream will attempt to clean up the cache automatically by expiring the least
|
|
156 |
+recently *used* artifacts.
|
|
157 |
+ |
|
158 |
+By default, cache expiry will begin once the file system which contains the cache
|
|
159 |
+approaches maximum usage. However, it is also possible to impose a quota on the local
|
|
160 |
+cache in the user configuration. This can be done in two ways:
|
|
161 |
+ |
|
162 |
+1. By restricting the maximum size of the cache directory itself.
|
|
163 |
+ |
|
164 |
+For example, to ensure that BuildStream's cache does not grow beyond 100 GB,
|
|
165 |
+simply declare the following in your user configuration (``~/.config/buildstream.conf``):
|
|
166 |
+ |
|
167 |
+.. code:: yaml
|
|
168 |
+ |
|
169 |
+ cache:
|
|
170 |
+ quota: 100G
|
|
171 |
+ |
|
172 |
+This quota defines the maximum size of the artifact cache in bytes.
|
|
173 |
+Other accepted values are: K, M, G or T (or you can simply declare the value in bytes, without the suffix).
|
|
174 |
+This uses the same format as systemd's
|
|
175 |
+`resource-control <https://www.freedesktop.org/software/systemd/man/systemd.resource-control.html>`_.
|
|
176 |
+ |
|
177 |
+2. By expiring artifacts once the file system which contains the cache exceeds a specified usage.
|
|
178 |
+ |
|
179 |
+To ensure that we start cleaning the cache once we've used 80% of local disk space (on the file system
|
|
180 |
+which mounts the cache):
|
|
181 |
+ |
|
182 |
+.. code:: yaml
|
|
183 |
+ |
|
184 |
+ cache:
|
|
185 |
+ quota: 80%
|
|
186 |
+ |
|
187 |
+ |
|
150 | 188 |
Default configuration
|
151 | 189 |
---------------------
|
152 | 190 |
The default BuildStream configuration is specified here for reference:
|
... | ... | @@ -4,6 +4,8 @@ import pytest |
4 | 4 |
from buildstream import _yaml
|
5 | 5 |
from buildstream._exceptions import ErrorDomain
|
6 | 6 |
|
7 |
+from conftest import clean_platform_cache
|
|
8 |
+ |
|
7 | 9 |
from tests.testutils import cli_integration as cli, create_artifact_share
|
8 | 10 |
from tests.testutils.site import IS_LINUX
|
9 | 11 |
|
... | ... | @@ -158,3 +160,40 @@ def test_push_cached_fail(cli, tmpdir, datafiles, on_error): |
158 | 160 |
assert cli.get_element_state(project, 'element.bst') == 'failed'
|
159 | 161 |
# This element should have been pushed to the remote
|
160 | 162 |
assert share.has_artifact('test', 'element.bst', cli.get_element_key(project, 'element.bst'))
|
163 |
+ |
|
164 |
+ |
|
165 |
+@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
|
|
166 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
167 |
+def test_host_tools_errors_are_not_cached(cli, tmpdir, datafiles):
|
|
168 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
169 |
+ element_path = os.path.join(project, 'elements', 'element.bst')
|
|
170 |
+ |
|
171 |
+ # Write out our test target
|
|
172 |
+ element = {
|
|
173 |
+ 'kind': 'script',
|
|
174 |
+ 'depends': [
|
|
175 |
+ {
|
|
176 |
+ 'filename': 'base.bst',
|
|
177 |
+ 'type': 'build',
|
|
178 |
+ },
|
|
179 |
+ ],
|
|
180 |
+ 'config': {
|
|
181 |
+ 'commands': [
|
|
182 |
+ 'true',
|
|
183 |
+ ],
|
|
184 |
+ },
|
|
185 |
+ }
|
|
186 |
+ _yaml.dump(element, element_path)
|
|
187 |
+ |
|
188 |
+ # Build without access to host tools, this will fail
|
|
189 |
+ result1 = cli.run(project=project, args=['build', 'element.bst'], env={'PATH': ''})
|
|
190 |
+ result1.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
|
|
191 |
+ assert cli.get_element_state(project, 'element.bst') == 'buildable'
|
|
192 |
+ |
|
193 |
+ # clean the cache before running again
|
|
194 |
+ clean_platform_cache()
|
|
195 |
+ |
|
196 |
+ # When rebuilding, this should work
|
|
197 |
+ result2 = cli.run(project=project, args=['build', 'element.bst'])
|
|
198 |
+ result2.assert_success()
|
|
199 |
+ assert cli.get_element_state(project, 'element.bst') == 'cached'
|
... | ... | @@ -27,4 +27,5 @@ def test_parse_size_over_1024T(cli, tmpdir): |
27 | 27 |
patched_statvfs = mock_os.mock_statvfs(f_bavail=bavail, f_bsize=BLOCK_SIZE)
|
28 | 28 |
with mock_os.monkey_patch("statvfs", patched_statvfs):
|
29 | 29 |
result = cli.run(project, args=["build", "file.bst"])
|
30 |
- assert "1025T of available system storage" in result.stderr
|
|
30 |
+ failure_msg = 'Your system does not have enough available space to support the cache quota specified.'
|
|
31 |
+ assert failure_msg in result.stderr
|