[Notes] [Git][BuildStream/buildstream][bschubert/dont-cache-errors-from-host-tools] 3 commits: plugin.py: Redirect DEBUG from `__del__` to `sys.stderr`

[Benjamin Schubert <gitlab mg gitlab com>]

Title: GitLab

Benjamin Schubert pushed to branch bschubert/dont-cache-errors-from-host-tools at BuildStream / buildstream

Commits:

• 7738f6df
  by Daniel Silverstone at 2018-11-09T11:25:44Z

  plugin.py: Redirect DEBUG from `__del__` to `sys.stderr`
  
  Previously, debug messages from `Plugin.__del__()` went to stdout
  which was a problem for the test suite in extremely rare and hard to
  predict circumstances.  This corrects that by sending the message to
  stderr where all the rest of the messages tend to go.
  
  Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>
  

• 62f59eaa
  by Valentin David at 2018-11-09T13:30:06Z

  Merge branch 'danielsilverstone-ct/plugin-destroy-debug-to-stderr' into 'master'
  
  plugin.py: Redirect DEBUG from `__del__` to `sys.stderr`
  
  See merge request BuildStream/buildstream!944

• a7f023ac
  by Benjamin Schubert at 2018-11-09T13:44:24Z

  Don't cache sandbox errors
  
  Sandbox errors (like missing host tools) are dependent on the host
  system and rarely on what is actually done.
  
  It is therefore better to not cache them as they are subject to
  change between two runs.
  
  Also add test to ensure sandbox failure are not cached
  

4 changed files:

• buildstream/element.py
• buildstream/plugin.py
• conftest.py
• tests/integration/cachedfail.py

Changes:

buildstream/element.py

---

...	...	@@ -85,7 +85,8 @@ import shutil
85	85	from . import _yaml
86	86	from ._variables import Variables
87	87	from ._versions import BST_CORE_ARTIFACT_VERSION
88		-from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, ErrorDomain
	88	+from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, \
	89	+ ErrorDomain, SandboxError
89	90	from .utils import UtilError
90	91	from . import Plugin, Consistency, Scope
91	92	from . import SandboxFlags
...	...	@@ -1554,6 +1555,8 @@ class Element(Plugin):
1554	1555
1555	1556	# Call the abstract plugin methods
1556	1557	collect = None
	1558	+ save_artifacts = True
	1559	+
1557	1560	try:
1558	1561	# Step 1 - Configure
1559	1562	self.configure_sandbox(sandbox)
...	...	@@ -1565,6 +1568,9 @@ class Element(Plugin):
1565	1568	collect = self.assemble(sandbox) # pylint: disable=assignment-from-no-return
1566	1569	self.__set_build_result(success=True, description="succeeded")
1567	1570	except BstError as e:
	1571	+ if isinstance(e, SandboxError):
	1572	+ save_artifacts = False
	1573	+
1568	1574	# Shelling into a sandbox is useful to debug this error
1569	1575	e.sandbox = True
1570	1576
...	...	@@ -1592,100 +1598,108 @@ class Element(Plugin):
1592	1598	self.__set_build_result(success=False, description=str(e), detail=e.detail)
1593	1599	raise
1594	1600	finally:
1595		- if collect is not None:
1596		- try:
1597		- sandbox_vroot = sandbox.get_virtual_directory()
1598		- collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
1599		- except VirtualDirectoryError:
1600		- # No collect directory existed
1601		- collectvdir = None
1602		-
1603		- # Create artifact directory structure
1604		- assembledir = os.path.join(rootdir, 'artifact')
1605		- filesdir = os.path.join(assembledir, 'files')
1606		- logsdir = os.path.join(assembledir, 'logs')
1607		- metadir = os.path.join(assembledir, 'meta')
1608		- buildtreedir = os.path.join(assembledir, 'buildtree')
1609		- os.mkdir(assembledir)
1610		- if collect is not None and collectvdir is not None:
1611		- os.mkdir(filesdir)
1612		- os.mkdir(logsdir)
1613		- os.mkdir(metadir)
1614		- os.mkdir(buildtreedir)
1615		-
1616		- # Hard link files from collect dir to files directory
1617		- if collect is not None and collectvdir is not None:
1618		- collectvdir.export_files(filesdir, can_link=True)
1619		-
1620		- try:
1621		- sandbox_vroot = sandbox.get_virtual_directory()
1622		- sandbox_build_dir = sandbox_vroot.descend(
1623		- self.get_variable('build-root').lstrip(os.sep).split(os.sep))
1624		- # Hard link files from build-root dir to buildtreedir directory
1625		- sandbox_build_dir.export_files(buildtreedir)
1626		- except VirtualDirectoryError:
1627		- # Directory could not be found. Pre-virtual
1628		- # directory behaviour was to continue silently
1629		- # if the directory could not be found.
1630		- pass
1631		-
1632		- # Copy build log
1633		- log_filename = context.get_log_filename()
1634		- self._build_log_path = os.path.join(logsdir, 'build.log')
1635		- if log_filename:
1636		- shutil.copyfile(log_filename, self._build_log_path)
1637		-
1638		- # Store public data
1639		- _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
1640		-
1641		- # Store result
1642		- build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
1643		- if self.__build_result[2] is not None:
1644		- build_result_dict["detail"] = self.__build_result[2]
1645		- _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
1646		-
1647		- # ensure we have cache keys
1648		- self._assemble_done()
1649		-
1650		- # Store keys.yaml
1651		- _yaml.dump(_yaml.node_sanitize({
1652		- 'strong': self._get_cache_key(),
1653		- 'weak': self._get_cache_key(_KeyStrength.WEAK),
1654		- }), os.path.join(metadir, 'keys.yaml'))
1655		-
1656		- # Store dependencies.yaml
1657		- _yaml.dump(_yaml.node_sanitize({
1658		- e.name: e._get_cache_key() for e in self.dependencies(Scope.BUILD)
1659		- }), os.path.join(metadir, 'dependencies.yaml'))
1660		-
1661		- # Store workspaced.yaml
1662		- _yaml.dump(_yaml.node_sanitize({
1663		- 'workspaced': True if self._get_workspace() else False
1664		- }), os.path.join(metadir, 'workspaced.yaml'))
1665		-
1666		- # Store workspaced-dependencies.yaml
1667		- _yaml.dump(_yaml.node_sanitize({
1668		- 'workspaced-dependencies': [
1669		- e.name for e in self.dependencies(Scope.BUILD)
1670		- if e._get_workspace()
1671		- ]
1672		- }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
1673		-
1674		- with self.timed_activity("Caching artifact"):
1675		- artifact_size = utils._get_dir_size(assembledir)
1676		- self.__artifacts.commit(self, assembledir, self.__get_cache_keys_for_commit())
1677		-
1678		- if collect is not None and collectvdir is None:
1679		- raise ElementError(
1680		- "Directory '{}' was not found inside the sandbox, "
1681		- "unable to collect artifact contents"
1682		- .format(collect))
	1601	+ if save_artifacts:
	1602	+ artifact_size = self._cache_artifact(rootdir, sandbox, context, collect)
	1603	+ else:
	1604	+ artifact_size = None
1683	1605
1684	1606	# Finally cleanup the build dir
1685	1607	cleanup_rootdir()
1686	1608
1687	1609	return artifact_size
1688	1610
	1611	+ def _cache_artifact(self, rootdir, sandbox, context, collect):
	1612	+ if collect is not None:
	1613	+ try:
	1614	+ sandbox_vroot = sandbox.get_virtual_directory()
	1615	+ collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
	1616	+ except VirtualDirectoryError:
	1617	+ # No collect directory existed
	1618	+ collectvdir = None
	1619	+
	1620	+ # Create artifact directory structure
	1621	+ assembledir = os.path.join(rootdir, 'artifact')
	1622	+ filesdir = os.path.join(assembledir, 'files')
	1623	+ logsdir = os.path.join(assembledir, 'logs')
	1624	+ metadir = os.path.join(assembledir, 'meta')
	1625	+ buildtreedir = os.path.join(assembledir, 'buildtree')
	1626	+ os.mkdir(assembledir)
	1627	+ if collect is not None and collectvdir is not None:
	1628	+ os.mkdir(filesdir)
	1629	+ os.mkdir(logsdir)
	1630	+ os.mkdir(metadir)
	1631	+ os.mkdir(buildtreedir)
	1632	+
	1633	+ # Hard link files from collect dir to files directory
	1634	+ if collect is not None and collectvdir is not None:
	1635	+ collectvdir.export_files(filesdir, can_link=True)
	1636	+
	1637	+ try:
	1638	+ sandbox_vroot = sandbox.get_virtual_directory()
	1639	+ sandbox_build_dir = sandbox_vroot.descend(
	1640	+ self.get_variable('build-root').lstrip(os.sep).split(os.sep))
	1641	+ # Hard link files from build-root dir to buildtreedir directory
	1642	+ sandbox_build_dir.export_files(buildtreedir)
	1643	+ except VirtualDirectoryError:
	1644	+ # Directory could not be found. Pre-virtual
	1645	+ # directory behaviour was to continue silently
	1646	+ # if the directory could not be found.
	1647	+ pass
	1648	+
	1649	+ # Copy build log
	1650	+ log_filename = context.get_log_filename()
	1651	+ self._build_log_path = os.path.join(logsdir, 'build.log')
	1652	+ if log_filename:
	1653	+ shutil.copyfile(log_filename, self._build_log_path)
	1654	+
	1655	+ # Store public data
	1656	+ _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
	1657	+
	1658	+ # Store result
	1659	+ build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
	1660	+ if self.__build_result[2] is not None:
	1661	+ build_result_dict["detail"] = self.__build_result[2]
	1662	+ _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
	1663	+
	1664	+ # ensure we have cache keys
	1665	+ self._assemble_done()
	1666	+
	1667	+ # Store keys.yaml
	1668	+ _yaml.dump(_yaml.node_sanitize({
	1669	+ 'strong': self._get_cache_key(),
	1670	+ 'weak': self._get_cache_key(_KeyStrength.WEAK),
	1671	+ }), os.path.join(metadir, 'keys.yaml'))
	1672	+
	1673	+ # Store dependencies.yaml
	1674	+ _yaml.dump(_yaml.node_sanitize({
	1675	+ e.name: e._get_cache_key() for e in self.dependencies(Scope.BUILD)
	1676	+ }), os.path.join(metadir, 'dependencies.yaml'))
	1677	+
	1678	+ # Store workspaced.yaml
	1679	+ _yaml.dump(_yaml.node_sanitize({
	1680	+ 'workspaced': True if self._get_workspace() else False
	1681	+ }), os.path.join(metadir, 'workspaced.yaml'))
	1682	+
	1683	+ # Store workspaced-dependencies.yaml
	1684	+ _yaml.dump(_yaml.node_sanitize({
	1685	+ 'workspaced-dependencies': [
	1686	+ e.name for e in self.dependencies(Scope.BUILD)
	1687	+ if e._get_workspace()
	1688	+ ]
	1689	+ }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
	1690	+
	1691	+ with self.timed_activity("Caching artifact"):
	1692	+ artifact_size = utils._get_dir_size(assembledir)
	1693	+ self.__artifacts.commit(self, assembledir, self.__get_cache_keys_for_commit())
	1694	+
	1695	+ if collect is not None and collectvdir is None:
	1696	+ raise ElementError(
	1697	+ "Directory '{}' was not found inside the sandbox, "
	1698	+ "unable to collect artifact contents"
	1699	+ .format(collect))
	1700	+
	1701	+ return artifact_size
	1702	+
1689	1703	def _get_build_log(self):
1690	1704	return self._build_log_path
1691	1705

buildstream/plugin.py

---

...	...	@@ -111,6 +111,7 @@ Class Reference
111	111
112	112	import os
113	113	import subprocess
	114	+import sys
114	115	from contextlib import contextmanager
115	116	from weakref import WeakValueDictionary
116	117
...	...	@@ -190,7 +191,7 @@ class Plugin():
190	191	# Dont send anything through the Message() pipeline at destruction time,
191	192	# any subsequent lookup of plugin by unique id would raise KeyError.
192	193	if self.__context.log_debug:
193		- print("DEBUG: Destroyed: {}".format(self))
	194	+ sys.stderr.write("DEBUG: Destroyed: {}\n".format(self))
194	195
195	196	def __str__(self):
196	197	return "{kind} {typetag} at {provenance}".format(

conftest.py

---

...	...	@@ -56,6 +56,10 @@ def integration_cache(request):
56	56	pass
57	57
58	58
59		-@pytest.fixture(autouse=True)
60	59	def clean_platform_cache():
61	60	Platform._instance = None
	61	+
	62	+
	63	+@pytest.fixture(autouse=True)
	64	+def ensure_platform_cache_is_clean():
	65	+ clean_platform_cache()

tests/integration/cachedfail.py

---

...	...	@@ -4,6 +4,8 @@ import pytest
4	4	from buildstream import _yaml
5	5	from buildstream._exceptions import ErrorDomain
6	6
	7	+from conftest import clean_platform_cache
	8	+
7	9	from tests.testutils import cli_integration as cli, create_artifact_share
8	10	from tests.testutils.site import IS_LINUX
9	11
...	...	@@ -158,3 +160,40 @@ def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
158	160	assert cli.get_element_state(project, 'element.bst') == 'failed'
159	161	# This element should have been pushed to the remote
160	162	assert share.has_artifact('test', 'element.bst', cli.get_element_key(project, 'element.bst'))
	163	+
	164	+
	165	+@pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
	166	+@pytest.mark.datafiles(DATA_DIR)
	167	+def test_host_tools_errors_are_not_cached(cli, tmpdir, datafiles):
	168	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	169	+ element_path = os.path.join(project, 'elements', 'element.bst')
	170	+
	171	+ # Write out our test target
	172	+ element = {
	173	+ 'kind': 'script',
	174	+ 'depends': [
	175	+ {
	176	+ 'filename': 'base.bst',
	177	+ 'type': 'build',
	178	+ },
	179	+ ],
	180	+ 'config': {
	181	+ 'commands': [
	182	+ 'true',
	183	+ ],
	184	+ },
	185	+ }
	186	+ _yaml.dump(element, element_path)
	187	+
	188	+ # Build without access to host tools, this will fail
	189	+ result1 = cli.run(project=project, args=['build', 'element.bst'], env={'PATH': ''})
	190	+ result1.assert_task_error(ErrorDomain.SANDBOX, 'unavailable-local-sandbox')
	191	+ assert cli.get_element_state(project, 'element.bst') == 'buildable'
	192	+
	193	+ # clean the cache before running again
	194	+ clean_platform_cache()
	195	+
	196	+ # When rebuilding, this should work
	197	+ result2 = cli.run(project=project, args=['build', 'element.bst'])
	198	+ result2.assert_success()
	199	+ assert cli.get_element_state(project, 'element.bst') == 'cached'

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.