[Notes] [Git][BuildStream/buildstream][tpollard/896] WIP: Optional creation of buildtrees

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch tpollard/896 at BuildStream / buildstream

Commits:

• 6f552d60
  by Tom Pollard at 2019-02-07T19:00:37Z

  WIP: Optional creation of buildtrees
  

9 changed files:

• NEWS
• buildstream/_context.py
• buildstream/_frontend/app.py
• buildstream/_frontend/cli.py
• buildstream/data/userconfig.yaml
• buildstream/element.py
• tests/frontend/completions.py
• tests/integration/artifact.py
• tests/integration/build-tree.py → tests/integration/shellbuildtrees.py

Changes:

NEWS

---

...	...	@@ -126,6 +126,13 @@ buildstream 1.3.1
126	126	Providing a remote will limit build's pull/push remote actions to the given
127	127	remote specifically, ignoring those defined via user or project configuration.
128	128
	129	+ o Artifacts can now be cached explicitly with an empty `build tree` when built.
	130	+ Element types without a build-root were already cached with an empty build tree
	131	+ directory, this can now be extended to all artifacts to save on cache overheads.
	132	+ The cli main option 'dont-cache-buildtrees' or the user configuration cache group
	133	+ option 'dont-cache-buildtrees = True' can be set to enable this behaviour. Note, as
	134	+ the cache-key for the artifact is independant of the cached build tree input it will
	135	+ remain unaltered, however the availbility of the build tree content may differ.
129	136
130	137	=================
131	138	buildstream 1.1.5

buildstream/_context.py

---

...	...	@@ -121,6 +121,9 @@ class Context():
121	121	# Whether or not to attempt to pull build trees globally
122	122	self.pull_buildtrees = None
123	123
	124	+ # Whether or not to cache build trees on artifact creation
	125	+ self.dont_cache_buildtrees = None
	126	+
124	127	# Boolean, whether we double-check with the user that they meant to
125	128	# close the workspace when they're using it to access the project.
126	129	self.prompt_workspace_close_project_inaccessible = None
...	...	@@ -201,7 +204,7 @@ class Context():
201	204	# our artifactdir - the artifactdir may not have been created
202	205	# yet.
203	206	cache = _yaml.node_get(defaults, Mapping, 'cache')
204		- _yaml.node_validate(cache, ['quota', 'pull-buildtrees'])
	207	+ _yaml.node_validate(cache, ['quota', 'pull-buildtrees', 'dont-cache-buildtrees'])
205	208
206	209	self.config_cache_quota = _yaml.node_get(cache, str, 'quota')
207	210
...	...	@@ -213,6 +216,9 @@ class Context():
213	216	# Load pull build trees configuration
214	217	self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
215	218
	219	+ # Load cache build trees configuration
	220	+ self.dont_cache_buildtrees = _yaml.node_get(cache, bool, 'dont-cache-buildtrees')
	221	+
216	222	# Load logging config
217	223	logging = _yaml.node_get(defaults, Mapping, 'logging')
218	224	_yaml.node_validate(logging, [

buildstream/_frontend/app.py

---

...	...	@@ -183,7 +183,8 @@ class App():
183	183	'builders': 'sched_builders',
184	184	'pushers': 'sched_pushers',
185	185	'network_retries': 'sched_network_retries',
186		- 'pull_buildtrees': 'pull_buildtrees'
	186	+ 'pull_buildtrees': 'pull_buildtrees',
	187	+ 'dont_cache_buildtrees': 'dont_cache_buildtrees'
187	188	}
188	189	for cli_option, context_attr in override_map.items():
189	190	option_value = self._main_options.get(cli_option)

buildstream/_frontend/cli.py

---

...	...	@@ -252,6 +252,8 @@ def print_version(ctx, param, value):
252	252	help="The mirror to fetch from first, before attempting other mirrors")
253	253	@click.option('--pull-buildtrees', is_flag=True, default=None,
254	254	help="Include an element's build tree when pulling remote element artifacts")
	255	+@click.option('--dont-cache-buildtrees', is_flag=True, default=None,
	256	+ help="Do not cache artifact build tree content on creation")
255	257	@click.pass_context
256	258	def cli(context, **kwargs):
257	259	"""Build and manipulate BuildStream projects

buildstream/data/userconfig.yaml

---

...	...	@@ -41,6 +41,9 @@ cache:
41	41	# Whether to pull build trees when downloading element artifacts
42	42	pull-buildtrees: False
43	43
	44	+ # Whether to cache build trees on artifact creation
	45	+ dont-cache-buildtrees: False
	46	+
44	47	#
45	48	# Scheduler
46	49	#

buildstream/element.py

---

...	...	@@ -1425,6 +1425,9 @@ class Element(Plugin):
1425	1425	elif usebuildtree:
1426	1426	artifact_base, _ = self.__extract()
1427	1427	import_dir = os.path.join(artifact_base, 'buildtree')
	1428	+ if not os.listdir(import_dir):
	1429	+ detail = "Element type either does not expect a buildtree or it was explictily cached without one."
	1430	+ self.warn("WARNING: {} Artifact contains an empty buildtree".format(self.name), detail=detail)
1428	1431	else:
1429	1432	# No workspace or cached buildtree, stage source directly
1430	1433	for source in self.sources():
...	...	@@ -1631,6 +1634,8 @@ class Element(Plugin):
1631	1634	# No collect directory existed
1632	1635	collectvdir = None
1633	1636
	1637	+ context = self._get_context()
	1638	+
1634	1639	# Create artifact directory structure
1635	1640	assembledir = os.path.join(rootdir, 'artifact')
1636	1641	filesdir = os.path.join(assembledir, 'files')
...	...	@@ -1648,20 +1653,27 @@ class Element(Plugin):
1648	1653	if collect is not None and collectvdir is not None:
1649	1654	collectvdir.export_files(filesdir, can_link=True)
1650	1655
1651		- try:
1652		- sandbox_vroot = sandbox.get_virtual_directory()
1653		- sandbox_build_dir = sandbox_vroot.descend(
1654		- self.get_variable('build-root').lstrip(os.sep).split(os.sep))
1655		- # Hard link files from build-root dir to buildtreedir directory
1656		- sandbox_build_dir.export_files(buildtreedir)
1657		- except VirtualDirectoryError:
1658		- # Directory could not be found. Pre-virtual
1659		- # directory behaviour was to continue silently
1660		- # if the directory could not be found.
1661		- pass
	1656	+ dont_cache_buildtrees = context.dont_cache_buildtrees
	1657	+
	1658	+ # dont_cache_buildtrees defaults to False, as such the
	1659	+ # default behaviour is to attempt to cache them. Element
	1660	+ # types without a build-root dir will be cached with an empty
	1661	+ # buildtreedir regardless of this configuration.
	1662	+ if not dont_cache_buildtrees:
	1663	+ try:
	1664	+ sandbox_vroot = sandbox.get_virtual_directory()
	1665	+ sandbox_build_dir = sandbox_vroot.descend(
	1666	+ self.get_variable('build-root').lstrip(os.sep).split(os.sep))
	1667	+ # Hard link files from build-root dir to buildtreedir directory
	1668	+ sandbox_build_dir.export_files(buildtreedir)
	1669	+ except VirtualDirectoryError:
	1670	+ # Directory could not be found. Pre-virtual
	1671	+ # directory behaviour was to continue silently
	1672	+ # if the directory could not be found.
	1673	+ pass
1662	1674
1663	1675	# Copy build log
1664		- log_filename = self._get_context().get_log_filename()
	1676	+ log_filename = context.get_log_filename()
1665	1677	self._build_log_path = os.path.join(logsdir, 'build.log')
1666	1678	if log_filename:
1667	1679	shutil.copyfile(log_filename, self._build_log_path)
...	...	@@ -1802,7 +1814,7 @@ class Element(Plugin):
1802	1814	return True
1803	1815
1804	1816	# Do not push elements that aren't cached, or that are cached with a dangling buildtree
1805		- # artifact unless element type is expected to have an an empty buildtree directory
	1817	+ # ref unless element type is expected to have an an empty buildtree directory
1806	1818	if not self._cached_buildtree():
1807	1819	return True
1808	1820
...	...	@@ -2004,6 +2016,8 @@ class Element(Plugin):
2004	2016	# Returns:
2005	2017	# (bool): True if artifact cached with buildtree, False if
2006	2018	# element not cached or missing expected buildtree.
	2019	+ # Note this only confirms if a buildtree is present,
	2020	+ # not it's contents.
2007	2021	#
2008	2022	def _cached_buildtree(self):
2009	2023	context = self._get_context()

tests/frontend/completions.py

---

...	...	@@ -28,6 +28,7 @@ MAIN_OPTIONS = [
28	28	"--debug ",
29	29	"--default-mirror ",
30	30	"--directory ",
	31	+ "--dont-cache-buildtrees ",
31	32	"--error-lines ",
32	33	"--fetchers ",
33	34	"--log-file ",

tests/integration/artifact.py

---

...	...	@@ -20,9 +20,11 @@
20	20
21	21	import os
22	22	import pytest
	23	+import shutil
23	24
24		-from tests.testutils import cli_integration as cli
25		-
	25	+from tests.testutils import cli_integration as cli, create_artifact_share
	26	+from tests.testutils.site import HAVE_SANDBOX
	27	+from buildstream._exceptions import ErrorDomain
26	28
27	29	pytestmark = pytest.mark.integration
28	30
...	...	@@ -66,3 +68,86 @@ def test_artifact_log(cli, tmpdir, datafiles):
66	68	assert result.exit_code == 0
67	69	# The artifact is cached under both a strong key and a weak key
68	70	assert (log + log) == result.output
	71	+
	72	+
	73	+# A test to capture the integration of the pullbuildtrees
	74	+# behaviour, which by default is to not include the buildtree
	75	+# directory of an element.
	76	+@pytest.mark.integration
	77	+@pytest.mark.datafiles(DATA_DIR)
	78	+@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
	79	+def test_dont_cache_buildtrees(cli, tmpdir, datafiles):
	80	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	81	+ element_name = 'autotools/amhello.bst'
	82	+
	83	+ # Create artifact shares for pull & push testing
	84	+ with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
	85	+ create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2:
	86	+ cli.configure({
	87	+ 'artifacts': {'url': share1.repo, 'push': True},
	88	+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
	89	+ })
	90	+
	91	+ # Build autotools element with --dont-cache-buildtrees set via the
	92	+ # cli. The artifact should be successfully pushed to the share1 remote
	93	+ # and cached locally with an 'empty' buildtree digest, as it's not a
	94	+ # dangling ref
	95	+ result = cli.run(project=project, args=['--dont-cache-buildtrees', 'build', element_name])
	96	+ assert result.exit_code == 0
	97	+ assert cli.get_element_state(project, element_name) == 'cached'
	98	+ assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
	99	+
	100	+ # The extracted buildtree dir should be empty, as we set the config
	101	+ # to not cache buildtrees
	102	+ cache_key = cli.get_element_key(project, element_name)
	103	+ elementdigest = share1.has_artifact('test', element_name, cache_key)
	104	+ buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
	105	+ elementdigest.hash, 'buildtree')
	106	+ assert os.path.isdir(buildtreedir)
	107	+ assert not os.listdir(buildtreedir)
	108	+
	109	+ # Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees
	110	+ # that is was cached in share1 as expected with an empty buildtree dir
	111	+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
	112	+ assert cli.get_element_state(project, element_name) != 'cached'
	113	+ result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
	114	+ assert element_name in result.get_pulled_elements()
	115	+ assert os.path.isdir(buildtreedir)
	116	+ assert not os.listdir(buildtreedir)
	117	+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
	118	+
	119	+ # Assert that the default behaviour of pull to not include buildtrees on the artifact
	120	+ # in share1 which was purposely cached with an empty one behaves as expected. As such the
	121	+ # pulled artifact will have a dangling ref for the buildtree dir, regardless of content,
	122	+ # leading to no buildtreedir being extracted
	123	+ result = cli.run(project=project, args=['artifact', 'pull', element_name])
	124	+ assert element_name in result.get_pulled_elements()
	125	+ assert not os.path.isdir(buildtreedir)
	126	+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
	127	+
	128	+ # Repeat building the artifacts, this time with the default behaviour of caching buildtrees,
	129	+ # as such the buildtree dir should not be empty
	130	+ cli.configure({
	131	+ 'artifacts': {'url': share2.repo, 'push': True},
	132	+ 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
	133	+ })
	134	+ result = cli.run(project=project, args=['build', element_name])
	135	+ assert result.exit_code == 0
	136	+ assert cli.get_element_state(project, element_name) == 'cached'
	137	+ assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
	138	+
	139	+ # Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths
	140	+ elementdigest = share2.has_artifact('test', element_name, cache_key)
	141	+ buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
	142	+ elementdigest.hash, 'buildtree')
	143	+ assert os.path.isdir(buildtreedir)
	144	+ assert os.listdir(buildtreedir) is not None
	145	+
	146	+ # Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees
	147	+ # that is was cached in share2 as expected with an empty populated buildtree dir
	148	+ shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
	149	+ assert cli.get_element_state(project, element_name) != 'cached'
	150	+ result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
	151	+ assert element_name in result.get_pulled_elements()
	152	+ assert os.path.isdir(buildtreedir)
	153	+ assert os.listdir(buildtreedir) is not None

tests/integration/build-tree.py → tests/integration/shellbuildtrees.py

---

...	...	@@ -51,6 +51,30 @@ def test_buildtree_staged_forced_true(cli_integration, tmpdir, datafiles):
51	51	assert 'Hi' in res.output
52	52
53	53
	54	+@pytest.mark.datafiles(DATA_DIR)
	55	+@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
	56	+def test_buildtree_staged_warn_empty(cli, tmpdir, datafiles):
	57	+ # Test that if we stage a cached and empty, we warn the user.
	58	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	59	+ element_name = 'build-shell/buildtree.bst'
	60	+
	61	+ # Switch to a temp artifact cache dir to ensure the artifact is rebuilt,
	62	+ # caching an empty buildtree
	63	+ cli.configure({
	64	+ 'artifactdir': os.path.join(cli.directory, 'artifacts2')
	65	+ })
	66	+
	67	+ res = cli.run(project=project, args=['--dont-cache-buildtrees', 'build', element_name])
	68	+ res.assert_success()
	69	+
	70	+ res = cli.run(project=project, args=[
	71	+ 'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test'
	72	+ ])
	73	+ res.assert_shell_error()
	74	+ assert "Artifact contains an empty buildtree" in res.stderr
	75	+ shutil.rmtree(os.path.join(cli.directory, 'artifacts2'))
	76	+
	77	+
54	78	@pytest.mark.datafiles(DATA_DIR)
55	79	@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
56	80	def test_buildtree_staged_if_available(cli_integration, tmpdir, datafiles):

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.