[Notes] [Git][BuildStream/buildstream][caching_build_trees] 2 commits: Adding build tree extraction functionality

[Phillip Smyth <gitlab mg gitlab com>]

Title: GitLab

Phillip Smyth pushed to branch caching_build_trees at BuildStream / buildstream

Commits:

• f1fac4d4
  by Phillip Smyth at 2018-08-03T15:25:16Z

  Adding build tree extraction functionality
  
  tests/completions/completions.py: Adding caching build tree flag to list of recognised flags in test
  _frontend/cli.py: Adding caching build tree flags to cli
  _artifactcache/artifactcache.py: Adding abstract get_buildtree_dir function
  _artifactcache/cascache.py: Adding get_buildtree_dir function
  element.py: Adding function to return build tree path
  
  Adding functionality to allow the use of a cached build tree in a workspace
  
  _stream.py: Adding staging of cached build tree
  

• 8b8d06d6
  by Phillip Smyth at 2018-08-03T15:25:16Z

  tests/integration/workspace.py: Adding tests for caching of build tree and using them in a workspace
  
  Adding Makefiles
  Updating expected keys
  

8 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/_frontend/cli.py
• buildstream/_stream.py
• buildstream/element.py
• + tests/integration/project/files/workspace-no-existing-cached-buildtree/Makefile
• + tests/integration/project/files/workspace-use-cached-buildtree/Makefile
• tests/integration/workspace.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -366,7 +366,7 @@ class ArtifactCache():
366	366	#
367	367	# Returns: path to extracted artifact
368	368	#
369		- def extract(self, element, key):
	369	+ def extract(self, element, key, dest=None):
370	370	raise ImplError("Cache '{kind}' does not implement extract()"
371	371	.format(kind=type(self).__name__))
372	372
...	...	@@ -483,6 +483,21 @@ class ArtifactCache():
483	483	raise ImplError("Cache '{kind}' does not implement calculate_cache_size()"
484	484	.format(kind=type(self).__name__))
485	485
	486	+ # does get_buildtree_dir():
	487	+ #
	488	+ # Returns build tree from cache if exists
	489	+ #
	490	+ # Args:
	491	+ # element (Element): The Element whose cache is being checked
	492	+ # key: the related cache key
	493	+ # directory: the directory to return if exists
	494	+ # Returns:
	495	+ # string: directory path or None
	496	+ #
	497	+ def get_buildtree_dir(self, element, key):
	498	+ raise ImplError("Cache '{kind}' does not implement get_buildtree_dir()"
	499	+ .format(kind=type(self).__name__))
	500	+
486	501	################################################
487	502	# Local Private Methods #
488	503	################################################

buildstream/_artifactcache/cascache.py

---

...	...	@@ -75,12 +75,14 @@ class CASCache(ArtifactCache):
75	75	# This assumes that the repository doesn't have any dangling pointers
76	76	return os.path.exists(refpath)
77	77
78		- def extract(self, element, key):
	78	+ def extract(self, element, key, dest=None):
79	79	ref = self.get_artifact_fullname(element, key)
80	80
81	81	tree = self.resolve_ref(ref, update_mtime=True)
82	82
83		- dest = os.path.join(self.extractdir, element._get_project().name, element.normal_name, tree.hash)
	83	+ if dest is None:
	84	+ dest = os.path.join(self.extractdir, element._get_project().name, element.normal_name, tree.hash)
	85	+
84	86	if os.path.isdir(dest):
85	87	# artifact has already been extracted
86	88	return dest
...	...	@@ -106,6 +108,9 @@ class CASCache(ArtifactCache):
106	108
107	109	return dest
108	110
	111	+ def get_buildtree_dir(self, element, key):
	112	+ return os.path.join(self.extract(element, key), "buildtree")
	113	+
109	114	def commit(self, element, content, keys):
110	115	refs = [self.get_artifact_fullname(element, key) for key in keys]
111	116

buildstream/_frontend/cli.py

---

...	...	@@ -446,7 +446,9 @@ def pull(app, elements, deps, remote):
446	446	all: All dependencies
447	447	"""
448	448	with app.initialized(session_name="Pull"):
449		- app.stream.pull(elements, selection=deps, remote=remote)
	449	+ app.stream.pull(elements,
	450	+ selection=deps,
	451	+ remote=remote)
450	452
451	453
452	454	##################################################################
...	...	@@ -681,11 +683,14 @@ def workspace():
681	683	help="Overwrite files existing in checkout directory")
682	684	@click.option('--track', 'track_', default=False, is_flag=True,
683	685	help="Track and fetch new source references before checking out the workspace")
	686	+@click.option('--use-cached-buildtree', 'use_cached_buildtree', default='when-local',
	687	+ type=click.Choice(['use-cached', 'ignore-cached', 'when-local']),
	688	+ help="Using cached build trees")
684	689	@click.argument('element',
685	690	type=click.Path(readable=False))
686	691	@click.argument('directory', type=click.Path(file_okay=False))
687	692	@click.pass_obj
688		-def workspace_open(app, no_checkout, force, track_, element, directory):
	693	+def workspace_open(app, no_checkout, force, track_, element, directory, use_cached_buildtree):
689	694	"""Open a workspace for manual source modification"""
690	695
691	696	if os.path.exists(directory):
...	...	@@ -702,7 +707,8 @@ def workspace_open(app, no_checkout, force, track_, element, directory):
702	707	app.stream.workspace_open(element, directory,
703	708	no_checkout=no_checkout,
704	709	track_first=track_,
705		- force=force)
	710	+ force=force,
	711	+ use_cached_buildtree=use_cached_buildtree)
706	712
707	713
708	714	##################################################################
...	...	@@ -762,10 +768,13 @@ def workspace_close(app, remove_dir, all_, elements):
762	768	help="Track and fetch the latest source before resetting")
763	769	@click.option('--all', '-a', 'all_', default=False, is_flag=True,
764	770	help="Reset all open workspaces")
	771	+@click.option('--use-cached-buildtree', 'use_cached_buildtree', default='when-local',
	772	+ type=click.Choice(['use-cached', 'ignore-cached', 'when-local']),
	773	+ help="Using cached build trees")
765	774	@click.argument('elements', nargs=-1,
766	775	type=click.Path(readable=False))
767	776	@click.pass_obj
768		-def workspace_reset(app, soft, track_, all_, elements):
	777	+def workspace_reset(app, soft, track_, all_, elements, use_cached_buildtree):
769	778	"""Reset a workspace to its original state"""
770	779
771	780	# Check that the workspaces in question exist
...	...	@@ -785,7 +794,10 @@ def workspace_reset(app, soft, track_, all_, elements):
785	794	if all_:
786	795	elements = tuple(element_name for element_name, _ in app.context.get_workspaces().list())
787	796
788		- app.stream.workspace_reset(elements, soft=soft, track_first=track_)
	797	+ app.stream.workspace_reset(elements,
	798	+ soft=soft,
	799	+ track_first=track_,
	800	+ use_cached_buildtree=use_cached_buildtree)
789	801
790	802
791	803	##################################################################

buildstream/_stream.py

---

...	...	@@ -435,6 +435,29 @@ class Stream():
435	435	raise StreamError("Error while staging dependencies into a sandbox"
436	436	": '{}'".format(e), detail=e.detail, reason=e.reason) from e
437	437
	438	+ # __stage_cached_buildtree
	439	+ #
	440	+ # Stage a cached build tree if it exists
	441	+ #
	442	+ # Args:
	443	+ # use_cached_buildtree (bool): Whether or not to use cached buildtrees
	444	+ # element: element in use
	445	+ # cache_activity, stage_activity, message: (str)
	446	+ #
	447	+ # Returns:
	448	+ # (bool): True if the build tree was staged
	449	+ #
	450	+ def __stage_cached_buildtree(self, element, target_dir):
	451	+ buildtree_path = None
	452	+ if element._cached():
	453	+ with element.timed_activity("Extracting cached build tree"):
	454	+ buildtree_path = element._buildtree_path(element._get_cache_key())
	455	+ if buildtree_path is not None:
	456	+ with element.timed_activity("Staging cached build tree to {}".format(target_dir)):
	457	+ shutil.copytree(buildtree_path, target_dir)
	458	+ return True
	459	+ return False
	460	+
438	461	# workspace_open
439	462	#
440	463	# Open a project workspace
...	...	@@ -445,11 +468,25 @@ class Stream():
445	468	# no_checkout (bool): Whether to skip checking out the source
446	469	# track_first (bool): Whether to track and fetch first
447	470	# force (bool): Whether to ignore contents in an existing directory
	471	+ # use_cached_buildtree(str): Whether or not to use cached buildtrees
448	472	#
449	473	def workspace_open(self, target, directory, *,
450	474	no_checkout,
451	475	track_first,
452		- force):
	476	+ force,
	477	+ use_cached_buildtree):
	478	+
	479	+ workspaces = self._context.get_workspaces()
	480	+ assert use_cached_buildtree in ('use-cached', 'when-local', 'ignore-cached')
	481	+
	482	+ # Make cached_buildtree a boolean based on the flag assigned to it
	483	+ # If flag was `when-local`, assigning value based on whether or not the project uses a remote cache
	484	+ if use_cached_buildtree == 'use-cached':
	485	+ use_cached_buildtree = True
	486	+ elif use_cached_buildtree == 'when-local' and not self._artifacts.has_fetch_remotes():
	487	+ use_cached_buildtree = True
	488	+ else:
	489	+ use_cached_buildtree = False
453	490
454	491	if track_first:
455	492	track_targets = (target,)
...	...	@@ -462,22 +499,6 @@ class Stream():
462	499	target = elements[0]
463	500	workdir = os.path.abspath(directory)
464	501
465		- if not list(target.sources()):
466		- build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
467		- if not build_depends:
468		- raise StreamError("The given element has no sources")
469		- detail = "Try opening a workspace on one of its dependencies instead:\n"
470		- detail += " \n".join(build_depends)
471		- raise StreamError("The given element has no sources", detail=detail)
472		-
473		- workspaces = self._context.get_workspaces()
474		-
475		- # Check for workspace config
476		- workspace = workspaces.get_workspace(target._get_full_name())
477		- if workspace and not force:
478		- raise StreamError("Workspace '{}' is already defined at: {}"
479		- .format(target.name, workspace.path))
480		-
481	502	# If we're going to checkout, we need at least a fetch,
482	503	# if we were asked to track first, we're going to fetch anyway.
483	504	#
...	...	@@ -487,6 +508,26 @@ class Stream():
487	508	track_elements = elements
488	509	self._fetch(elements, track_elements=track_elements)
489	510
	511	+ # Check for workspace config
	512	+ workspace = workspaces.get_workspace(target._get_full_name())
	513	+ if workspace and not force:
	514	+ raise StreamError("Workspace '{}' is already defined at: {}"
	515	+ .format(target.name, workspace.path))
	516	+
	517	+ if use_cached_buildtree:
	518	+ if self.__stage_cached_buildtree(target, workdir):
	519	+ workspaces.save_config()
	520	+ self._message(MessageType.INFO, "Saved workspace configuration")
	521	+ return
	522	+
	523	+ if not list(target.sources()):
	524	+ build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
	525	+ if not build_depends:
	526	+ raise StreamError("The given element has no sources")
	527	+ detail = "Try opening a workspace on one of its dependencies instead:\n"
	528	+ detail += " \n".join(build_depends)
	529	+ raise StreamError("The given element has no sources", detail=detail)
	530	+
490	531	if not no_checkout and target._get_consistency() != Consistency.CACHED:
491	532	raise StreamError("Could not stage uncached source. " +
492	533	"Use `--track` to track and " +
...	...	@@ -547,8 +588,9 @@ class Stream():
547	588	# targets (list of str): The target elements to reset the workspace for
548	589	# soft (bool): Only reset workspace state
549	590	# track_first (bool): Whether to also track the sources first
	591	+ # use_cached_buildtree(str): Whether or not to use cached buildtrees
550	592	#
551		- def workspace_reset(self, targets, *, soft, track_first):
	593	+ def workspace_reset(self, targets, *, soft, track_first, use_cached_buildtree):
552	594
553	595	if track_first:
554	596	track_targets = targets
...	...	@@ -592,6 +634,15 @@ class Stream():
592	634	workspaces.delete_workspace(element._get_full_name())
593	635	workspaces.create_workspace(element._get_full_name(), workspace.path)
594	636
	637	+ if use_cached_buildtree == 'when-local':
	638	+ use_cached_buildtree = os.path.isdir(os.path.join(workspace.path, 'buildtree'))
	639	+
	640	+ if use_cached_buildtree and self.__stage_cached_buildtree(element, workspace.path):
	641	+ self._message(MessageType.INFO, "Reset workspace state for {} at: {}"
	642	+ .format(element.name, workspace.path))
	643	+ return
	644	+
	645	+ workspaces.create_workspace(element.name, workspace.path)
595	646	with element.timed_activity("Staging sources to {}".format(workspace.path)):
596	647	element._open_workspace()
597	648

buildstream/element.py

---

...	...	@@ -893,6 +893,13 @@ class Element(Plugin):
893	893	# Private Methods used in BuildStream #
894	894	#############################################################
895	895
	896	+ # _buildtree_path():
	897	+ #
	898	+ # Returns the path of the cached build tree if it exists
	899	+ #
	900	+ def _buildtree_path(self, key):
	901	+ return self.__artifacts.get_buildtree_dir(self, key)
	902	+
896	903	# _new_from_meta():
897	904	#
898	905	# Recursively instantiate a new Element instance, it's sources

tests/integration/project/files/workspace-no-existing-cached-buildtree/Makefile

---

	1	+test:
	2	+ touch test.o

tests/integration/project/files/workspace-use-cached-buildtree/Makefile

---

	1	+test:
	2	+ touch test.o

tests/integration/workspace.py

---

...	...	@@ -5,6 +5,7 @@ from buildstream import _yaml
5	5	from tests.testutils import cli_integration as cli
6	6	from tests.testutils.site import IS_LINUX
7	7	from tests.testutils.integration import walk_dir
	8	+from tests.frontend import configure_project
8	9
9	10
10	11	pytestmark = pytest.mark.integration
...	...	@@ -255,3 +256,119 @@ def test_incremental_configure_commands_run_only_once(cli, tmpdir, datafiles):
255	256	res = cli.run(project=project, args=['build', element_name])
256	257	res.assert_success()
257	258	assert not os.path.exists(os.path.join(workspace, 'prepared-again'))
	259	+
	260	+
	261	+@pytest.mark.integration
	262	+@pytest.mark.datafiles(DATA_DIR)
	263	+def test_use_cached_buildtree(cli, tmpdir, datafiles):
	264	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	265	+ workspace = os.path.join(cli.directory, 'workspace')
	266	+ element_path = os.path.join(project, 'elements')
	267	+ element_name = 'workspace/workspace-use-cached-buildtree.bst'
	268	+
	269	+ element = {
	270	+ 'kind': 'manual',
	271	+ 'depends': [{
	272	+ 'filename': 'base.bst',
	273	+ 'type': 'build'
	274	+ }],
	275	+ 'sources': [{
	276	+ 'kind': 'local',
	277	+ 'path': 'files/workspace-use-cached-buildtree'
	278	+ }],
	279	+ 'config': {
	280	+ 'build-commands': [
	281	+ 'make'
	282	+ ]
	283	+ }
	284	+ }
	285	+ os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
	286	+ _yaml.dump(element, os.path.join(element_path, element_name))
	287	+
	288	+ res = cli.run(project=project, args=['build', element_name])
	289	+ res.assert_success()
	290	+
	291	+ res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
	292	+ res.assert_success()
	293	+ assert os.path.isdir(workspace)
	294	+ assert os.path.exists(os.path.join(workspace, "test.o"))
	295	+
	296	+
	297	+@pytest.mark.integration
	298	+@pytest.mark.datafiles(DATA_DIR)
	299	+def test_dont_use_cached_buildtree(cli, tmpdir, datafiles):
	300	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	301	+ workspace = os.path.join(cli.directory, 'workspace')
	302	+ element_path = os.path.join(project, 'elements')
	303	+ element_name = 'workspace/workspace-use-cached-buildtree.bst'
	304	+
	305	+ element = {
	306	+ 'kind': 'manual',
	307	+ 'depends': [{
	308	+ 'filename': 'base.bst',
	309	+ 'type': 'build'
	310	+ }],
	311	+ 'sources': [{
	312	+ 'kind': 'local',
	313	+ 'path': 'files/workspace-use-cached-buildtree'
	314	+ }],
	315	+ 'config': {
	316	+ 'build-commands': [
	317	+ 'make'
	318	+ ]
	319	+ }
	320	+ }
	321	+ os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
	322	+ _yaml.dump(element, os.path.join(element_path, element_name))
	323	+
	324	+ res = cli.run(project=project, args=['build', element_name])
	325	+ res.assert_success()
	326	+
	327	+ res = cli.run(project=project, args=['workspace', 'open',
	328	+ '--use-cached-buildtree=ignore-cached',
	329	+ element_name, workspace])
	330	+ res.assert_success()
	331	+ assert os.path.isdir(workspace)
	332	+ assert not os.path.exists(os.path.join(workspace, "test.o"))
	333	+
	334	+
	335	+@pytest.mark.integration
	336	+@pytest.mark.datafiles(DATA_DIR)
	337	+# This tests the output if a build is done with the caching of build trees disabled
	338	+# and then is reenabled and a workspace is opened
	339	+def test_no_existing_cached_buildtree(cli, tmpdir, datafiles):
	340	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	341	+ workspace = os.path.join(cli.directory, 'workspace')
	342	+ element_path = os.path.join(project, 'elements')
	343	+ element_name = 'workspace/workspace-no-existing-cached-buildtree.bst'
	344	+
	345	+ element = {
	346	+ 'kind': 'manual',
	347	+ 'depends': [{
	348	+ 'filename': 'base.bst',
	349	+ 'type': 'build'
	350	+ }],
	351	+ 'sources': [{
	352	+ 'kind': 'local',
	353	+ 'path': 'files/workspace-no-existing-cached-buildtree'
	354	+ }],
	355	+ 'config': {
	356	+ 'build-commands': [
	357	+ 'make'
	358	+ ]
	359	+ }
	360	+ }
	361	+ os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
	362	+ _yaml.dump(element, os.path.join(element_path, element_name))
	363	+
	364	+ res = cli.run(project=project,
	365	+ project_config={'variables': {'cache-buildtree': False}},
	366	+ args=['build', element_name])
	367	+ res.assert_success()
	368	+
	369	+ res = cli.run(project=project,
	370	+ project_config={'variables': {'cache-buildtrees': True}},
	371	+ args=['workspace', 'open', element_name, workspace])
	372	+ res.assert_success()
	373	+ assert os.path.isdir(workspace)
	374	+ assert not os.path.exists(os.path.join(workspace, "test.o"))

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.