[Notes] [Git][BuildStream/buildstream][tpollard/workspacebuildtree] WIP: Opening a workspace with a cached build

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch tpollard/workspacebuildtree at BuildStream / buildstream

Commits:

• fbea7acc
  by Tom Pollard at 2018-10-17T14:48:45Z

  WIP: Opening a workspace with a cached build
  

6 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/_context.py
• buildstream/_frontend/cli.py
• buildstream/_stream.py
• buildstream/element.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -600,6 +600,21 @@ class ArtifactCache():
600	600	raise ImplError("Cache '{kind}' does not implement calculate_cache_size()"
601	601	.format(kind=type(self).__name__))
602	602
	603	+ # checkout_artifact_subdir()
	604	+ #
	605	+ # Checkout given artifact subdir into provided directory
	606	+ #
	607	+ # Args:
	608	+ # element (Element): The Element
	609	+ # key (str): The cache key to use
	610	+ # subdir (str): The subdir to checkout
	611	+ # tmpdir (str): The dir to place the buildtree content
	612	+ #
	613	+ def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
	614	+ raise ImplError("Cache '{kind}' does not implement checkout_artifact_subdir()"
	615	+ .format(kind=type(self).__name__))
	616	+
	617	+
603	618	################################################
604	619	# Local Private Methods #
605	620	################################################

buildstream/_artifactcache/cascache.py

---

...	...	@@ -442,6 +442,14 @@ class CASCache(ArtifactCache):
442	442
443	443	return pushed
444	444
	445	+ def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
	446	+ tree = self.resolve_ref(self.get_artifact_fullname(element, key))
	447	+
	448	+ # This assumes that the subdir digest is present in the element tree
	449	+ subdirdigest = self._get_subdir(tree, subdir)
	450	+ self._checkout(tmpdir, subdirdigest)
	451	+
	452	+
445	453	################################################
446	454	# API Private Methods #
447	455	################################################

buildstream/_context.py

---

...	...	@@ -113,6 +113,9 @@ class Context():
113	113	# Whether or not to attempt to pull buildtrees globally
114	114	self.pullbuildtrees = False
115	115
	116	+ # Whether to not include artifact buildtrees in workspaces if available
	117	+ self.workspacebuildtrees = True
	118	+
116	119	# Private variables
117	120	self._cache_key = None
118	121	self._message_handler = None
...	...	@@ -163,7 +166,7 @@ class Context():
163	166	_yaml.node_validate(defaults, [
164	167	'sourcedir', 'builddir', 'artifactdir', 'logdir',
165	168	'scheduler', 'artifacts', 'logging', 'projects',
166		- 'cache', 'pullbuildtrees'
	169	+ 'cache', 'pullbuildtrees', 'workspacebuildtrees'
167	170	])
168	171
169	172	for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
...	...	@@ -191,6 +194,9 @@ class Context():
191	194	# Load pull buildtrees configuration
192	195	self.pullbuildtrees = _yaml.node_get(defaults, bool, 'pullbuildtrees', default_value='False')
193	196
	197	+ # Load workspace buildtrees configuration
	198	+ self.workspacebuildtrees = _yaml.node_get(defaults, bool, 'workspacebuildtrees', default_value='True')
	199	+
194	200	# Load logging config
195	201	logging = _yaml.node_get(defaults, Mapping, 'logging')
196	202	_yaml.node_validate(logging, [

buildstream/_frontend/cli.py

---

...	...	@@ -686,12 +686,16 @@ def workspace():
686	686	help="Overwrite files existing in checkout directory")
687	687	@click.option('--track', 'track_', default=False, is_flag=True,
688	688	help="Track and fetch new source references before checking out the workspace")
	689	+@click.option('--no-cache', default=False, is_flag=True,
	690	+ help="Do not checkout the cached buildtree")
689	691	@click.argument('element',
690	692	type=click.Path(readable=False))
691	693	@click.argument('directory', type=click.Path(file_okay=False))
692	694	@click.pass_obj
693		-def workspace_open(app, no_checkout, force, track_, element, directory):
694		- """Open a workspace for manual source modification"""
	695	+def workspace_open(app, no_checkout, force, track_, no_cache, element, directory):
	696	+ """Open a workspace for manual source modification, the elements buildtree
	697	+ will be provided if available in the local artifact cache.
	698	+ """
695	699
696	700	if os.path.exists(directory):
697	701
...	...	@@ -703,11 +707,15 @@ def workspace_open(app, no_checkout, force, track_, element, directory):
703	707	click.echo("Checkout directory is not empty: {}".format(directory), err=True)
704	708	sys.exit(-1)
705	709
	710	+ if not no_cache:
	711	+ click.echo("WARNING: Workspace will be opened without the cached buildtree if not cached locally")
	712	+
706	713	with app.initialized():
707	714	app.stream.workspace_open(element, directory,
708	715	no_checkout=no_checkout,
709	716	track_first=track_,
710		- force=force)
	717	+ force=force,
	718	+ no_cache=no_cache)
711	719
712	720
713	721	##################################################################

buildstream/_stream.py

---

...	...	@@ -456,11 +456,17 @@ class Stream():
456	456	# no_checkout (bool): Whether to skip checking out the source
457	457	# track_first (bool): Whether to track and fetch first
458	458	# force (bool): Whether to ignore contents in an existing directory
	459	+ # no_cache (bool): Whether to not include the cached buildtree
459	460	#
460	461	def workspace_open(self, target, directory, *,
461	462	no_checkout,
462	463	track_first,
463		- force):
	464	+ force,
	465	+ no_cache):
	466	+
	467	+ # Override no_cache if the global user conf workspacebuildtrees is false
	468	+ if not self._context.workspacebuildtrees:
	469	+ no_cache = True
464	470
465	471	if track_first:
466	472	track_targets = (target,)
...	...	@@ -473,7 +479,21 @@ class Stream():
473	479	target = elements[0]
474	480	directory = os.path.abspath(directory)
475	481
476		- if not list(target.sources()):
	482	+ # Check if given target has a buildtree artifact cached locally
	483	+ buildtree = None
	484	+ if target._cached():
	485	+ buildtree = self._artifacts.contains_subdir_artifact(target, target._get_cache_key(), 'buildtree')
	486	+
	487	+ # If we're running in the default state, make the user aware of buildtree usage
	488	+ if not no_cache:
	489	+ if buildtree:
	490	+ self._message(MessageType.INFO, "{} buildtree artifact is available,"
	491	+ " workspace will be opened with it".format(target.name))
	492	+ else:
	493	+ self._message(MessageType.WARN, "{} buildtree artifact not available,"
	494	+ " workspace will be opened with source checkout".format(target.name))
	495	+
	496	+ if (not buildtree or no_cache) and not list(target.sources()):
477	497	build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
478	498	if not build_depends:
479	499	raise StreamError("The given element has no sources")
...	...	@@ -492,18 +512,21 @@ class Stream():
492	512	# If we're going to checkout, we need at least a fetch,
493	513	# if we were asked to track first, we're going to fetch anyway.
494	514	#
495		- if not no_checkout or track_first:
496		- track_elements = []
497		- if track_first:
498		- track_elements = elements
499		- self._fetch(elements, track_elements=track_elements)
500		-
501		- if not no_checkout and target._get_consistency() != Consistency.CACHED:
502		- raise StreamError("Could not stage uncached source. " +
503		- "Use `--track` to track and " +
504		- "fetch the latest version of the " +
505		- "source.")
506		-
	515	+ if not buildtree or no_cache:
	516	+ if not no_checkout or track_first:
	517	+ track_elements = []
	518	+ if track_first:
	519	+ track_elements = elements
	520	+ self._fetch(elements, track_elements=track_elements)
	521	+
	522	+ if not buildtree or no_cache:
	523	+ if not no_checkout and target._get_consistency() != Consistency.CACHED:
	524	+ raise StreamError("Could not stage uncached source. " +
	525	+ "Use `--track` to track and " +
	526	+ "fetch the latest version of the " +
	527	+ "source.")
	528	+
	529	+ # Presume workspace to be forced if previous StreamError not raised
507	530	if workspace:
508	531	workspaces.delete_workspace(target._get_full_name())
509	532	workspaces.save_config()
...	...	@@ -515,9 +538,14 @@ class Stream():
515	538
516	539	workspaces.create_workspace(target._get_full_name(), directory)
517	540
518		- if not no_checkout:
	541	+ if (not buildtree or no_cache) and not no_checkout:
519	542	with target.timed_activity("Staging sources to {}".format(directory)):
520	543	target._open_workspace()
	544	+ # Handle opening workspace with buildtree instead of source staging
	545	+ elif buildtree and not no_cache:
	546	+ with target.timed_activity("Staging buildtree to {}".format(directory)):
	547	+ target._open_workspace(buildtree=buildtree)
	548	+
521	549
522	550	workspaces.save_config()
523	551	self._message(MessageType.INFO, "Saved workspace configuration")

buildstream/element.py

---

...	...	@@ -1910,7 +1910,10 @@ class Element(Plugin):
1910	1910	# This requires that a workspace already be created in
1911	1911	# the workspaces metadata first.
1912	1912	#
1913		- def _open_workspace(self):
	1913	+ # Args:
	1914	+ # buildtree (bool): Whether to open workspace with artifact buildtree
	1915	+ #
	1916	+ def _open_workspace(self, buildtree=None):
1914	1917	context = self._get_context()
1915	1918	workspace = self._get_workspace()
1916	1919	assert workspace is not None
...	...	@@ -1923,11 +1926,15 @@ class Element(Plugin):
1923	1926	# files in the target directory actually works without any
1924	1927	# additional support from Source implementations.
1925	1928	#
	1929	+
1926	1930	os.makedirs(context.builddir, exist_ok=True)
1927	1931	with utils._tempdir(dir=context.builddir, prefix='workspace-{}'
1928	1932	.format(self.normal_name)) as temp:
1929		- for source in self.sources():
1930		- source._init_workspace(temp)
	1933	+ if not buildtree:
	1934	+ for source in self.sources():
	1935	+ source._init_workspace(temp)
	1936	+ else:
	1937	+ self.__artifacts.checkout_artifact_subdir(self, self._get_cache_key(), 'buildtree', temp)
1931	1938
1932	1939	# Now hardlink the files into the workspace target.
1933	1940	utils.link_files(temp, workspace.get_absolute_path())

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.