[Notes] [Git][BuildStream/buildstream][jmac/remote_execution_rebase] 6 commits: Add "remote-execution" configuration option.

[Jim MacArthur <gitlab mg gitlab com>]

Title: GitLab

Jim MacArthur pushed to branch jmac/remote_execution_rebase at BuildStream / buildstream

Commits:

• 5f7f7a20
  by Jim MacArthur at 2018-08-17T09:06:37Z

  Add "remote-execution" configuration option.
  
  This just adds one option, "remote-execution/url". Affects multiple files.
  

• a2d7b910
  by Jim MacArthur at 2018-08-17T09:09:36Z

  linux.py: Switch back to bwrap since we now have the configuration option
  

• b2a9027f
  by Jim MacArthur at 2018-08-17T09:10:05Z

  autotools.py: Mark this as a BST_VIRTUAL_DIRECTORY plugin
  

• 7b763b95
  by Jim MacArthur at 2018-08-17T09:11:14Z

  element.py: Switch to SandboxRemote if config option is set
  

• c067b85e
  by Jim MacArthur at 2018-08-17T09:13:09Z

  _sandboxremote: Use the remote url from configuration
  

• 29dab6fc
  by Jim MacArthur at 2018-08-17T09:13:37Z

  _sandboxremote.py: More helpful error message if GRPC fails
  

8 changed files:

• buildstream/_loader/loader.py
• buildstream/_loader/metaelement.py
• buildstream/_loader/types.py
• buildstream/_platform/linux.py
• buildstream/_project.py
• buildstream/element.py
• buildstream/plugins/elements/autotools.py
• buildstream/sandbox/_sandboxremote.py

Changes:

buildstream/_loader/loader.py

---

...	...	@@ -446,6 +446,7 @@ class Loader():
446	446	_yaml.node_get(node, list, Symbol.ENV_NOCACHE, default_value=[]),
447	447	_yaml.node_get(node, Mapping, Symbol.PUBLIC, default_value={}),
448	448	_yaml.node_get(node, Mapping, Symbol.SANDBOX, default_value={}),
	449	+ _yaml.node_get(node, Mapping, Symbol.REMOTE_EXECUTION, default_value={}),
449	450	element_kind == 'junction')
450	451
451	452	# Cache it now, make sure it's already there before recursing

buildstream/_loader/metaelement.py

---

...	...	@@ -39,7 +39,7 @@ class MetaElement():
39	39	# first_pass: The element is to be loaded with first pass configuration (junction)
40	40	#
41	41	def __init__(self, project, name, kind, provenance, sources, config,
42		- variables, environment, env_nocache, public, sandbox,
	42	+ variables, environment, env_nocache, public, sandbox, remote_execution,
43	43	first_pass):
44	44	self.project = project
45	45	self.name = name
...	...	@@ -52,6 +52,7 @@ class MetaElement():
52	52	self.env_nocache = env_nocache
53	53	self.public = public
54	54	self.sandbox = sandbox
	55	+ self.remote_execution = remote_execution
55	56	self.build_dependencies = []
56	57	self.dependencies = []
57	58	self.first_pass = first_pass

buildstream/_loader/types.py

---

...	...	@@ -39,6 +39,7 @@ class Symbol():
39	39	DIRECTORY = "directory"
40	40	JUNCTION = "junction"
41	41	SANDBOX = "sandbox"
	42	+ REMOTE_EXECUTION = "remote-execution"
42	43
43	44
44	45	# Dependency()

buildstream/_platform/linux.py

---

...	...	@@ -47,7 +47,7 @@ class Linux(Platform):
47	47	# Inform the bubblewrap sandbox as to whether it can use user namespaces or not
48	48	kwargs['user_ns_available'] = self._user_ns_available
49	49	kwargs['die_with_parent_available'] = self._die_with_parent_available
50		- return SandboxRemote(*args, **kwargs)
	50	+ return SandboxBwrap(*args, **kwargs)
51	51
52	52	################################################
53	53	# Private Methods #

buildstream/_project.py

---

...	...	@@ -127,6 +127,7 @@ class Project():
127	127
128	128	self.artifact_cache_specs = None
129	129	self._sandbox = None
	130	+ self._remote_execution = None
130	131	self._splits = None
131	132
132	133	self._context.add_project(self)
...	...	@@ -458,7 +459,8 @@ class Project():
458	459	'aliases', 'name',
459	460	'artifacts', 'options',
460	461	'fail-on-overlap', 'shell',
461		- 'ref-storage', 'sandbox', 'mirrors'
	462	+ 'ref-storage', 'sandbox',
	463	+ 'remote-execution', 'mirrors'
462	464	])
463	465
464	466	#
...	...	@@ -476,6 +478,9 @@ class Project():
476	478	# Load sandbox configuration
477	479	self._sandbox = _yaml.node_get(config, Mapping, 'sandbox')
478	480
	481	+ # Load remote execution configuration
	482	+ self._remote_execution = _yaml.node_get(config, Mapping, 'remote-execution')
	483	+
479	484	# Load project split rules
480	485	self._splits = _yaml.node_get(config, Mapping, 'split-rules')
481	486

buildstream/element.py

---

...	...	@@ -94,6 +94,7 @@ from . import _signals
94	94	from . import _site
95	95	from ._platform import Platform
96	96	from .sandbox._config import SandboxConfig
	97	+from .sandbox._sandboxremote import SandboxRemote
97	98
98	99	from .storage.directory import Directory
99	100	from .storage._filebaseddirectory import FileBasedDirectory
...	...	@@ -249,6 +250,9 @@ class Element(Plugin):
249	250	# Extract Sandbox config
250	251	self.__sandbox_config = self.__extract_sandbox_config(meta)
251	252
	253	+ # Extract remote execution URL
	254	+ self.__remote_execution_url = self.__extract_remote_execution_config(meta)
	255	+
252	256	def __lt__(self, other):
253	257	return self.name < other.name
254	258
...	...	@@ -2122,7 +2126,24 @@ class Element(Plugin):
2122	2126	project = self._get_project()
2123	2127	platform = Platform.get_platform()
2124	2128
2125		- if directory is not None and os.path.exists(directory):
	2129	+ if self.__remote_execution_url is not None and self.BST_VIRTUAL_DIRECTORY:
	2130	+ if not self.__artifacts.has_push_remotes(element=self):
	2131	+ # Give an early warning if remote execution will not work
	2132	+ raise ElementError("Artifact {} is configured to use remote execution but has no push remotes. "
	2133	+ .format(self.name) +
	2134	+ "The remote artifact server(s) may not be correctly configured or contactable.")
	2135	+
	2136	+ self.info("Using a remote 'sandbox' for artifact {}".format(self.name))
	2137	+ sandbox = SandboxRemote(context, project,
	2138	+ directory,
	2139	+ stdout=stdout,
	2140	+ stderr=stderr,
	2141	+ config=config,
	2142	+ server_url=self.__remote_execution_url,
	2143	+ allow_real_directory=False)
	2144	+ yield sandbox
	2145	+ elif directory is not None and os.path.exists(directory):
	2146	+ self.info("Using a local sandbox for artifact {}".format(self.name))
2126	2147	sandbox = platform.create_sandbox(context, project,
2127	2148	directory,
2128	2149	stdout=stdout,
...	...	@@ -2294,6 +2315,24 @@ class Element(Plugin):
2294	2315	return SandboxConfig(self.node_get_member(sandbox_config, int, 'build-uid'),
2295	2316	self.node_get_member(sandbox_config, int, 'build-gid'))
2296	2317
	2318	+ def __extract_remote_execution_config(self, meta):
	2319	+ project = self._get_project()
	2320	+ project.ensure_fully_loaded()
	2321	+ rexec_config = _yaml.node_chain_copy(project._remote_execution)
	2322	+
	2323	+ # The default config is already composited with the project overrides
	2324	+ rexec_defaults = _yaml.node_get(self.__defaults, Mapping, 'remote-execution', default_value={})
	2325	+ rexec_defaults = _yaml.node_chain_copy(rexec_defaults)
	2326	+
	2327	+ _yaml.composite(rexec_config, rexec_defaults)
	2328	+ _yaml.composite(rexec_config, meta.remote_execution)
	2329	+ _yaml.node_final_assertions(rexec_config)
	2330	+
	2331	+ # Rexec config, unlike others, has fixed members so we should validate them
	2332	+ _yaml.node_validate(rexec_config, ['url'])
	2333	+
	2334	+ return self.node_get_member(rexec_config, str, 'url')
	2335	+
2297	2336	# This makes a special exception for the split rules, which
2298	2337	# elements may extend but whos defaults are defined in the project.
2299	2338	#

buildstream/plugins/elements/autotools.py

---

...	...	@@ -57,7 +57,7 @@ from buildstream import BuildElement
57	57
58	58	# Element implementation for the 'autotools' kind.
59	59	class AutotoolsElement(BuildElement):
60		- pass
	60	+ BST_VIRTUAL_DIRECTORY = True
61	61
62	62
63	63	# Plugin entry point

buildstream/sandbox/_sandboxremote.py

---

...	...	@@ -19,6 +19,7 @@
19	19	# Jim MacArthur <jim macarthur codethink co uk>
20	20
21	21	import os
	22	+import re
22	23	import time
23	24
24	25	import grpc
...	...	@@ -45,9 +46,16 @@ class SandboxRemote(Sandbox):
45	46
46	47	def __init__(self, *args, **kwargs):
47	48	super().__init__(*args, **kwargs)
48		- self.user_ns_available = kwargs['user_ns_available']
49		- self.die_with_parent_available = kwargs['die_with_parent_available']
50	49	self.cascache = None
	50	+ self.server_url = kwargs['server_url']
	51	+ # Check the format of the url ourselves to save the user from
	52	+ # whatever error messages grpc will produce
	53	+ m = re.match('^(.+):(\d+)$', self.server_url)
	54	+ if m is None:
	55	+ raise SandboxError("Configured remote URL '{}' does not match the expected layout. " +
	56	+ "It should be of the form <protocol>://<domain name>:<port>."
	57	+ .format(server_url))
	58	+
51	59
52	60	def _get_cascache(self):
53	61	if self.cascache is None:
...	...	@@ -92,16 +100,16 @@ class SandboxRemote(Sandbox):
92	100	return None
93	101
94	102	# Next, try to create a communication channel to the BuildGrid server.
95		- port = 50051
96		- channel = grpc.insecure_channel('dekatron.office.codethink.co.uk:{}'.format(port))
97		- stub = remote_execution_pb2_grpc.ExecutionStub(channel)
98	103
	104	+ channel = grpc.insecure_channel(self.server_url)
	105	+ stub = remote_execution_pb2_grpc.ExecutionStub(channel)
99	106	request = remote_execution_pb2.ExecuteRequest(instance_name='default',
100	107	action_digest=action_digest,
101	108	skip_cache_lookup=True)
102	109
103	110	operation_iterator = stub.Execute(request)
104		-
	111	+ if operation_iterator.code() != grpc.StatusCode.OK:
	112	+ raise SandboxError("GRPC Execute failed; is the remote system connected?")
105	113	for operation in operation_iterator:
106	114	if operation.done:
107	115	break

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.