[Notes] [Git][BuildStream/buildstream][valentindavid/remote_execution_configuration] 2 commits: Add support for https channel to remote execution and actions servers

[Valentin David <gitlab mg gitlab com>]

Title: GitLab

Valentin David pushed to branch valentindavid/remote_execution_configuration at BuildStream / buildstream

Commits:

• be32c71e
  by Valentin David at 2018-12-20T13:41:50Z

  Add support for https channel to remote execution and actions servers
  
  Fixes #780.
  

• f3542548
  by Valentin David at 2018-12-20T13:42:21Z

  Allow selecting user or project configuration for remote execution
  

7 changed files:

• buildstream/_context.py
• buildstream/_frontend/app.py
• buildstream/_frontend/cli.py
• buildstream/_project.py
• buildstream/sandbox/_sandboxremote.py
• doc/source/format_project.rst
• tests/completions/completions.py

Changes:

buildstream/_context.py

---

...	...	@@ -121,6 +121,9 @@ class Context():
121	121	# Whether or not to attempt to pull build trees globally
122	122	self.pull_buildtrees = None
123	123
	124	+ # Which configured remote execution server to use
	125	+ self.remote_execution = 'any'
	126	+
124	127	# Boolean, whether to offer to create a project for the user, if we are
125	128	# invoked outside of a directory where we can resolve the project.
126	129	self.prompt_auto_init = None

buildstream/_frontend/app.py

---

...	...	@@ -183,7 +183,8 @@ class App():
183	183	'builders': 'sched_builders',
184	184	'pushers': 'sched_pushers',
185	185	'network_retries': 'sched_network_retries',
186		- 'pull_buildtrees': 'pull_buildtrees'
	186	+ 'pull_buildtrees': 'pull_buildtrees',
	187	+ 'remote_execution': 'remote_execution'
187	188	}
188	189	for cli_option, context_attr in override_map.items():
189	190	option_value = self._main_options.get(cli_option)

buildstream/_frontend/cli.py

---

...	...	@@ -239,6 +239,9 @@ def print_version(ctx, param, value):
239	239	help="The mirror to fetch from first, before attempting other mirrors")
240	240	@click.option('--pull-buildtrees', is_flag=True, default=None,
241	241	help="Include an element's build tree when pulling remote element artifacts")
	242	+@click.option('--remote-execution', default='any',
	243	+ type=click.Choice(['any', 'user', 'project', 'none']),
	244	+ help='Select which remote execution server to use')
242	245	@click.pass_context
243	246	def cli(context, **kwargs):
244	247	"""Build and manipulate BuildStream projects

buildstream/_project.py

---

...	...	@@ -511,12 +511,23 @@ class Project():
511	511	override_specs = SandboxRemote.specs_from_config_node(
512	512	self._context.get_overrides(self.name), self.directory)
513	513
514		- if override_specs is not None:
515		- self.remote_execution_specs = override_specs
516		- elif project_specs is not None:
	514	+ if self._context.remote_execution == 'any':
	515	+ if override_specs is not None:
	516	+ self.remote_execution_specs = override_specs
	517	+ elif project_specs is not None:
	518	+ self.remote_execution_specs = project_specs
	519	+ else:
	520	+ self.remote_execution_specs = self._context.remote_execution_specs
	521	+ elif self._context.remote_execution == 'user':
	522	+ if override_specs is not None:
	523	+ self.remote_execution_specs = override_specs
	524	+ else:
	525	+ self.remote_execution_specs = self._context.remote_execution_specs
	526	+ elif self._context.remote_execution == 'project':
517	527	self.remote_execution_specs = project_specs
518	528	else:
519		- self.remote_execution_specs = self._context.remote_execution_specs
	529	+ assert self._context.remote_execution == 'none'
	530	+ self.remote_execution_specs = None
520	531
521	532	# Load sandbox environment variables
522	533	self.base_environment = _yaml.node_get(config, Mapping, 'environment')

buildstream/sandbox/_sandboxremote.py

---

...	...	@@ -68,10 +68,33 @@ class SandboxRemote(Sandbox):
68	68	self.storage_url = config.storage_service['url']
69	69	self.exec_url = config.exec_service['url']
70	70
	71	+ exec_certs = {}
	72	+ for key in ['client-cert', 'client-key', 'server-cert']:
	73	+ if key in config.exec_service:
	74	+ with open(resolve_path(config.exec_service[key]), 'rb') as f:
	75	+ exec_certs[key] = f.read()
	76	+
	77	+ self.exec_credentials = grpc.ssl_channel_credentials(
	78	+ root_certificates=exec_certs.get('server-cert'),
	79	+ private_key=exec_certs.get('client-key'),
	80	+ certificate_chain=exec_certs.get('client-cert'))
	81	+
	82	+ action_certs = {}
	83	+ for key in ['client-cert', 'client-key', 'server-cert']:
	84	+ if key in config.action_service:
	85	+ with open(resolve_path(config.exec_service[key]), 'rb') as f:
	86	+ action_certs[key] = f.read()
	87	+
71	88	if config.action_service:
72	89	self.action_url = config.action_service['url']
	90	+ self.action_credentials = grpc.ssl_channel_credentials(
	91	+ root_certificates=action_certs.get('server-cert'),
	92	+ private_key=action_certs.get('client-key'),
	93	+ certificate_chain=action_certs.get('client-cert'))
	94	+ )
73	95	else:
74	96	self.action_url = None
	97	+ self.action_credentials = None
75	98
76	99	self.server_instance = config.exec_service.get('instance', None)
77	100	self.storage_instance = config.storage_service.get('instance', None)
...	...	@@ -117,7 +140,7 @@ class SandboxRemote(Sandbox):
117	140	remote_exec_storage_config = require_node(remote_config, 'storage-service')
118	141	remote_exec_action_config = remote_config.get('action-cache-service', {})
119	142
120		- _yaml.node_validate(remote_exec_service_config, ['url', 'instance'])
	143	+ _yaml.node_validate(remote_exec_service_config, ['url', 'instance'] + tls_keys)
121	144	_yaml.node_validate(remote_exec_storage_config, ['url', 'instance'] + tls_keys)
122	145	if remote_exec_action_config:
123	146	_yaml.node_validate(remote_exec_action_config, ['url'])
...	...	@@ -304,6 +327,8 @@ class SandboxRemote(Sandbox):
304	327	"for example: http://buildservice:50051.")
305	328	if url.scheme == 'http':
306	329	channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
	330	+ elif url.scheme == 'https':
	331	+ channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.exec_credentials)
307	332	else:
308	333	raise SandboxError("Remote execution currently only supports the 'http' protocol "
309	334	"and '{}' was supplied.".format(url.scheme))
...	...	@@ -361,11 +386,11 @@ class SandboxRemote(Sandbox):
361	386	if not url.port:
362	387	raise SandboxError("You must supply a protocol and port number in the action-cache-service url, "
363	388	"for example: http://buildservice:50051.")
364		- if not url.scheme == "http":
365		- raise SandboxError("Currently only support http for the action cache"
366		- "and {} was supplied".format(url.scheme))
	389	+ if url.scheme == 'http':
	390	+ channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
	391	+ elif url.scheme == 'https':
	392	+ channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.action_credentials)
367	393
368		- channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
369	394	request = remote_execution_pb2.GetActionResultRequest(action_digest=action_digest)
370	395	stub = remote_execution_pb2_grpc.ActionCacheStub(channel)
371	396	try:

doc/source/format_project.rst

---

...	...	@@ -244,9 +244,6 @@ using the `remote-execution` option:
244	244	action-cache-service:
245	245	url: http://bar.action.com:50052
246	246
247		-The execution-service part of remote execution does not support encrypted
248		-connections yet, so the protocol must always be http.
249		-
250	247	storage-service specifies a remote CAS store and the parameters are the
251	248	same as those used to specify an :ref:`artifact server <artifacts>`.
252	249

tests/completions/completions.py

---

...	...	@@ -42,6 +42,7 @@ MAIN_OPTIONS = [
42	42	"--option ",
43	43	"--on-error ",
44	44	"--pull-buildtrees ",
	45	+ "--remote-execution ",
45	46	"--pushers ",
46	47	"--strict ",
47	48	"--verbose ",

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.