[Notes] [Git][BuildStream/buildstream][jmac/remote_execution_split] 3 commits: Documentation: Update to show new config options

[Jim MacArthur <gitlab mg gitlab com>]

Title: GitLab

Jim MacArthur pushed to branch jmac/remote_execution_split at BuildStream / buildstream

Commits:

• ce98e3f6
  by Jim MacArthur at 2018-11-26T11:53:34Z

  Documentation: Update to show new config options
  

• 392c2803
  by Jim MacArthur at 2018-11-26T11:53:34Z

  tests/sandboxes/remote-exec-config.py: New test.
  
  Tests that we get a sensible error message when we specify incomplete
  or wrong information in the configuration for remote execution.
  

• 26c3e97f
  by Jim MacArthur at 2018-11-26T11:53:34Z

  artifactcache.py: Remove unused verify_digest_pushed
  

6 changed files:

• buildstream/_artifactcache/artifactcache.py
• doc/source/format_project.rst
• + tests/sandboxes/remote-exec-config.py
• + tests/sandboxes/remote-exec-config/missing-certs/certificates/client.crt
• + tests/sandboxes/remote-exec-config/missing-certs/certificates/client.key
• + tests/sandboxes/remote-exec-config/missing-certs/element.bst

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -751,34 +751,6 @@ class ArtifactCache():
751	751
752	752	return message_digest
753	753
754		- # verify_digest_pushed():
755		- #
756		- # Check whether the object is already on the server in which case
757		- # there is no need to upload it.
758		- #
759		- # Args:
760		- # project (Project): The current project
761		- # digest (Digest): The object digest.
762		- #
763		- def verify_digest_pushed(self, project, digest):
764		-
765		- if self._has_push_remotes:
766		- push_remotes = [r for r in self._remotes[project] if r.spec.push]
767		- else:
768		- push_remotes = []
769		-
770		- if not push_remotes:
771		- raise ArtifactError("verify_digest_pushed was called, but no remote artifact " +
772		- "servers are configured as push remotes.")
773		-
774		- pushed = False
775		-
776		- for remote in push_remotes:
777		- if self.cas.verify_digest_on_remote(remote, digest):
778		- pushed = True
779		-
780		- return pushed
781		-
782	754	# link_key():
783	755	#
784	756	# Add a key for an existing artifact.

doc/source/format_project.rst

---

...	...	@@ -231,10 +231,24 @@ using the `remote-execution` option:
231	231	remote-execution:
232	232
233	233	# A url defining a remote execution server
234		- url: http://buildserver.example.com:50051
	234	+ execution-service:
	235	+ url: http://buildserver.example.com:50051
	236	+ storage-service:
	237	+ - url: https://foo.com/artifacts:11002
	238	+ server-cert: server.crt
	239	+ client-cert: client.crt
	240	+ client-key: client.key
	241	+
	242	+The execution-service part of remote execution does not support encrypted
	243	+connections yet, so the protocol must always be http.
	244	+
	245	+storage-service specifies a remote CAS store and the parameters are the
	246	+same as those used to specify an :ref:`artifact server <artifacts>`.
235	247
236		-The url should contain a hostname and port separated by ':'. Only plain HTTP is
237		-currently suported (no HTTPS).
	248	+The storage service may be the same endpoint used for artifact
	249	+caching. Remote execution cannot work without push access to the
	250	+storage endpoint, so you must specify a client certificate and key,
	251	+and a server certificate.
238	252
239	253	The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis.
240	254

tests/sandboxes/remote-exec-config.py

---

	1	+import pytest
	2	+
	3	+import itertools
	4	+import os
	5	+
	6	+from buildstream import _yaml
	7	+from buildstream._exceptions import ErrorDomain, LoadErrorReason
	8	+
	9	+from tests.testutils.runcli import cli
	10	+
	11	+DATA_DIR = os.path.join(
	12	+ os.path.dirname(os.path.realpath(__file__)),
	13	+ "remote-exec-config"
	14	+)
	15	+
	16	+# Tests that we get a useful error message when supplying invalid
	17	+# remote execution configurations.
	18	+
	19	+
	20	+# Assert that if both 'url' (the old style) and 'execution-service' (the new style)
	21	+# are used at once, a LoadError results.
	22	+@pytest.mark.datafiles(DATA_DIR)
	23	+def test_old_and_new_configs(cli, datafiles):
	24	+ project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
	25	+
	26	+ project_conf = {
	27	+ 'name': 'test',
	28	+
	29	+ 'remote-execution': {
	30	+ 'url': 'https://cache.example.com:12345',
	31	+ 'execution-service': {
	32	+ 'url': 'http://localhost:8088'
	33	+ },
	34	+ 'storage-service': {
	35	+ 'url': 'http://charactron:11001',
	36	+ }
	37	+ }
	38	+ }
	39	+ project_conf_file = os.path.join(project, 'project.conf')
	40	+ _yaml.dump(project_conf, project_conf_file)
	41	+
	42	+ # Use `pull` here to ensure we try to initialize the remotes, triggering the error
	43	+ #
	44	+ # This does not happen for a simple `bst show`.
	45	+ result = cli.run(project=project, args=['pull', 'element.bst'])
	46	+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one")
	47	+
	48	+
	49	+# Assert that if either the client key or client cert is specified
	50	+# without specifying its counterpart, we get a comprehensive LoadError
	51	+# instead of an unhandled exception.
	52	+@pytest.mark.datafiles(DATA_DIR)
	53	+@pytest.mark.parametrize('config_key, config_value', [
	54	+ ('client-cert', 'client.crt'),
	55	+ ('client-key', 'client.key')
	56	+])
	57	+def test_missing_certs(cli, datafiles, config_key, config_value):
	58	+ project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
	59	+
	60	+ project_conf = {
	61	+ 'name': 'test',
	62	+
	63	+ 'remote-execution': {
	64	+ 'execution-service': {
	65	+ 'url': 'http://localhost:8088'
	66	+ },
	67	+ 'storage-service': {
	68	+ 'url': 'http://charactron:11001',
	69	+ config_key: config_value,
	70	+ }
	71	+ }
	72	+ }
	73	+ project_conf_file = os.path.join(project, 'project.conf')
	74	+ _yaml.dump(project_conf, project_conf_file)
	75	+
	76	+ # Use `pull` here to ensure we try to initialize the remotes, triggering the error
	77	+ #
	78	+ # This does not happen for a simple `bst show`.
	79	+ result = cli.run(project=project, args=['show', 'element.bst'])
	80	+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "Your config is missing")
	81	+
	82	+
	83	+# Assert that if incomplete information is supplied we get a sensible error message.
	84	+@pytest.mark.datafiles(DATA_DIR)
	85	+def test_empty_config(cli, datafiles):
	86	+ project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
	87	+
	88	+ project_conf = {
	89	+ 'name': 'test',
	90	+
	91	+ 'remote-execution': {
	92	+ }
	93	+ }
	94	+ project_conf_file = os.path.join(project, 'project.conf')
	95	+ _yaml.dump(project_conf, project_conf_file)
	96	+
	97	+ # Use `pull` here to ensure we try to initialize the remotes, triggering the error
	98	+ #
	99	+ # This does not happen for a simple `bst show`.
	100	+ result = cli.run(project=project, args=['pull', 'element.bst'])
	101	+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one")

tests/sandboxes/remote-exec-config/missing-certs/certificates/client.crt

---

tests/sandboxes/remote-exec-config/missing-certs/certificates/client.key

---

tests/sandboxes/remote-exec-config/missing-certs/element.bst

---

1

+kind: autotools

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.