[Notes] [Git][BuildGrid/buildgrid][finn/81-precon-fail] 5 commits: Added ReferenceCache parser.

[finn <gitlab mg gitlab com>]

Title: GitLab

finn pushed to branch finn/81-precon-fail at BuildGrid / buildgrid

Commits:

• becd3571
  by finnball at 2018-09-14T07:55:57Z

  Added ReferenceCache parser.
  
  Added refrence storage to cas config.
  

• 886ef946
  by finnball at 2018-09-17T15:18:04Z

  Fails precondition if cannot fetch action from CAS.
  
  Also added new exception for precondition fail.
  

• 5869bb29
  by finnball at 2018-09-17T15:18:04Z

  Tests now account for precondition fail if no action in CAS
  

• 353173e6
  by finnball at 2018-09-17T15:18:04Z

  Added new upload-dummy command.
  
  Uploads dummy action to CAS.
  

• b38d1cad
  by finnball at 2018-09-17T15:18:04Z

  Included new upload-dummy command to CI and docs.
  

11 changed files:

• .gitlab-ci.yml
• buildgrid/_app/commands/cmd_cas.py
• buildgrid/_app/commands/cmd_execute.py
• buildgrid/_app/settings/cas.yml
• buildgrid/_app/settings/parser.py
• buildgrid/server/_exceptions.py
• buildgrid/server/execution/instance.py
• buildgrid/server/execution/service.py
• docs/source/using_dummy_build.rst
• tests/integration/execution_service.py
• tests/integration/operations_service.py

Changes:

.gitlab-ci.yml

---

...	...	@@ -33,6 +33,7 @@ before_script:
33	33	- ${BGD} server start buildgrid/_app/settings/default.yml &
34	34	- sleep 1 # Allow server to boot
35	35	- ${BGD} bot dummy &
	36	+ - ${BGD} cas upload-dummy
36	37	- ${BGD} execute request-dummy --wait-for-completion
37	38
38	39

buildgrid/_app/commands/cmd_cas.py

---

...	...	@@ -65,6 +65,23 @@ def cli(context, remote, instance_name, client_key, client_cert, server_cert):
65	65	context.logger.debug("Starting for remote {}".format(context.remote))
66	66
67	67
	68	+@cli.command('upload-dummy', short_help="Upload a dummy action. Should be used with `execute dummy-request`")
	69	+@pass_context
	70	+def upload_dummy(context):
	71	+ context.logger.info("Uploading dummy action...")
	72	+ action = remote_execution_pb2.Action(do_not_cache=True)
	73	+ action_digest = create_digest(action.SerializeToString())
	74	+
	75	+ request = remote_execution_pb2.BatchUpdateBlobsRequest(instance_name=context.instance_name)
	76	+ request.requests.add(digest=action_digest,
	77	+ data=action.SerializeToString())
	78	+
	79	+ stub = remote_execution_pb2_grpc.ContentAddressableStorageStub(context.channel)
	80	+ response = stub.BatchUpdateBlobs(request)
	81	+
	82	+ context.logger.info(response)
	83	+
	84	+
68	85	@cli.command('upload-files', short_help="Upload files to the CAS server.")
69	86	@click.argument('files', nargs=-1, type=click.File('rb'), required=True)
70	87	@pass_context

buildgrid/_app/commands/cmd_execute.py

---

...	...	@@ -76,9 +76,11 @@ def cli(context, remote, instance_name, client_key, client_cert, server_cert):
76	76	help="Stream updates until jobs are completed.")
77	77	@pass_context
78	78	def request_dummy(context, number, wait_for_completion):
79		- action_digest = remote_execution_pb2.Digest()
80	79
81	80	context.logger.info("Sending execution request...")
	81	+ action = remote_execution_pb2.Action(do_not_cache=True)
	82	+ action_digest = create_digest(action.SerializeToString())
	83	+
82	84	stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
83	85
84	86	request = remote_execution_pb2.ExecuteRequest(instance_name=context.instance_name,
...	...	@@ -90,9 +92,18 @@ def request_dummy(context, number, wait_for_completion):
90	92	responses.append(stub.Execute(request))
91	93
92	94	for response in responses:
	95	+
93	96	if wait_for_completion:
	97	+ result = None
94	98	for stream in response:
95		- context.logger.info(stream)
	99	+ result = stream
	100	+ context.logger.info(result)
	101	+
	102	+ if not result.done:
	103	+ click.echo("Result did not return True." +
	104	+ "Was the action uploaded to CAS?", err=True)
	105	+ sys.exit(-1)
	106	+
96	107	else:
97	108	context.logger.info(next(response))
98	109

buildgrid/_app/settings/cas.yml

---

...	...	@@ -7,7 +7,7 @@ server:
7	7	tls-client-certs: null
8	8
9	9	description: |
10		- Just a CAS.
	10	+ Just a CAS with some reference storage.
11	11
12	12	instances:
13	13	- name: main
...	...	@@ -24,3 +24,8 @@ instances:
24	24
25	25	- !bytestream
26	26	storage: *main-storage
	27	+
	28	+ - !reference-cache
	29	+ storage: *main-storage
	30	+ max_cached_refs: 256
	31	+ allow_updates: true

buildgrid/_app/settings/parser.py

---

...	...	@@ -23,6 +23,7 @@ import yaml
23	23
24	24	from buildgrid.server.controller import ExecutionController
25	25	from buildgrid.server.actioncache.storage import ActionCache
	26	+from buildgrid.server.referencestorage.storage import ReferenceCache
26	27	from buildgrid.server.cas.instance import ByteStreamInstance, ContentAddressableStorageInstance
27	28	from buildgrid.server.cas.storage.disk import DiskStorage
28	29	from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
...	...	@@ -126,10 +127,18 @@ class Action(YamlFactory):
126	127
127	128	yaml_tag = u'!action-cache'
128	129
129		- def __new__(cls, storage, max_cached_refs=0, allow_updates=True):
	130	+ def __new__(cls, storage, max_cached_refs, allow_updates=True):
130	131	return ActionCache(storage, max_cached_refs, allow_updates)
131	132
132	133
	134	+class Reference(YamlFactory):
	135	+
	136	+ yaml_tag = u'!reference-cache'
	137	+
	138	+ def __new__(cls, storage, max_cached_refs, allow_updates=True):
	139	+ return ReferenceCache(storage, max_cached_refs, allow_updates)
	140	+
	141	+
133	142	class CAS(YamlFactory):
134	143
135	144	yaml_tag = u'!cas'
...	...	@@ -160,9 +169,9 @@ def _parse_size(size):
160	169
161	170	def get_parser():
162	171
163		- yaml.SafeLoader.add_constructor(Execution.yaml_tag, Execution.from_yaml)
164	172	yaml.SafeLoader.add_constructor(Execution.yaml_tag, Execution.from_yaml)
165	173	yaml.SafeLoader.add_constructor(Action.yaml_tag, Action.from_yaml)
	174	+ yaml.SafeLoader.add_constructor(Reference.yaml_tag, Reference.from_yaml)
166	175	yaml.SafeLoader.add_constructor(Disk.yaml_tag, Disk.from_yaml)
167	176	yaml.SafeLoader.add_constructor(LRU.yaml_tag, LRU.from_yaml)
168	177	yaml.SafeLoader.add_constructor(S3.yaml_tag, S3.from_yaml)

buildgrid/server/_exceptions.py

---

...	...	@@ -46,3 +46,12 @@ class OutOfRangeError(BgdError):
46	46
47	47	def __init__(self, message, detail=None, reason=None):
48	48	super().__init__(message, detail=detail, domain=ErrorDomain.SERVER, reason=reason)
	49	+
	50	+
	51	+class FailedPreconditionError(BgdError):
	52	+ """ One or more errors occurred in setting up the action requested, such as a missing input
	53	+ or command or no worker being available. The client may be able to fix the errors and retry.
	54	+ """
	55	+
	56	+ def __init__(self, message, detail=None, reason=None):
	57	+ super().__init__(message, detail=detail, domain=ErrorDomain.SERVER, reason=reason)

buildgrid/server/execution/instance.py

---

...	...	@@ -24,12 +24,12 @@ import logging
24	24	from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import Action
25	25
26	26	from ..job import Job
27		-from .._exceptions import InvalidArgumentError
	27	+from .._exceptions import InvalidArgumentError, FailedPreconditionError
28	28
29	29
30	30	class ExecutionInstance:
31	31
32		- def __init__(self, scheduler, storage=None):
	32	+ def __init__(self, scheduler, storage):
33	33	self.logger = logging.getLogger(__name__)
34	34	self._storage = storage
35	35	self._scheduler = scheduler
...	...	@@ -40,13 +40,12 @@ class ExecutionInstance:
40	40	this action.
41	41	"""
42	42
43		- do_not_cache = False
44		- if self._storage is not None:
45		- action = self._storage.get_message(action_digest, Action)
46		- if action is not None:
47		- do_not_cache = action.do_not_cache
	43	+ action = self._storage.get_message(action_digest, Action)
48	44
49		- job = Job(action_digest, do_not_cache, message_queue)
	45	+ if not action:
	46	+ raise FailedPreconditionError("Could not get action from storage.")
	47	+
	48	+ job = Job(action_digest, action.do_not_cache, message_queue)
50	49	self.logger.info("Operation name: {}".format(job.name))
51	50
52	51	self._scheduler.append_job(job, skip_cache_lookup)

buildgrid/server/execution/service.py

---

...	...	@@ -30,7 +30,7 @@ from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_p
30	30
31	31	from buildgrid._protos.google.longrunning import operations_pb2
32	32
33		-from .._exceptions import InvalidArgumentError
	33	+from .._exceptions import InvalidArgumentError, FailedPreconditionError
34	34
35	35
36	36	class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
...	...	@@ -61,6 +61,12 @@ class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
61	61	context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
62	62	yield operations_pb2.Operation()
63	63
	64	+ except FailedPreconditionError as e:
	65	+ self.logger.error(e)
	66	+ context.set_details(str(e))
	67	+ context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
	68	+ yield operations_pb2.Operation()
	69	+
64	70	def WaitExecution(self, request, context):
65	71	try:
66	72	names = request.name.split("/")

docs/source/using_dummy_build.rst

---

...	...	@@ -9,7 +9,13 @@ In one terminal, start a server:
9	9
10	10	bgd server start buildgrid/_app/settings/default.yml
11	11
12		-In another terminal, send a request for work:
	12	+In another terminal, upload an action to CAS:
	13	+
	14	+.. code-block::sh
	15	+
	16	+ bgd cas upload-dummy
	17	+
	18	+Then send a request for work:
13	19
14	20	.. code-block:: sh
15	21

tests/integration/execution_service.py

---

...	...	@@ -28,6 +28,7 @@ import pytest
28	28	from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
29	29	from buildgrid._protos.google.longrunning import operations_pb2
30	30
	31	+from buildgrid.utils import create_digest
31	32	from buildgrid.server import job
32	33	from buildgrid.server.controller import ExecutionController
33	34	from buildgrid.server.cas.storage import lru_memory_cache
...	...	@@ -37,6 +38,8 @@ from buildgrid.server.execution.service import ExecutionService
37	38
38	39
39	40	server = mock.create_autospec(grpc.server)
	41	+action = remote_execution_pb2.Action(do_not_cache=True)
	42	+action_digest = create_digest(action.SerializeToString())
40	43
41	44
42	45	@pytest.fixture
...	...	@@ -47,12 +50,16 @@ def context():
47	50
48	51	@pytest.fixture(params=["action-cache", "no-action-cache"])
49	52	def controller(request):
	53	+ storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
	54	+ write_session = storage.begin_write(action_digest)
	55	+ storage.commit_write(action_digest, write_session)
	56	+
50	57	if request.param == "action-cache":
51		- storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
52	58	cache = ActionCache(storage, 50)
53	59	yield ExecutionController(cache, storage)
	60	+
54	61	else:
55		- yield ExecutionController()
	62	+ yield ExecutionController(None, storage)
56	63
57	64
58	65	# Instance to test
...	...	@@ -65,9 +72,6 @@ def instance(controller):
65	72
66	73	@pytest.mark.parametrize("skip_cache_lookup", [True, False])
67	74	def test_execute(skip_cache_lookup, instance, context):
68		- action_digest = remote_execution_pb2.Digest()
69		- action_digest.hash = 'zhora'
70		-
71	75	request = remote_execution_pb2.ExecuteRequest(instance_name='',
72	76	action_digest=action_digest,
73	77	skip_cache_lookup=skip_cache_lookup)
...	...	@@ -90,10 +94,16 @@ def test_wrong_execute_instance(instance, context):
90	94	context.set_code.assert_called_once_with(grpc.StatusCode.INVALID_ARGUMENT)
91	95
92	96
93		-def test_wait_execution(instance, controller, context):
94		- action_digest = remote_execution_pb2.Digest()
95		- action_digest.hash = 'zhora'
	97	+def test_no_action_digest_in_storage(instance, context):
	98	+ request = remote_execution_pb2.ExecuteRequest(instance_name='',
	99	+ skip_cache_lookup=True)
	100	+ response = instance.Execute(request, context)
	101	+
	102	+ next(response)
	103	+ context.set_code.assert_called_once_with(grpc.StatusCode.FAILED_PRECONDITION)
96	104
	105	+
	106	+def test_wait_execution(instance, controller, context):
97	107	j = job.Job(action_digest, None)
98	108	j._operation.done = True
99	109

tests/integration/operations_service.py

---

...	...	@@ -24,18 +24,21 @@ import grpc
24	24	from grpc._server import _Context
25	25	import pytest
26	26
27		-from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
28		-from buildgrid._protos.google.longrunning import operations_pb2
29		-
	27	+from buildgrid.utils import create_digest
30	28	from buildgrid.server.controller import ExecutionController
31		-from buildgrid.server._exceptions import InvalidArgumentError
32		-
	29	+from buildgrid.server.cas.storage import lru_memory_cache
33	30	from buildgrid.server.operations import service
34	31	from buildgrid.server.operations.service import OperationsService
	32	+from buildgrid.server._exceptions import InvalidArgumentError
	33	+
	34	+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
	35	+from buildgrid._protos.google.longrunning import operations_pb2
35	36
36	37
37	38	server = mock.create_autospec(grpc.server)
38	39	instance_name = "blade"
	40	+action = remote_execution_pb2.Action(do_not_cache=True)
	41	+action_digest = create_digest(action.SerializeToString())
39	42
40	43
41	44	# Can mock this
...	...	@@ -47,9 +50,6 @@ def context():
47	50	# Requests to make
48	51	@pytest.fixture
49	52	def execute_request():
50		- action_digest = remote_execution_pb2.Digest()
51		- action_digest.hash = 'zhora'
52		-
53	53	yield remote_execution_pb2.ExecuteRequest(instance_name='',
54	54	action_digest=action_digest,
55	55	skip_cache_lookup=True)
...	...	@@ -57,7 +57,11 @@ def execute_request():
57	57
58	58	@pytest.fixture
59	59	def controller():
60		- yield ExecutionController()
	60	+ storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
	61	+ write_session = storage.begin_write(action_digest)
	62	+ storage.commit_write(action_digest, write_session)
	63	+
	64	+ yield ExecutionController(None, storage)
61	65
62	66
63	67	# Instance to test

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.