[Notes] [Git][BuildGrid/buildgrid][mablanch/157-action-browser-url] 5 commits: controller.py: Fix server instances registration

[Martin Blanchard <gitlab mg gitlab com>]

Title: GitLab

Martin Blanchard pushed to branch mablanch/157-action-browser-url at BuildGrid / buildgrid

Commits:

• 85a99f60
  by Martin Blanchard at 2019-02-13T16:02:29Z

  controller.py: Fix server instances registration
  
  Delegate server registration to actual instance code instead of trying
  to get it right from the controller class.
  

• ca9597d8
  by Martin Blanchard at 2019-02-13T16:02:29Z

  scheduler.py: Hold parent's instance name
  

• 804fcff5
  by Martin Blanchard at 2019-02-13T16:02:29Z

  utils.py: Introduce a CAS browser URL helper
  

• 45176746
  by Martin Blanchard at 2019-02-13T16:02:29Z

  Introduce an action browser URL config. key
  

• a301d22f
  by Martin Blanchard at 2019-02-13T16:02:29Z

  Generate per-job action browser URLs
  

10 changed files:

• buildgrid/_app/settings/parser.py
• buildgrid/_app/settings/reference.yml
• buildgrid/server/controller.py
• buildgrid/server/execution/instance.py
• buildgrid/server/job.py
• buildgrid/server/scheduler.py
• buildgrid/settings.py
• buildgrid/utils.py
• tests/integration/execution_service.py
• tests/integration/operations_service.py

Changes:

buildgrid/_app/settings/parser.py

---

...	...	@@ -235,8 +235,8 @@ class Execution(YamlFactory):
235	235
236	236	yaml_tag = u'!execution'
237	237
238		- def __new__(cls, storage, action_cache=None):
239		- return ExecutionController(action_cache, storage)
	238	+ def __new__(cls, storage, action_cache=None, action_browser_url=None):
	239	+ return ExecutionController(storage, action_cache, action_browser_url)
240	240
241	241
242	242	class Action(YamlFactory):

buildgrid/_app/settings/reference.yml

---

...	...	@@ -75,7 +75,7 @@ instances:
75	75	# Whether or not writing to the cache is allowed.
76	76	allow-updates: true
77	77	##
78		- # Whether failed actions (non-zero exit code) are stored
	78	+ # Whether failed actions (non-zero exit code) are stored.
79	79	cache-failed-actions: true
80	80
81	81	- !execution
...	...	@@ -85,6 +85,9 @@ instances:
85	85	##
86	86	# Alias to an action-cache service.
87	87	action-cache: *main-action
	88	+ ##
	89	+ # Base URL for external build action (web) browser service.
	90	+ action-browser-url: http://localhost:8080
88	91
89	92	- !cas
90	93	##

buildgrid/server/controller.py

---

...	...	@@ -36,19 +36,19 @@ from .operations.instance import OperationsInstance
36	36
37	37	class ExecutionController:
38	38
39		- def __init__(self, action_cache=None, storage=None):
	39	+ def __init__(self, storage=None, action_cache=None, action_browser_url=None):
40	40	self.__logger = logging.getLogger(__name__)
41	41
42		- scheduler = Scheduler(action_cache)
	42	+ scheduler = Scheduler(action_cache, action_browser_url=action_browser_url)
43	43
44	44	self._execution_instance = ExecutionInstance(scheduler, storage)
45	45	self._bots_interface = BotsInterface(scheduler)
46	46	self._operations_instance = OperationsInstance(scheduler)
47	47
48	48	def register_instance_with_server(self, instance_name, server):
49		- server.add_execution_instance(self._execution_instance, instance_name)
50		- server.add_bots_interface(self._bots_interface, instance_name)
51		- server.add_operations_instance(self._operations_instance, instance_name)
	49	+ self._execution_instance.register_instance_with_server(instance_name, server)
	50	+ self._bots_interface.register_instance_with_server(instance_name, server)
	51	+ self._operations_instance.register_instance_with_server(instance_name, server)
52	52
53	53	def stream_operation_updates(self, message_queue, operation_name):
54	54	operation = message_queue.get()

buildgrid/server/execution/instance.py

---

...	...	@@ -52,6 +52,8 @@ class ExecutionInstance:
52	52	server.add_execution_instance(self, instance_name)
53	53
54	54	self._instance_name = instance_name
	55	+ if self._scheduler is not None:
	56	+ self._scheduler.set_instance_name(instance_name)
55	57
56	58	else:
57	59	raise AssertionError("Instance already registered")

buildgrid/server/job.py

---

...	...	@@ -37,10 +37,8 @@ class Job:
37	37	self._action = remote_execution_pb2.Action()
38	38	self._lease = None
39	39
40		- self.__execute_response = None
	40	+ self.__execute_response = remote_execution_pb2.ExecuteResponse()
41	41	self.__operation_metadata = remote_execution_pb2.ExecuteOperationMetadata()
42		- self.__operations_by_name = {} # Name to Operation 1:1 mapping
43		- self.__operations_by_peer = {} # Peer to Operation 1:1 mapping
44	42
45	43	self.__queued_timestamp = timestamp_pb2.Timestamp()
46	44	self.__queued_time_duration = duration_pb2.Duration()
...	...	@@ -48,6 +46,8 @@ class Job:
48	46	self.__worker_completed_timestamp = timestamp_pb2.Timestamp()
49	47
50	48	self.__operations_message_queues = {}
	49	+ self.__operations_by_name = {} # Name to Operation 1:1 mapping
	50	+ self.__operations_by_peer = {} # Peer to Operation 1:1 mapping
51	51	self.__operations_cancelled = set()
52	52	self.__lease_cancelled = False
53	53	self.__job_cancelled = False
...	...	@@ -146,6 +146,11 @@ class Job:
146	146	else:
147	147	return False
148	148
	149	+ def set_action_url(self, url):
	150	+ """Generates a CAS browser URL for the job's action."""
	151	+ if url.for_message('action', self.__operation_metadata.action_digest):
	152	+ self.__execute_response.message = url.generate()
	153	+
149	154	def set_cached_result(self, action_result):
150	155	"""Allows specifying an action result form the action cache for the job.
151	156
...	...	@@ -155,7 +160,6 @@ class Job:
155	160	Args:
156	161	action_result (ActionResult): The result from cache.
157	162	"""
158		- self.__execute_response = remote_execution_pb2.ExecuteResponse()
159	163	self.__execute_response.result.CopyFrom(action_result)
160	164	self.__execute_response.cached_result = True
161	165
...	...	@@ -445,7 +449,6 @@ class Job:
445	449	action_metadata.worker_start_timestamp.CopyFrom(self.__worker_start_timestamp)
446	450	action_metadata.worker_completed_timestamp.CopyFrom(self.__worker_completed_timestamp)
447	451
448		- self.__execute_response = remote_execution_pb2.ExecuteResponse()
449	452	self.__execute_response.result.CopyFrom(action_result)
450	453	self.__execute_response.cached_result = False
451	454	self.__execute_response.status.CopyFrom(status)

buildgrid/server/scheduler.py

---

...	...	@@ -26,15 +26,18 @@ import logging
26	26	from buildgrid._enums import LeaseState, OperationStage
27	27	from buildgrid._exceptions import NotFoundError
28	28	from buildgrid.server.job import Job
	29	+from buildgrid.utils import BrowserURL
29	30
30	31
31	32	class Scheduler:
32	33
33	34	MAX_N_TRIES = 5
34	35
35		- def __init__(self, action_cache=None, monitor=False):
	36	+ def __init__(self, action_cache=None, action_browser_url=False, monitor=False):
36	37	self.__logger = logging.getLogger(__name__)
37	38
	39	+ self._instance_name = None
	40	+
38	41	self.__build_metadata_queues = None
39	42
40	43	self.__operations_by_stage = None
...	...	@@ -43,6 +46,7 @@ class Scheduler:
43	46	self.__retries_count = 0
44	47
45	48	self._action_cache = action_cache
	49	+ self._action_browser_url = action_browser_url
46	50
47	51	self.__jobs_by_action = {} # Action to Job 1:1 mapping
48	52	self.__jobs_by_operation = {} # Operation to Job 1:1 mapping
...	...	@@ -57,6 +61,14 @@ class Scheduler:
57	61
58	62	# --- Public API ---
59	63
	64	+ @property
	65	+ def instance_name(self):
	66	+ return self._instance_name
	67	+
	68	+ def set_instance_name(self, instance_name):
	69	+ if not self._instance_name:
	70	+ self._instance_name = instance_name
	71	+
60	72	def list_current_jobs(self):
61	73	"""Returns a list of the :class:`Job` names currently managed."""
62	74	return self.__jobs_by_name.keys()
...	...	@@ -186,6 +198,10 @@ class Scheduler:
186	198	platform_requirements=platform_requirements,
187	199	priority=priority)
188	200
	201	+ if self._action_browser_url:
	202	+ job.set_action_url(
	203	+ BrowserURL(self._action_browser_url, self._instance_name))
	204	+
189	205	self.__logger.debug("Job created for action [%s]: [%s]",
190	206	action_digest.hash[:8], job.name)
191	207

buildgrid/settings.py

---

...	...	@@ -35,3 +35,11 @@ MAX_REQUEST_COUNT = 500
35	35	LOG_RECORD_FORMAT = '%(asctime)s:[%(name)36.36s][%(levelname)5.5s]: %(message)s'
36	36	# The different log record attributes are documented here:
37	37	# https://docs.python.org/3/library/logging.html#logrecord-attributes
	38	+
	39	+# URL scheme for the CAS content browser:
	40	+BROWSER_URL_FORMAT = '%(type)s/%(instance)s/%(hash)s/%(sizebytes)s/'
	41	+# The string markers that are substituted are:
	42	+# instance - CAS instance's name.
	43	+# type - Type of CAS object, eg. 'action_result', 'command'...
	44	+# hash - Object's digest hash.
	45	+# sizebytes - Object's digest size in bytes.

buildgrid/utils.py

---

...	...	@@ -13,14 +13,61 @@
13	13	# limitations under the License.
14	14
15	15
	16	+from urllib.parse import urljoin
16	17	from operator import attrgetter
17	18	import os
18	19	import socket
19	20
20		-from buildgrid.settings import HASH, HASH_LENGTH
	21	+from buildgrid.settings import HASH, HASH_LENGTH, BROWSER_URL_FORMAT
21	22	from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
22	23
23	24
	25	+class BrowserURL:
	26	+
	27	+ __url_markers = (
	28	+ '%(instance)s',
	29	+ '%(type)s',
	30	+ '%(hash)s',
	31	+ '%(sizebytes)s',
	32	+ )
	33	+
	34	+ def __init__(self, base_url, instance_name=None):
	35	+ """Begins browser URL helper initialization."""
	36	+ self.__base_url = base_url
	37	+ self.__initialized = False
	38	+ self.__url_spec = {
	39	+ '%(instance)s': instance_name or '',
	40	+ }
	41	+
	42	+ def for_message(self, message_type, message_digest):
	43	+ """Completes browser URL initialization for a protobuf message."""
	44	+ if self.__initialized:
	45	+ return False
	46	+
	47	+ self.__url_spec['%(type)s'] = message_type
	48	+ self.__url_spec['%(hash)s'] = message_digest.hash
	49	+ self.__url_spec['%(sizebytes)s'] = str(message_digest.size_bytes)
	50	+
	51	+ self.__initialized = True
	52	+ return True
	53	+
	54	+ def generate(self):
	55	+ """Generates a browser URL string."""
	56	+ if not self.__base_url or not self.__initialized:
	57	+ return None
	58	+
	59	+ url_tail = BROWSER_URL_FORMAT
	60	+
	61	+ for url_marker in self.__url_markers:
	62	+ if url_marker not in self.__url_spec:
	63	+ return None
	64	+ if url_marker not in url_tail:
	65	+ continue
	66	+ url_tail = url_tail.replace(url_marker, self.__url_spec[url_marker])
	67	+
	68	+ return urljoin(self.__base_url, url_tail)
	69	+
	70	+
24	71	def get_hostname():
25	72	"""Returns the hostname of the machine executing that function.
26	73

tests/integration/execution_service.py

---

...	...	@@ -66,10 +66,10 @@ def controller(request):
66	66
67	67	if request.param == "action-cache":
68	68	cache = ActionCache(storage, 50)
69		- yield ExecutionController(cache, storage)
	69	+ yield ExecutionController(storage=storage, action_cache=cache)
70	70
71	71	else:
72		- yield ExecutionController(None, storage)
	72	+ yield ExecutionController(storage=storage)
73	73
74	74
75	75	# Instance to test

tests/integration/operations_service.py

---

...	...	@@ -71,7 +71,7 @@ def controller():
71	71	write_session.write(action.SerializeToString())
72	72	storage.commit_write(action_digest, write_session)
73	73
74		- yield ExecutionController(None, storage)
	74	+ yield ExecutionController(storage=storage)
75	75
76	76
77	77	# Instance to test

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.