[Notes] [Git][BuildGrid/buildgrid][mablanch/139-emit-build-metrics] 3 commits: job.py: Expose its ActionResult origin

[Martin Blanchard <gitlab mg gitlab com>]

Title: GitLab

Martin Blanchard pushed to branch mablanch/139-emit-build-metrics at BuildGrid / buildgrid

Commits:

• 1dde5014
  by Martin Blanchard at 2018-12-04T14:34:08Z

  job.py: Expose its ActionResult origin
  
  https://gitlab.com/BuildGrid/buildgrid/issues/139
  

• d25e33fd
  by Martin Blanchard at 2018-12-04T14:34:19Z

  scheduler.py: Allow registering for build metadata updates
  
  https://gitlab.com/BuildGrid/buildgrid/issues/139
  

• 8e27bbc8
  by Martin Blanchard at 2018-12-04T14:34:22Z

  server/instance.py: Register tasks for build metrics
  
  https://gitlab.com/BuildGrid/buildgrid/issues/139
  

3 changed files:

• buildgrid/server/instance.py
• buildgrid/server/job.py
• buildgrid/server/scheduler.py

Changes:

buildgrid/server/instance.py

---

...	...	@@ -71,7 +71,10 @@ class BuildGridServer:
71	71	self.__logging_formatter = logging.Formatter(fmt=LOG_RECORD_FORMAT)
72	72	self.__print_log_records = True
73	73
	74	+ self.__build_metadata_queues = None
	75	+
74	76	self.__state_monitoring_task = None
	77	+ self.__build_monitoring_tasks = None
75	78	self.__logging_task = None
76	79
77	80	# We always want a capabilities service
...	...	@@ -95,6 +98,8 @@ class BuildGridServer:
95	98	self.__main_loop, endpoint_type=MonitoringOutputType.STDOUT,
96	99	serialisation_format=MonitoringOutputFormat.JSON)
97	100
	101	+ self.__build_monitoring_tasks = []
	102	+
98	103	# Setup the main logging handler:
99	104	root_logger = logging.getLogger()
100	105
...	...	@@ -119,6 +124,18 @@ class BuildGridServer:
119	124	self._state_monitoring_worker(period=MONITORING_PERIOD),
120	125	loop=self.__main_loop)
121	126
	127	+ self.__build_monitoring_tasks.clear()
	128	+ for instance_name, scheduler in self._schedulers.items():
	129	+ if not scheduler.is_instrumented:
	130	+ continue
	131	+
	132	+ message_queue = janus.Queue(loop=self.__main_loop)
	133	+ scheduler.register_build_metadata_watcher(message_queue.sync_q)
	134	+
	135	+ self.__build_monitoring_tasks.append(asyncio.ensure_future(
	136	+ self._build_monitoring_worker(instance_name, message_queue),
	137	+ loop=self.__main_loop))
	138	+
122	139	self.__logging_task = asyncio.ensure_future(
123	140	self._logging_worker(), loop=self.__main_loop)
124	141
...	...	@@ -132,6 +149,10 @@ class BuildGridServer:
132	149	if self.__state_monitoring_task is not None:
133	150	self.__state_monitoring_task.cancel()
134	151
	152	+ for build_monitoring_task in self.__build_monitoring_tasks:
	153	+ build_monitoring_task.cancel()
	154	+ self.__build_monitoring_tasks.clear()
	155	+
135	156	self.__monitoring_bus.stop()
136	157
137	158	if self.__logging_task is not None:
...	...	@@ -352,6 +373,58 @@ class BuildGridServer:
352	373
353	374	return log_record
354	375
	376	+ async def _build_monitoring_worker(self, instance_name, message_queue):
	377	+ """Publishes builds metadata to the monitoring bus."""
	378	+ async def __build_monitoring_worker():
	379	+ metadata = await message_queue.async_q.get()
	380	+
	381	+ # Emit build inputs fetching time record:
	382	+ fetch_start = metadata.input_fetch_start_timestamp.ToDatetime()
	383	+ fetch_completed = metadata.input_fetch_completed_timestamp.ToDatetime()
	384	+ input_fetch_time = fetch_completed - fetch_start
	385	+ timer_record = self._forge_timer_metric_record(
	386	+ MetricRecordDomain.BUILD, 'inputs-fetching-time', input_fetch_time,
	387	+ metadata={'instance-name': instance_name or 'void'})
	388	+
	389	+ await self.__monitoring_bus.send_record(timer_record)
	390	+
	391	+ # Emit build execution time record:
	392	+ execution_start = metadata.execution_start_timestamp.ToDatetime()
	393	+ execution_completed = metadata.execution_completed_timestamp.ToDatetime()
	394	+ execution_time = execution_completed - execution_start
	395	+ timer_record = self._forge_timer_metric_record(
	396	+ MetricRecordDomain.BUILD, 'execution-time', execution_time,
	397	+ metadata={'instance-name': instance_name or 'void'})
	398	+
	399	+ await self.__monitoring_bus.send_record(timer_record)
	400	+
	401	+ # Emit build outputs uploading time record:
	402	+ upload_start = metadata.output_upload_start_timestamp.ToDatetime()
	403	+ upload_completed = metadata.output_upload_completed_timestamp.ToDatetime()
	404	+ output_upload_time = upload_completed - upload_start
	405	+ timer_record = self._forge_timer_metric_record(
	406	+ MetricRecordDomain.BUILD, 'outputs-uploading-time', output_upload_time,
	407	+ metadata={'instance-name': instance_name or 'void'})
	408	+
	409	+ await self.__monitoring_bus.send_record(timer_record)
	410	+
	411	+ # Emit total build handling time record:
	412	+ queued = metadata.queued_timestamp.ToDatetime()
	413	+ worker_completed = metadata.worker_completed_timestamp.ToDatetime()
	414	+ total_handling_time = worker_completed - queued
	415	+ timer_record = self._forge_timer_metric_record(
	416	+ MetricRecordDomain.BUILD, 'total-handling-time', total_handling_time,
	417	+ metadata={'instance-name': instance_name or 'void'})
	418	+
	419	+ await self.__monitoring_bus.send_record(timer_record)
	420	+
	421	+ try:
	422	+ while True:
	423	+ await __build_monitoring_worker()
	424	+
	425	+ except asyncio.CancelledError:
	426	+ pass
	427	+
355	428	async def _state_monitoring_worker(self, period=1.0):
356	429	"""Periodically publishes state metrics to the monitoring bus."""
357	430	async def __state_monitoring_worker():

buildgrid/server/job.py

---

...	...	@@ -83,6 +83,13 @@ class Job:
83	83	else:
84	84	return None
85	85
	86	+ @property
	87	+ def holds_cached_action_result(self):
	88	+ if self.__execute_response is not None:
	89	+ return self.__execute_response.cached_result
	90	+ else:
	91	+ return False
	92	+
86	93	@property
87	94	def operation(self):
88	95	return self._operation

buildgrid/server/scheduler.py

---

...	...	@@ -34,6 +34,8 @@ class Scheduler:
34	34	def __init__(self, action_cache=None, monitor=False):
35	35	self.__logger = logging.getLogger(__name__)
36	36
	37	+ self.__build_metadata_queues = None
	38	+
37	39	self.__operations_by_stage = None
38	40	self.__leases_by_state = None
39	41	self.__queue_time_average = None
...	...	@@ -46,6 +48,8 @@ class Scheduler:
46	48	self._is_instrumented = monitor
47	49
48	50	if self._is_instrumented:
	51	+ self.__build_metadata_queues = []
	52	+
49	53	self.__operations_by_stage = {}
50	54	self.__leases_by_state = {}
51	55	self.__queue_time_average = 0, timedelta()
...	...	@@ -228,6 +232,10 @@ class Scheduler:
228	232	def is_instrumented(self):
229	233	return self._is_instrumented
230	234
	235	+ def register_build_metadata_watcher(self, message_queue):
	236	+ if self.__build_metadata_queues is not None:
	237	+ self.__build_metadata_queues.append(message_queue)
	238	+
231	239	def query_n_jobs(self):
232	240	return len(self.jobs)
233	241
...	...	@@ -319,3 +327,7 @@ class Scheduler:
319	327	average_time = average_time + ((queue_time - average_time) / average_order)
320	328
321	329	self.__queue_time_average = average_order, average_time
	330	+
	331	+ if not job.holds_cached_action_result:
	332	+ for message_queue in self.__build_metadata_queues:
	333	+ message_queue.put(job.action_result.execution_metadata)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.