[Notes] [Git][BuildGrid/buildgrid][finn/one-lease-per-worker] 5 commits: client/cas.py: Fix empty instance-name issue

[finn <gitlab mg gitlab com>]

Title: GitLab

finn pushed to branch finn/one-lease-per-worker at BuildGrid / buildgrid

Commits:

• 65f4b337
  by Martin Blanchard at 2018-10-03T14:08:14Z

  client/cas.py: Fix empty instance-name issue
  
  If the Uploader or Downloader instance-name parameter is set to the
  empty string instead of None, the generated ByteStream resource-name
  would contain a leading slash, which is an invalid REAPI request. If
  the instance-name is not set, the resource-name should be of the form:
  
      blobs/{hash}/{size}
  

• a88e0715
  by Finn at 2018-10-03T14:35:20Z

  Improved cas logging for debugging.
  

• 4fa35bc9
  by Finn at 2018-10-03T14:35:20Z

  NotFoundError now logged as debug.
  

• bb2cd92b
  by Finn at 2018-10-03T15:06:01Z

  One lease per bot session.
  
  Will be like this until we can fix #47.
  

• 74bca84f
  by Finn at 2018-10-03T15:06:01Z

  Updated unittest for one lease per bot at a time.
  

7 changed files:

• buildgrid/client/cas.py
• buildgrid/server/actioncache/service.py
• buildgrid/server/bots/instance.py
• buildgrid/server/cas/service.py
• buildgrid/server/referencestorage/service.py
• buildgrid/server/scheduler.py
• tests/integration/bots_service.py

Changes:

buildgrid/client/cas.py

---

...	...	@@ -241,7 +241,7 @@ class Downloader:
241	241	"""Fetches a blob using ByteStream.Read()"""
242	242	read_blob = bytearray()
243	243
244		- if self.instance_name is not None:
	244	+ if self.instance_name:
245	245	resource_name = '/'.join([self.instance_name, 'blobs',
246	246	digest.hash, str(digest.size_bytes)])
247	247	else:
...	...	@@ -313,7 +313,7 @@ class Downloader:
313	313
314	314	def _fetch_file(self, digest, file_path):
315	315	"""Fetches a file using ByteStream.Read()"""
316		- if self.instance_name is not None:
	316	+ if self.instance_name:
317	317	resource_name = '/'.join([self.instance_name, 'blobs',
318	318	digest.hash, str(digest.size_bytes)])
319	319	else:
...	...	@@ -699,7 +699,7 @@ class Uploader:
699	699	else:
700	700	blob_digest.hash = HASH(blob).hexdigest()
701	701	blob_digest.size_bytes = len(blob)
702		- if self.instance_name is not None:
	702	+ if self.instance_name:
703	703	resource_name = '/'.join([self.instance_name, 'uploads', self.u_uid, 'blobs',
704	704	blob_digest.hash, str(blob_digest.size_bytes)])
705	705	else:

buildgrid/server/actioncache/service.py

---

...	...	@@ -52,7 +52,7 @@ class ActionCacheService(remote_execution_pb2_grpc.ActionCacheServicer):
52	52	context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
53	53
54	54	except NotFoundError as e:
55		- self.logger.info(e)
	55	+ self.logger.debug(e)
56	56	context.set_code(grpc.StatusCode.NOT_FOUND)
57	57
58	58	return remote_execution_pb2.ActionResult()

buildgrid/server/bots/instance.py

---

...	...	@@ -66,7 +66,9 @@ class BotsInterface:
66	66	self._bot_sessions[name] = bot_session
67	67	self.logger.info("Created bot session name=[{}] with bot_id=[{}]".format(name, bot_id))
68	68
69		- for lease in self._scheduler.create_leases():
	69	+ # For now, one lease at a time.
	70	+ lease = self._scheduler.create_lease()
	71	+ if lease:
70	72	bot_session.leases.extend([lease])
71	73
72	74	return bot_session
...	...	@@ -83,8 +85,11 @@ class BotsInterface:
83	85	del bot_session.leases[:]
84	86	bot_session.leases.extend(leases)
85	87
86		- for lease in self._scheduler.create_leases():
87		- bot_session.leases.extend([lease])
	88	+ # For now, one lease at a time
	89	+ if not bot_session.leases:
	90	+ lease = self._scheduler.create_lease()
	91	+ if lease:
	92	+ bot_session.leases.extend([lease])
88	93
89	94	self._bot_sessions[name] = bot_session
90	95	return bot_session

buildgrid/server/cas/service.py

---

...	...	@@ -46,8 +46,11 @@ class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressa
46	46
47	47	def FindMissingBlobs(self, request, context):
48	48	try:
	49	+ self.logger.debug("FindMissingBlobs request: [{}]".format(request))
49	50	instance = self._get_instance(request.instance_name)
50		- return instance.find_missing_blobs(request.blob_digests)
	51	+ response = instance.find_missing_blobs(request.blob_digests)
	52	+ self.logger.debug("FindMissingBlobs response: [{}]".format(response))
	53	+ return response
51	54
52	55	except InvalidArgumentError as e:
53	56	self.logger.error(e)
...	...	@@ -58,8 +61,11 @@ class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressa
58	61
59	62	def BatchUpdateBlobs(self, request, context):
60	63	try:
	64	+ self.logger.debug("BatchUpdateBlobs request: [{}]".format(request))
61	65	instance = self._get_instance(request.instance_name)
62		- return instance.batch_update_blobs(request.requests)
	66	+ response = instance.batch_update_blobs(request.requests)
	67	+ self.logger.debug("FindMissingBlobs response: [{}]".format(response))
	68	+ return response
63	69
64	70	except InvalidArgumentError as e:
65	71	self.logger.error(e)
...	...	@@ -102,6 +108,7 @@ class ByteStreamService(bytestream_pb2_grpc.ByteStreamServicer):
102	108
103	109	def Read(self, request, context):
104	110	try:
	111	+ self.logger.debug("Read request: [{}]".format(request))
105	112	path = request.resource_name.split("/")
106	113	instance_name = path[0]
107	114
...	...	@@ -141,10 +148,13 @@ class ByteStreamService(bytestream_pb2_grpc.ByteStreamServicer):
141	148	context.set_code(grpc.StatusCode.OUT_OF_RANGE)
142	149	yield bytestream_pb2.ReadResponse()
143	150
	151	+ self.logger.debug("Read finished.")
	152	+
144	153	def Write(self, requests, context):
145	154	try:
146	155	requests, request_probe = tee(requests, 2)
147	156	first_request = next(request_probe)
	157	+ self.logger.debug("First write request: [{}]".format(first_request))
148	158
149	159	path = first_request.resource_name.split("/")
150	160
...	...	@@ -164,7 +174,9 @@ class ByteStreamService(bytestream_pb2_grpc.ByteStreamServicer):
164	174	raise InvalidArgumentError("Invalid resource name: [{}]".format(first_request.resource_name))
165	175
166	176	instance = self._get_instance(instance_name)
167		- return instance.write(requests)
	177	+ response = instance.write(requests)
	178	+ self.logger.debug("Write response: [{}]".format(response))
	179	+ return response
168	180
169	181	except NotImplementedError as e:
170	182	self.logger.error(e)

buildgrid/server/referencestorage/service.py

---

...	...	@@ -47,7 +47,8 @@ class ReferenceStorageService(buildstream_pb2_grpc.ReferenceStorageServicer):
47	47	context.set_details(str(e))
48	48	context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
49	49
50		- except NotFoundError:
	50	+ except NotFoundError as e:
	51	+ self.logger.debug(e)
51	52	context.set_code(grpc.StatusCode.NOT_FOUND)
52	53
53	54	return buildstream_pb2.GetReferenceResponse()

buildgrid/server/scheduler.py

---

...	...	@@ -108,10 +108,11 @@ class Scheduler:
108	108	if state in (LeaseState.PENDING.value, LeaseState.ACTIVE.value):
109	109	self.retry_job(name)
110	110
111		- def create_leases(self):
112		- while self.queue:
	111	+ def create_lease(self):
	112	+ if self.queue:
113	113	job = self.queue.popleft()
114	114	job.update_execute_stage(ExecuteStage.EXECUTING)
115	115	job.create_lease()
116	116	job.lease.state = LeaseState.PENDING.value
117		- yield job.lease
	117	+ return job.lease
	118	+ return None

tests/integration/bots_service.py

---

...	...	@@ -129,7 +129,7 @@ def test_number_of_leases(number_of_jobs, bot_session, context, instance):
129	129	request = bots_pb2.CreateBotSessionRequest(bot_session=bot_session)
130	130	response = instance.CreateBotSession(request, context)
131	131
132		- assert len(response.leases) == number_of_jobs
	132	+ assert len(response.leases) == min(number_of_jobs, 1)
133	133
134	134
135	135	def test_update_leases_with_work(bot_session, context, instance):

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.