[Notes] [Git][BuildStream/buildstream][mablanch/823-action-cache-instance-name] 2 commits: casremote.py: Honor instance name configuration

[Martin Blanchard <gitlab mg gitlab com>]

Title: GitLab

Martin Blanchard pushed to branch mablanch/823-action-cache-instance-name at BuildStream / buildstream

Commits:

• 40ce6e33
  by Martin Blanchard at 2019-02-21T17:56:10Z

  casremote.py: Honor instance name configuration
  
  The remote-execution configuration allows to specify the remote
  instance to query by giving its name. That instance name should be
  attached to every requests submitted to the remote server. This patch
  ensures it.
  

• c71fb34f
  by Martin Blanchard at 2019-02-21T17:56:15Z

  doc: Update configuration examples with 'instance-name'
  

3 changed files:

• buildstream/_cas/casremote.py
• doc/source/format_project.rst
• doc/source/using_config.rst

Changes:

buildstream/_cas/casremote.py

---

...	...	@@ -83,6 +83,7 @@ class CASRemote():
83	83	self.spec = spec
84	84	self._initialized = False
85	85	self.channel = None
	86	+ self.instance_name = None
86	87	self.bytestream = None
87	88	self.cas = None
88	89	self.ref_storage = None
...	...	@@ -125,6 +126,8 @@ class CASRemote():
125	126	else:
126	127	raise CASRemoteError("Unsupported URL: {}".format(self.spec.url))
127	128
	129	+ self.instance_name = self.spec.instance_name or None
	130	+
128	131	self.bytestream = bytestream_pb2_grpc.ByteStreamStub(self.channel)
129	132	self.cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(self.channel)
130	133	self.capabilities = remote_execution_pb2_grpc.CapabilitiesStub(self.channel)
...	...	@@ -133,6 +136,8 @@ class CASRemote():
133	136	self.max_batch_total_size_bytes = _MAX_PAYLOAD_BYTES
134	137	try:
135	138	request = remote_execution_pb2.GetCapabilitiesRequest()
	139	+ if self.instance_name:
	140	+ request.instance_name = self.instance_name
136	141	response = self.capabilities.GetCapabilities(request)
137	142	server_max_batch_total_size_bytes = response.cache_capabilities.max_batch_total_size_bytes
138	143	if 0 < server_max_batch_total_size_bytes < self.max_batch_total_size_bytes:
...	...	@@ -146,6 +151,8 @@ class CASRemote():
146	151	self.batch_read_supported = False
147	152	try:
148	153	request = remote_execution_pb2.BatchReadBlobsRequest()
	154	+ if self.instance_name:
	155	+ request.instance_name = self.instance_name
149	156	response = self.cas.BatchReadBlobs(request)
150	157	self.batch_read_supported = True
151	158	except grpc.RpcError as e:
...	...	@@ -156,6 +163,8 @@ class CASRemote():
156	163	self.batch_update_supported = False
157	164	try:
158	165	request = remote_execution_pb2.BatchUpdateBlobsRequest()
	166	+ if self.instance_name:
	167	+ request.instance_name = self.instance_name
159	168	response = self.cas.BatchUpdateBlobs(request)
160	169	self.batch_update_supported = True
161	170	except grpc.RpcError as e:
...	...	@@ -224,6 +233,8 @@ class CASRemote():
224	233	self.init()
225	234
226	235	request = remote_execution_pb2.FindMissingBlobsRequest()
	236	+ if self.instance_name:
	237	+ request.instance_name = self.instance_name
227	238	request.blob_digests.extend([digest])
228	239
229	240	response = self.cas.FindMissingBlobs(request)
...	...	@@ -258,7 +269,13 @@ class CASRemote():
258	269	# Local Private Methods #
259	270	################################################
260	271	def _fetch_blob(self, digest, stream):
261		- resource_name = '/'.join(['blobs', digest.hash, str(digest.size_bytes)])
	272	+ if self.instance_name:
	273	+ resource_name = '/'.join([self.instance_name, 'blobs',
	274	+ digest.hash, str(digest.size_bytes)])
	275	+ else:
	276	+ resource_name = '/'.join(['blobs',
	277	+ digest.hash, str(digest.size_bytes)])
	278	+
262	279	request = bytestream_pb2.ReadRequest()
263	280	request.resource_name = resource_name
264	281	request.read_offset = 0
...	...	@@ -269,8 +286,12 @@ class CASRemote():
269	286	assert digest.size_bytes == os.fstat(stream.fileno()).st_size
270	287
271	288	def _send_blob(self, digest, stream, u_uid=uuid.uuid4()):
272		- resource_name = '/'.join(['uploads', str(u_uid), 'blobs',
273		- digest.hash, str(digest.size_bytes)])
	289	+ if self.instance_name:
	290	+ resource_name = '/'.join([self.instance_name, 'uploads', str(u_uid), 'blobs',
	291	+ digest.hash, str(digest.size_bytes)])
	292	+ else:
	293	+ resource_name = '/'.join(['uploads', str(u_uid), 'blobs',
	294	+ digest.hash, str(digest.size_bytes)])
274	295
275	296	def request_stream(resname, instream):
276	297	offset = 0
...	...	@@ -304,6 +325,8 @@ class _CASBatchRead():
304	325	self._remote = remote
305	326	self._max_total_size_bytes = remote.max_batch_total_size_bytes
306	327	self._request = remote_execution_pb2.BatchReadBlobsRequest()
	328	+ if remote.instance_name:
	329	+ self._request.instance_name = remote.instance_name
307	330	self._size = 0
308	331	self._sent = False
309	332
...	...	@@ -351,6 +374,8 @@ class _CASBatchUpdate():
351	374	self._remote = remote
352	375	self._max_total_size_bytes = remote.max_batch_total_size_bytes
353	376	self._request = remote_execution_pb2.BatchUpdateBlobsRequest()
	377	+ if remote.instance_name:
	378	+ self._request.instance_name = remote.instance_name
354	379	self._size = 0
355	380	self._sent = False
356	381

doc/source/format_project.rst

---

...	...	@@ -243,6 +243,7 @@ using the `remote-execution` option:
243	243	instance-name: development-emea-1
244	244	action-cache-service:
245	245	url: http://bar.action.com:50052
	246	+ instance-name: development-emea-1
246	247
247	248	storage-service specifies a remote CAS store and the parameters are the
248	249	same as those used to specify an :ref:`artifact server <artifacts>`.

doc/source/using_config.rst

---

...	...	@@ -122,13 +122,14 @@ configuration will be used as fallback.
122	122	url: http://execution.fallback.example.com:50051
123	123	instance-name: main
124	124	storage-service:
125		- url: https://storage.fallback.example.com:11002/
	125	+ url: https://storage.fallback.example.com:11002
126	126	server-cert: /keys/server.crt
127	127	client-cert: /keys/client.crt
128	128	client-key: /keys/client.key
129	129	instance-name: main
130	130	action-cache-service:
131		- url: http://action.flalback.example.com:50052
	131	+ url: http://cache.flalback.example.com:50052
	132	+ instance-name: main
132	133
133	134	2. Project override:
134	135
...	...	@@ -141,13 +142,14 @@ configuration will be used as fallback.
141	142	url: http://execution.some_project.example.com:50051
142	143	instance-name: main
143	144	storage-service:
144		- url: https://storage.some_project.example.com:11002/
	145	+ url: https://storage.some_project.example.com:11002
145	146	server-cert: /some_project_keys/server.crt
146	147	client-cert: /some_project_keys/client.crt
147	148	client-key: /some_project_keys/client.key
148	149	instance-name: main
149	150	action-cache-service:
150		- url: http://action.some_project.example.com:50052
	151	+ url: http://cache.some_project.example.com:50052
	152	+ instance-name: main
151	153
152	154
153	155	Strict build plan

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.