[Notes] [Git][BuildGrid/buildgrid][arber/91-get-tree] Update cas/test_client.py in order to throughly test getTree. Add GRPC limits to getTree method.

[Arber Xhindoli <gitlab mg gitlab com>]

Title: GitLab

Arber Xhindoli pushed to branch arber/91-get-tree at BuildGrid / buildgrid

Commits:

• 9b5b7484
  by Arber Xhindoli at 2018-11-20T20:06:56Z

  Update cas/test_client.py in order to throughly test getTree. Add GRPC limits to getTree method.
  

6 changed files:

• buildgrid/client/cas.py
• buildgrid/server/cas/instance.py
• buildgrid/server/cas/service.py
• tests/cas/data/hello/hello.h → tests/cas/data/hello/hello2/hello.h
• + tests/cas/data/hello/hello3/hello4/hello5/hello.h
• tests/cas/test_client.py

Changes:

buildgrid/client/cas.py

---

...	...	@@ -31,7 +31,7 @@ from buildgrid.utils import merkle_tree_maker
31	31	FILE_SIZE_THRESHOLD = 1 * 1024 * 1024
32	32
33	33	# Maximum size for a single gRPC request:
34		-MAX_REQUEST_SIZE = 2 * 1024 * 1024
	34	+MAX_REQUEST_SIZE = 4 * 1024 * 1024
35	35
36	36	# Maximum number of elements per gRPC request:
37	37	MAX_REQUEST_COUNT = 500

buildgrid/server/cas/instance.py

---

...	...	@@ -26,6 +26,8 @@ from buildgrid._protos.google.bytestream import bytestream_pb2
26	26	from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2
27	27	from buildgrid.settings import HASH
28	28
	29	+GRPC_MAX_REQUEST_SIZE = (4 * 1024 * 1024)
	30	+
29	31
30	32	class ContentAddressableStorageInstance:
31	33
...	...	@@ -58,37 +60,31 @@ class ContentAddressableStorageInstance:
58	60
59	61	return response
60	62
61		- def get_tree(self, request):
	63	+ def get_tree(self, request, cache):
62	64	storage = self._storage
63		-
64	65	response = re_pb2.GetTreeResponse()
65		- page_size = request.page_size
66		- next_page_token = ""
67		- directories = []
68		-
69		- if not page_size:
70		- # MAX_REQUEST_COUNT
71		- page_size = 500
72	66
73	67	def _get_tree(node_digest):
74		- nonlocal directories, page_size, next_page_token
75		- if next_page_token:
76		- # next page token has been set unwind the stack.
77		- return
78		- if page_size <= 0:
79		- # save the next digest hash in order to continue later
80		- next_page_token = str(node_digest.hash)
	68	+ nonlocal cache, response
	69	+
	70	+ if response.ByteSize() + node_digest.size_bytes >= (GRPC_MAX_REQUEST_SIZE - 100):
	71	+ response.next_page_token = str(node_digest)
81	72	return
82		- directory_from_digest = storage.get_message(node_digest, re_pb2.Directory)
83		- directories.append(directory_from_digest)
	73	+
	74	+ hash_node = node_digest.hash
	75	+
	76	+ if hash_node in cache:
	77	+ directory_from_digest = cache[hash_node]
	78	+
	79	+ else:
	80	+ directory_from_digest = storage.get_message(node_digest, re_pb2.Directory)
	81	+ response.directories.extend([directory_from_digest])
	82	+ cache[hash_node] = directory_from_digest
	83	+
84	84	for directory in directory_from_digest.directories:
85		- page_size -= 1
86	85	_get_tree(directory.digest)
87	86	return
88		-
89	87	_get_tree(request.root_digest)
90		- response.directories.extend(directories)
91		- response.next_page_token = next_page_token
92	88	return response
93	89
94	90

buildgrid/server/cas/service.py

---

...	...	@@ -87,10 +87,12 @@ class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressa
87	87	def GetTree(self, request, context):
88	88	self.__logger.debug("GetTree request from [%s]", context.peer())
89	89
	90	+ cache = {}
	91	+
90	92	try:
91	93	instance = self._get_instance(request.instance_name)
92	94	while True:
93		- response = instance.get_tree(request)
	95	+ response = instance.get_tree(request, cache)
94	96	if not response.next_page_token:
95	97	# This is ugly, but handles the case in which there is only one iteration
96	98	# of the while loop. We need to manually raise stopiteration from this generator.
...	...	@@ -99,7 +101,6 @@ class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressa
99	101	raise StopIteration # pylint: disable=stop-iteration-return
100	102
101	103	yield response
102		- request.root_digest.hash = response.next_page_token
103	104
104	105	except InvalidArgumentError as e:
105	106	self.__logger.error(e)

tests/cas/data/hello/hello.h → tests/cas/data/hello/hello2/hello.h

---

tests/cas/data/hello/hello3/hello4/hello5/hello.h

---

1

+#define HELLO_WORLD "Hello, World!"

tests/cas/test_client.py

---

...	...	@@ -39,16 +39,31 @@ MESSAGES = [
39	39	]
40	40	DATA_DIR = os.path.join(
41	41	os.path.dirname(os.path.realpath(__file__)), 'data')
	42	+
	43	+HELLO_DIR = os.path.join(DATA_DIR, 'hello')
	44	+HELLO2_DIR = os.path.join(HELLO_DIR, 'hello2')
	45	+HELLO3_DIR = os.path.join(HELLO_DIR, 'hello3')
	46	+HELLO4_DIR = os.path.join(HELLO3_DIR, 'hello4')
	47	+HELLO5_DIR = os.path.join(HELLO4_DIR, 'hello5')
	48	+
42	49	FILES = [
43	50	(os.path.join(DATA_DIR, 'void'),),
44	51	(os.path.join(DATA_DIR, 'hello.cc'),),
45	52	(os.path.join(DATA_DIR, 'hello', 'hello.c'),
46		- os.path.join(DATA_DIR, 'hello', 'hello.h'))]
	53	+ os.path.join(DATA_DIR, 'hello', 'hello.sh')),
	54	+ (os.path.join(HELLO2_DIR, 'hello.h'),),
	55	+ (os.path.join(HELLO5_DIR, 'hello.h'),), ]
	56	+
47	57	FOLDERS = [
48		- (os.path.join(DATA_DIR, 'hello'),)]
	58	+ (HELLO_DIR, HELLO2_DIR, HELLO3_DIR, HELLO4_DIR, HELLO5_DIR)]
	59	+
49	60	DIRECTORIES = [
50		- (os.path.join(DATA_DIR, 'hello'),),
51		- (os.path.join(DATA_DIR, 'hello'), DATA_DIR)]
	61	+ (HELLO_DIR,),
	62	+ (DATA_DIR,),
	63	+ (HELLO2_DIR,),
	64	+ (HELLO3_DIR,),
	65	+ (HELLO4_DIR,),
	66	+ (HELLO5_DIR,), ]
52	67
53	68
54	69	@pytest.mark.parametrize('blobs', BLOBS)
...	...	@@ -355,7 +370,6 @@ def test_download_directory(instance, folder_paths):
355	370	path = os.path.relpath(folder_path, start=DATA_DIR)
356	371	path = os.path.join(temp_folder, path)
357	372	paths.append(path)
358		-
359	373	run_in_subprocess(__test_download_directory,
360	374	server.remote, instance, digests, paths)
361	375

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.