[Notes] [Git][BuildGrid/buildgrid][santi/146-write-dir-force-flush] 2 commits: Add files to the `tests/cas/data/` directory

[Santiago Gil <gitlab mg gitlab com>]

Title: GitLab

Santiago Gil pushed to branch santi/146-write-dir-force-flush at BuildGrid / buildgrid

Commits:

• f30adefb
  by Santiago Gil at 2019-01-24T10:06:45Z

  Add files to the `tests/cas/data/` directory
  
  This allows tests to consider more complex directory structures.
  
  (This is the tests/ directory from branch
  `mablanch/91-download-tree-test-error` mentioned in
  https://gitlab.com/BuildGrid/buildgrid/issues/146).
  

• f2cae65e
  by Santiago Gil at 2019-01-24T10:06:52Z

  Downloader.download_directory(): allow digest to multiple paths
  
  These changes refactor the request queue, adding support for the
  scenario where we want to download multiple copies of a file with the
  same digest. (Previously only one instance of that file was created,
  which resulted in files missing from the output.)
  
  Addresses https://gitlab.com/BuildGrid/buildgrid/issues/146
  

5 changed files:

• buildgrid/client/cas.py
• + tests/cas/data/hello/docs/reference/api.xml
• + tests/cas/data/hello/utils/hello.h
• + tests/cas/data/hello/utils/hello2.sh
• tests/cas/test_client.py

Changes:

buildgrid/client/cas.py

---

...	...	@@ -13,9 +13,10 @@
13	13	# limitations under the License.
14	14
15	15
16		-from contextlib import contextmanager
17		-import uuid
18	16	import os
	17	+import uuid
	18	+from collections import namedtuple
	19	+from contextlib import contextmanager
19	20
20	21	import grpc
21	22
...	...	@@ -30,6 +31,8 @@ from buildgrid.utils import merkle_tree_maker
30	31	# Maximum size for a queueable file:
31	32	FILE_SIZE_THRESHOLD = 1 * 1024 * 1024
32	33
	34	+_FileRequest = namedtuple('FileRequest', ['digest', 'output_paths'])
	35	+
33	36
34	37	class _CallCache:
35	38	"""Per remote grpc.StatusCode.UNIMPLEMENTED call cache."""
...	...	@@ -339,24 +342,41 @@ class Downloader:
339	342	elif self.__file_request_count >= MAX_REQUEST_COUNT:
340	343	self.flush()
341	344
342		- self.__file_requests[digest.hash] = (digest, file_path, is_executable)
343		- self.__file_request_count += 1
344		- self.__file_request_size += digest.ByteSize()
345		- self.__file_response_size += digest.size_bytes
	345	+ output_path = (file_path, is_executable)
	346	+
	347	+ # When queueing a file we take into account the cases where
	348	+ # we might want to download the same digest to different paths.
	349	+ if digest.hash not in self.__file_requests:
	350	+ request = _FileRequest(digest=digest, output_paths=[output_path])
	351	+ self.__file_requests[digest.hash] = request
346	352
347		- def _fetch_file_batch(self, batch):
348		- """Sends queued data using ContentAddressableStorage.BatchReadBlobs()"""
349		- batch_digests = [digest for digest, _, _ in batch.values()]
	353	+ self.__file_request_count += 1
	354	+ self.__file_request_size += digest.ByteSize()
	355	+ self.__file_response_size += digest.size_bytes
	356	+ else:
	357	+ # We already have that hash queued; we'll fetch the blob
	358	+ # once and write copies of it:
	359	+ self.__file_requests[digest.hash].output_paths.append(output_path)
	360	+
	361	+ def _fetch_file_batch(self, requests):
	362	+ """Sends queued data using ContentAddressableStorage.BatchReadBlobs().
	363	+
	364	+ Takes a dictionary (digest.hash, _FileRequest) as input.
	365	+ """
	366	+ batch_digests = [request.digest for request in requests.values()]
350	367	batch_blobs = self._fetch_blob_batch(batch_digests)
351	368
352		- for (_, file_path, is_executable), file_blob in zip(batch.values(), batch_blobs):
353		- os.makedirs(os.path.dirname(file_path), exist_ok=True)
	369	+ for file_digest, file_blob in zip(batch_digests, batch_blobs):
	370	+ output_paths = requests[file_digest.hash].output_paths
	371	+
	372	+ for (file_path, is_executable) in output_paths:
	373	+ os.makedirs(os.path.dirname(file_path), exist_ok=True)
354	374
355		- with open(file_path, 'wb') as byte_file:
356		- byte_file.write(file_blob)
	375	+ with open(file_path, 'wb') as byte_file:
	376	+ byte_file.write(file_blob)
357	377
358		- if is_executable:
359		- os.chmod(file_path, 0o755) # rwxr-xr-x / 755
	378	+ if is_executable:
	379	+ os.chmod(file_path, 0o755) # rwxr-xr-x / 755
360	380
361	381	def _fetch_directory(self, digest, directory_path):
362	382	"""Fetches a file using ByteStream.GetTree()"""

tests/cas/data/hello/docs/reference/api.xml

---

tests/cas/data/hello/utils/hello.h

---

1

+#define HELLO_WORLD "Hello, World!"

tests/cas/data/hello/utils/hello2.sh

---

	1	+#!/bin/bash
	2	+
	3	+echo "Hello, World!"

tests/cas/test_client.py

---

...	...	@@ -44,12 +44,18 @@ FILES = [
44	44	(os.path.join(DATA_DIR, 'void'),),
45	45	(os.path.join(DATA_DIR, 'hello.cc'),),
46	46	(os.path.join(DATA_DIR, 'hello', 'hello.c'),
47		- os.path.join(DATA_DIR, 'hello', 'hello.h'))]
	47	+ os.path.join(DATA_DIR, 'hello', 'hello.h'),
	48	+ os.path.join(DATA_DIR, 'hello', 'hello.sh')),
	49	+ (os.path.join(DATA_DIR, 'hello', 'docs', 'reference', 'api.xml'),)]
48	50	FOLDERS = [
49		- (os.path.join(DATA_DIR, 'hello'),)]
50		-DIRECTORIES = [
	51	+ (DATA_DIR,),
51	52	(os.path.join(DATA_DIR, 'hello'),),
52		- (os.path.join(DATA_DIR, 'hello'), DATA_DIR)]
	53	+ (os.path.join(DATA_DIR, 'hello', 'docs'),),
	54	+ (os.path.join(DATA_DIR, 'hello', 'utils'),),
	55	+ (os.path.join(DATA_DIR, 'hello', 'docs', 'reference'),)]
	56	+DIRECTORIES = [
	57	+ (DATA_DIR,),
	58	+ (os.path.join(DATA_DIR, 'hello'),)]
53	59
54	60
55	61	@pytest.mark.parametrize('blobs', BLOBS)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.