[Notes] [Git][BuildGrid/buildgrid][santi/76-batch-read-blobs] `download_file` command: accept list of files

[Santiago Gil <gitlab mg gitlab com>]

Title: GitLab

Santiago Gil pushed to branch santi/76-batch-read-blobs at BuildGrid / buildgrid

Commits:

• 0ef8490c
  by Santiago Gil at 2019-01-18T12:32:04Z

  `download_file` command: accept list of files
  
  Now a batch of files can be downloaded with a single
  command.
  
  https://gitlab.com/BuildGrid/buildgrid/issues/76
  
  Signed-off-by: Santiago Gil <santiago gil codethink co uk>
  

2 changed files:

• buildgrid/_app/commands/cmd_cas.py
• buildgrid/server/cas/instance.py

Changes:

buildgrid/_app/commands/cmd_cas.py

---

...	...	@@ -147,33 +147,41 @@ def _create_digest(digest_string):
147	147	return digest
148	148
149	149
150		-@cli.command('download-file', short_help="Download a file from the CAS server.")
151		-@click.argument('digest-string', nargs=1, type=click.STRING, required=True)
152		-@click.argument('file-path', nargs=1, type=click.Path(exists=False), required=True)
	150	+@cli.command('download-file', short_help="Download one or more files from the CAS server. "
	151	+ "(Specified as a space-separated list of DIGEST FILE_PATH)")
	152	+@click.argument('digest-path-list', nargs=-1, type=str, required=True) # 'digest path' pairs
153	153	@click.option('--verify', is_flag=True, show_default=True,
154	154	help="Check downloaded file's integrity.")
155	155	@pass_context
156		-def download_file(context, digest_string, file_path, verify):
157		- if os.path.exists(file_path):
158		- click.echo("Error: Invalid value for " +
159		- "path=[{}] already exists.".format(file_path), err=True)
160		- return
161		-
162		- digest = _create_digest(digest_string)
	156	+def download_file(context, digest_path_list, verify):
	157	+ # Downloading files:
	158	+ downloaded_files = {}
163	159	with download(context.channel, instance=context.instance_name) as downloader:
164		- downloader.download_file(digest, file_path)
165		-
166		- if verify:
167		- file_digest = create_digest(read_file(file_path))
168		- if file_digest != digest:
169		- click.echo("Error: Failed to verify path=[{}]".format(file_path), err=True)
170		- return
171		-
172		- if os.path.isfile(file_path):
173		- click.echo("Success: Pulled path=[{}] from digest=[{}/{}]"
174		- .format(file_path, digest.hash, digest.size_bytes))
175		- else:
176		- click.echo('Error: Failed pulling "{}"'.format(file_path), err=True)
	160	+ for (digest_string, file_path) in zip(digest_path_list[0::2],
	161	+ digest_path_list[1::2]):
	162	+ if os.path.exists(file_path):
	163	+ click.echo("Error: Invalid value for " +
	164	+ "path=[{}] already exists.".format(file_path), err=True)
	165	+ continue
	166	+
	167	+ digest = _create_digest(digest_string)
	168	+
	169	+ downloader.download_file(digest, file_path)
	170	+ downloaded_files[file_path] = digest
	171	+
	172	+ # Verifying:
	173	+ for (file_path, digest) in downloaded_files.items():
	174	+ if verify:
	175	+ file_digest = create_digest(read_file(file_path))
	176	+ if file_digest != digest:
	177	+ click.echo("Error: Failed to verify path=[{}]".format(file_path), err=True)
	178	+ continue
	179	+
	180	+ if os.path.isfile(file_path):
	181	+ click.echo("Success: Pulled path=[{}] from digest=[{}/{}]"
	182	+ .format(file_path, digest.hash, digest.size_bytes))
	183	+ else:
	184	+ click.echo('Error: Failed pulling "{}"'.format(file_path), err=True)
177	185
178	186
179	187	@cli.command('download-dir', short_help="Download a directory from the CAS server.")

buildgrid/server/cas/instance.py

---

...	...	@@ -72,6 +72,8 @@ class ContentAddressableStorageInstance:
72	72	return response
73	73
74	74	def batch_read_blobs(self, digests):
	75	+ storage = self._storage
	76	+
75	77	response = re_pb2.BatchReadBlobsResponse()
76	78
77	79	requested_bytes = sum((digest.size_bytes for digest in digests))
...	...	@@ -87,7 +89,7 @@ class ContentAddressableStorageInstance:
87	89	response_proto = response.responses.add()
88	90	response_proto.digest.CopyFrom(digest)
89	91
90		- blob = self._storage.get_blob(digest)
	92	+ blob = storage.get_blob(digest)
91	93	if blob:
92	94	response_proto.data = blob.read()
93	95	status_code = code_pb2.OK

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.