| ... |
... |
@@ -171,7 +171,7 @@ class Downloader: |
|
171
|
171
|
|
|
172
|
172
|
return messages
|
|
173
|
173
|
|
|
174
|
|
- def download_file(self, digest, file_path, queue=True):
|
|
|
174
|
+ def download_file(self, digest, file_path, is_executable=False, queue=True):
|
|
175
|
175
|
"""Retrieves a file from the remote CAS server.
|
|
176
|
176
|
|
|
177
|
177
|
If queuing is allowed (`queue=True`), the download request **may** be
|
| ... |
... |
@@ -181,6 +181,7 @@ class Downloader: |
|
181
|
181
|
Args:
|
|
182
|
182
|
digest (:obj:`Digest`): the file's digest to fetch.
|
|
183
|
183
|
file_path (str): absolute or relative path to the local file to write.
|
|
|
184
|
+ is_executable (bool): whether the file is executable or not.
|
|
184
|
185
|
queue (bool, optional): whether or not the download request may be
|
|
185
|
186
|
queued and submitted as part of a batch upload request. Defaults
|
|
186
|
187
|
to True.
|
| ... |
... |
@@ -193,9 +194,9 @@ class Downloader: |
|
193
|
194
|
file_path = os.path.abspath(file_path)
|
|
194
|
195
|
|
|
195
|
196
|
if not queue or digest.size_bytes > FILE_SIZE_THRESHOLD:
|
|
196
|
|
- self._fetch_file(digest, file_path)
|
|
|
197
|
+ self._fetch_file(digest, file_path, is_executable=is_executable)
|
|
197
|
198
|
else:
|
|
198
|
|
- self._queue_file(digest, file_path)
|
|
|
199
|
+ self._queue_file(digest, file_path, is_executable=is_executable)
|
|
199
|
200
|
|
|
200
|
201
|
def download_directory(self, digest, directory_path):
|
|
201
|
202
|
"""Retrieves a :obj:`Directory` from the remote CAS server.
|
| ... |
... |
@@ -311,7 +312,7 @@ class Downloader: |
|
311
|
312
|
|
|
312
|
313
|
return read_blobs
|
|
313
|
314
|
|
|
314
|
|
- def _fetch_file(self, digest, file_path):
|
|
|
315
|
+ def _fetch_file(self, digest, file_path, is_executable=False):
|
|
315
|
316
|
"""Fetches a file using ByteStream.Read()"""
|
|
316
|
317
|
if self.instance_name:
|
|
317
|
318
|
resource_name = '/'.join([self.instance_name, 'blobs',
|
| ... |
... |
@@ -332,7 +333,10 @@ class Downloader: |
|
332
|
333
|
|
|
333
|
334
|
assert byte_file.tell() == digest.size_bytes
|
|
334
|
335
|
|
|
335
|
|
- def _queue_file(self, digest, file_path):
|
|
|
336
|
+ if is_executable:
|
|
|
337
|
+ os.chmod(file_path, 0o755) # rwxr-xr-x / 755
|
|
|
338
|
+
|
|
|
339
|
+ def _queue_file(self, digest, file_path, is_executable=False):
|
|
336
|
340
|
"""Queues a file for later batch download"""
|
|
337
|
341
|
if self.__file_request_size + digest.ByteSize() > MAX_REQUEST_SIZE:
|
|
338
|
342
|
self.flush()
|
| ... |
... |
@@ -341,22 +345,25 @@ class Downloader: |
|
341
|
345
|
elif self.__file_request_count >= MAX_REQUEST_COUNT:
|
|
342
|
346
|
self.flush()
|
|
343
|
347
|
|
|
344
|
|
- self.__file_requests[digest.hash] = (digest, file_path)
|
|
|
348
|
+ self.__file_requests[digest.hash] = (digest, file_path, is_executable)
|
|
345
|
349
|
self.__file_request_count += 1
|
|
346
|
350
|
self.__file_request_size += digest.ByteSize()
|
|
347
|
351
|
self.__file_response_size += digest.size_bytes
|
|
348
|
352
|
|
|
349
|
353
|
def _fetch_file_batch(self, batch):
|
|
350
|
354
|
"""Sends queued data using ContentAddressableStorage.BatchReadBlobs()"""
|
|
351
|
|
- batch_digests = [digest for digest, _ in batch.values()]
|
|
|
355
|
+ batch_digests = [digest for digest, _, _ in batch.values()]
|
|
352
|
356
|
batch_blobs = self._fetch_blob_batch(batch_digests)
|
|
353
|
357
|
|
|
354
|
|
- for (_, file_path), file_blob in zip(batch.values(), batch_blobs):
|
|
|
358
|
+ for (_, file_path, is_executable), file_blob in zip(batch.values(), batch_blobs):
|
|
355
|
359
|
os.makedirs(os.path.dirname(file_path), exist_ok=True)
|
|
356
|
360
|
|
|
357
|
361
|
with open(file_path, 'wb') as byte_file:
|
|
358
|
362
|
byte_file.write(file_blob)
|
|
359
|
363
|
|
|
|
364
|
+ if is_executable:
|
|
|
365
|
+ os.chmod(file_path, 0o755) # rwxr-xr-x / 755
|
|
|
366
|
+
|
|
360
|
367
|
def _fetch_directory(self, digest, directory_path):
|
|
361
|
368
|
"""Fetches a file using ByteStream.GetTree()"""
|
|
362
|
369
|
# Better fail early if the local root path cannot be created:
|
| ... |
... |
@@ -414,7 +421,7 @@ class Downloader: |
|
414
|
421
|
for file_node in root_directory.files:
|
|
415
|
422
|
file_path = os.path.join(root_path, file_node.name)
|
|
416
|
423
|
|
|
417
|
|
- self._queue_file(file_node.digest, file_path)
|
|
|
424
|
+ self._queue_file(file_node.digest, file_path, is_executable=file_node.is_executable)
|
|
418
|
425
|
|
|
419
|
426
|
for directory_node in root_directory.directories:
|
|
420
|
427
|
directory_path = os.path.join(root_path, directory_node.name)
|
