Arber Xhindoli pushed to branch arber/91-get-tree at BuildGrid / buildgrid
Commits:
-
be1205a7
by Arber Xhindoli at 2018-10-26T22:56:40Z
3 changed files:
Changes:
| ... | ... | @@ -54,6 +54,19 @@ class ContentAddressableStorageInstance: |
| 54 | 54 |
|
| 55 | 55 |
return response
|
| 56 | 56 |
|
| 57 |
+ def get_tree(self, request, directory_list, digest_list):
|
|
| 58 |
+ """
|
|
| 59 |
+ This function will start reading directories at request.root_digest.
|
|
| 60 |
+ It will push the directories, and their corresponding digests into the
|
|
| 61 |
+ directory_list and digest_list.
|
|
| 62 |
+ |
|
| 63 |
+ It will continue to do a level-order traversal until either: directory_list reaches the end,
|
|
| 64 |
+ or we have made request.page_size reads. If the latter case, it will return len(directory_list) - 1 so
|
|
| 65 |
+ subsequent calls can pick up where it left off.
|
|
| 66 |
+ Otherwise, returns None, meaning we have read the directory tree.
|
|
| 67 |
+ """
|
|
| 68 |
+ return None
|
|
| 69 |
+ |
|
| 57 | 70 |
|
| 58 | 71 |
class ByteStreamInstance:
|
| 59 | 72 |
|
| ... | ... | @@ -84,6 +84,56 @@ class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressa |
| 84 | 84 |
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
| 85 | 85 |
context.set_details('Method not implemented!')
|
| 86 | 86 |
|
| 87 |
+ # Stores the directories as long as a page token is returned.
|
|
| 88 |
+ directories = []
|
|
| 89 |
+ # Stores the digests of those directories
|
|
| 90 |
+ digests = []
|
|
| 91 |
+ |
|
| 92 |
+ # if page_size is not set
|
|
| 93 |
+ if request.page_size == 0:
|
|
| 94 |
+ request.page_size = 500
|
|
| 95 |
+ |
|
| 96 |
+ # Set to 0, will be used to index into directory list,
|
|
| 97 |
+ # and updated in instance.get_tree. This is the only way this makes sense to me.
|
|
| 98 |
+ request.page_token = 0
|
|
| 99 |
+ |
|
| 100 |
+ # start at index 1, to not return root
|
|
| 101 |
+ start_index = 1
|
|
| 102 |
+ |
|
| 103 |
+ try:
|
|
| 104 |
+ instance = self._get_instance(request.instance_name)
|
|
| 105 |
+ while True:
|
|
| 106 |
+ self.logger.debug("GetTree request: [{}]".format(request))
|
|
| 107 |
+ # Returns next page_token once page_size directories is reached.
|
|
| 108 |
+ # The page_token, is essentially an index into the directories/digests list.
|
|
| 109 |
+ page_token = instance.get_tree(
|
|
| 110 |
+ request, directories, digests)
|
|
| 111 |
+ |
|
| 112 |
+ response = remote_execution_pb2.GetTreeResponse()
|
|
| 113 |
+ if not page_token:
|
|
| 114 |
+ # get directories from last request to the end since no page_token
|
|
| 115 |
+ response.directories = directories[start_index:]
|
|
| 116 |
+ response.page_token = None
|
|
| 117 |
+ # stop the generator no more directories
|
|
| 118 |
+ return response
|
|
| 119 |
+ else:
|
|
| 120 |
+ # return from last request, to current request directories
|
|
| 121 |
+ response.directories = directories[start_index:page_token]
|
|
| 122 |
+ response.page_token = str(page_token)
|
|
| 123 |
+ yield response
|
|
| 124 |
+ |
|
| 125 |
+ # create new request using returned page token, update start_index
|
|
| 126 |
+ request = remote_execution_pb2.GetTreeRequest()
|
|
| 127 |
+ request.page_size = 500
|
|
| 128 |
+ request.page_token = page_token
|
|
| 129 |
+ request.root_digest = digests[page_token]
|
|
| 130 |
+ start_index = page_token
|
|
| 131 |
+ |
|
| 132 |
+ except InvalidArgumentError as e:
|
|
| 133 |
+ self.logger.error(e)
|
|
| 134 |
+ context.set_details(str(e))
|
|
| 135 |
+ context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
|
|
| 136 |
+ |
|
| 87 | 137 |
return iter([remote_execution_pb2.GetTreeResponse()])
|
| 88 | 138 |
|
| 89 | 139 |
def _get_instance(self, instance_name):
|
| ... | ... | @@ -18,11 +18,11 @@ |
| 18 | 18 |
# pylint: disable=redefined-outer-name
|
| 19 | 19 |
|
| 20 | 20 |
import tempfile
|
| 21 |
+from moto import mock_s3
|
|
| 21 | 22 |
|
| 22 | 23 |
import boto3
|
| 23 | 24 |
import grpc
|
| 24 | 25 |
import pytest
|
| 25 |
-from moto import mock_s3
|
|
| 26 | 26 |
|
| 27 | 27 |
from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
|
| 28 | 28 |
from buildgrid.server.cas.storage.remote import RemoteStorage
|