[Notes] [Git][BuildGrid/buildgrid][arber/91-get-tree] Start the implementation of the getTree method



Title: GitLab

Arber Xhindoli pushed to branch arber/91-get-tree at BuildGrid / buildgrid

Commits:

3 changed files:

Changes:

  • buildgrid/server/cas/instance.py
    ... ... @@ -54,6 +54,19 @@ class ContentAddressableStorageInstance:
    54 54
     
    
    55 55
             return response
    
    56 56
     
    
    57
    +    def get_tree(self, request, directory_list, digest_list):
    
    58
    +        """
    
    59
    +        This function will start reading directories at request.root_digest.
    
    60
    +        It will push the directories, and their corresponding digests into the
    
    61
    +        directory_list and digest_list.
    
    62
    +
    
    63
    +        It will continue to do a level-order traversal until either: directory_list reaches the end,
    
    64
    +        or we have made request.page_size reads. If the latter case, it will return len(directory_list) - 1 so
    
    65
    +        subsequent calls can pick up where it left off.
    
    66
    +        Otherwise, returns None, meaning we have read the directory tree.
    
    67
    +        """
    
    68
    +        return None
    
    69
    +
    
    57 70
     
    
    58 71
     class ByteStreamInstance:
    
    59 72
     
    

  • buildgrid/server/cas/service.py
    ... ... @@ -84,6 +84,56 @@ class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressa
    84 84
             context.set_code(grpc.StatusCode.UNIMPLEMENTED)
    
    85 85
             context.set_details('Method not implemented!')
    
    86 86
     
    
    87
    +        # Stores the directories as long as a page token is returned.
    
    88
    +        directories = []
    
    89
    +        # Stores the digests of those directories
    
    90
    +        digests = []
    
    91
    +
    
    92
    +        # if page_size is not set
    
    93
    +        if request.page_size == 0:
    
    94
    +            request.page_size = 500
    
    95
    +
    
    96
    +        # Set to 0, will be used to index into directory list,
    
    97
    +        # and updated in instance.get_tree. This is the only way this makes sense to me.
    
    98
    +        request.page_token = 0
    
    99
    +
    
    100
    +        # start at index 1, to not return root
    
    101
    +        start_index = 1
    
    102
    +
    
    103
    +        try:
    
    104
    +            instance = self._get_instance(request.instance_name)
    
    105
    +            while True:
    
    106
    +                self.logger.debug("GetTree request: [{}]".format(request))
    
    107
    +                # Returns next page_token once page_size directories is reached.
    
    108
    +                # The page_token, is essentially an index into the directories/digests list.
    
    109
    +                page_token = instance.get_tree(
    
    110
    +                    request, directories, digests)
    
    111
    +
    
    112
    +                response = remote_execution_pb2.GetTreeResponse()
    
    113
    +                if not page_token:
    
    114
    +                    # get directories from last request to the end since no page_token
    
    115
    +                    response.directories = directories[start_index:]
    
    116
    +                    response.page_token = None
    
    117
    +                    # stop the generator no more directories
    
    118
    +                    return response
    
    119
    +                else:
    
    120
    +                    # return from last request, to current request directories
    
    121
    +                    response.directories = directories[start_index:page_token]
    
    122
    +                    response.page_token = str(page_token)
    
    123
    +                    yield response
    
    124
    +
    
    125
    +                # create new request using returned page token, update start_index
    
    126
    +                request = remote_execution_pb2.GetTreeRequest()
    
    127
    +                request.page_size = 500
    
    128
    +                request.page_token = page_token
    
    129
    +                request.root_digest = digests[page_token]
    
    130
    +                start_index = page_token
    
    131
    +
    
    132
    +        except InvalidArgumentError as e:
    
    133
    +            self.logger.error(e)
    
    134
    +            context.set_details(str(e))
    
    135
    +            context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
    
    136
    +
    
    87 137
             return iter([remote_execution_pb2.GetTreeResponse()])
    
    88 138
     
    
    89 139
         def _get_instance(self, instance_name):
    

  • tests/cas/test_storage.py
    ... ... @@ -18,11 +18,11 @@
    18 18
     # pylint: disable=redefined-outer-name
    
    19 19
     
    
    20 20
     import tempfile
    
    21
    +from moto import mock_s3
    
    21 22
     
    
    22 23
     import boto3
    
    23 24
     import grpc
    
    24 25
     import pytest
    
    25
    -from moto import mock_s3
    
    26 26
     
    
    27 27
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    
    28 28
     from buildgrid.server.cas.storage.remote import RemoteStorage
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]