GNOME.org
 

[Notes] [Git][BuildStream/buildstream][raoul/628-RE-flow-optimisation] 2 commits: _sandboxremote.py: Add action-service endpoint

Title: GitLab

Raoul Hidalgo Charman pushed to branch raoul/628-RE-flow-optimisation at BuildStream / buildstream

Commits:

2 changed files:

Changes:

  • buildstream/sandbox/_sandboxremote.py
    ... ... @@ -26,6 +26,7 @@ from functools import partial
    26 26
    27 27 
     import grpc
    28 28
    29 
    +from buildstream._message import Message, MessageType
    29 30 
     from buildstream import utils
    30 31 
     from . import Sandbox, SandboxCommandError
    31 32 
     from .sandbox import _SandboxBatch
    ... ... @@ -40,7 +41,7 @@ from .._protos.google.longrunning import operations_pb2, operations_pb2_grpc
    40 41 
     from .._artifactcache.cascache import CASRemote, CASRemoteSpec
    41 42
    42 43
    43 
    -class RemoteExecutionSpec(namedtuple('RemoteExecutionSpec', 'exec_service storage_service')):
    44 
    +class RemoteExecutionSpec(namedtuple('RemoteExecutionSpec', 'exec_service storage_service action_service')):
    44 45 
         pass
    45 46
    46 47
    ... ... @@ -60,6 +61,10 @@ class SandboxRemote(Sandbox):
    60 61
    61 62 
             self.storage_url = config.storage_service['url']
    62 63 
             self.exec_url = config.exec_service['url']
    64 
    +        if config.action_service:
    65 
    +            self.action_url = config.action_service['url']
    66 
    +        else:
    67 
    +            self.action_url = None
    63 68
    64 69 
             self.storage_remote_spec = CASRemoteSpec(self.storage_url, push=True,
    65 70 
                                                      server_cert=config.storage_service['server-cert'],
    ... ... @@ -67,6 +72,9 @@ class SandboxRemote(Sandbox):
    67 72 
                                                      client_cert=config.storage_service['client-cert'])
    68 73 
             self.operation_name = None
    69 74
    75 
    +    def info(self, msg):
    76 
    +        self._get_context().message(Message(None, MessageType.INFO, msg))
    77 
    +
    70 78 
         @staticmethod
    71 79 
         def specs_from_config_node(config_node, basedir):
    72 80
    ... ... @@ -89,12 +97,17 @@ class SandboxRemote(Sandbox):
    89 97
    90 98 
             tls_keys = ['client-key', 'client-cert', 'server-cert']
    91 99
    92 
    -        _yaml.node_validate(remote_config, ['execution-service', 'storage-service', 'url'])
    100 
    +        _yaml.node_validate(remote_config, ['execution-service', 'storage-service', 'url', 'action-service'])
    93 101 
             remote_exec_service_config = require_node(remote_config, 'execution-service')
    94 102 
             remote_exec_storage_config = require_node(remote_config, 'storage-service')
    103 
    +        remote_exec_action_config = remote_config.get('action-service')
    95 104
    96 105 
             _yaml.node_validate(remote_exec_service_config, ['url'])
    97 106 
             _yaml.node_validate(remote_exec_storage_config, ['url'] + tls_keys)
    107 
    +        if remote_exec_action_config:
    108 
    +            _yaml.node_validate(remote_exec_action_config, ['url'])
    109 
    +        else:
    110 
    +            remote_config['action-service'] = None
    98 111
    99 112 
             if 'url' in remote_config:
    100 113 
                 if 'execution-service' not in remote_config:
    ... ... @@ -115,7 +128,9 @@ class SandboxRemote(Sandbox):
    115 128 
                                           "remote-execution configuration. Your config is missing '{}'."
    116 129 
                                           .format(str(provenance), tls_keys, key))
    117 130
    118 
    -        spec = RemoteExecutionSpec(remote_config['execution-service'], remote_config['storage-service'])
    131 
    +        spec = RemoteExecutionSpec(remote_config['execution-service'],
    132 
    +                                   remote_config['storage-service'],
    133 
    +                                   remote_config['action-service'])
    119 134 
             return spec
    120 135
    121 136 
         def run_remote_command(self, channel, action_digest):
    ... ... @@ -277,7 +292,7 @@ class SandboxRemote(Sandbox):
    277 292 
                                    "and '{}' was supplied.".format(url.scheme))
    278 293
    279 294 
             # check action cache download and download if there
    280 
    -        action_result = self._check_action_cache(channel, action_digest)
    295 
    +        action_result = self._check_action_cache(action_digest)
    281 296
    282 297 
             if not action_result:
    283 298 
                 casremote = CASRemote(self.storage_remote_spec)
    ... ... @@ -318,21 +333,28 @@ class SandboxRemote(Sandbox):
    318 333
    319 334 
             return 0
    320 335
    321 
    -    def _check_action_cache(self, channel, action_digest):
    336 
    +    def _check_action_cache(self, action_digest):
    322 337 
             # Checks the action cache to see if this artifact has already been built
    323 338 
             #
    324 339 
             # Should return either the action response or None if not found, raise
    325 340 
             # Sandboxerror if other grpc error was raised
    341 
    +        if not self.action_url:
    342 
    +            return None
    343 
    +        url = urlparse(self.action_url)
    344 
    +        channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
    326 345 
             request = remote_execution_pb2.GetActionResultRequest(action_digest=action_digest)
    327 346 
             stub = remote_execution_pb2_grpc.ActionCacheStub(channel)
    328 347 
             try:
    329 
    -            return stub.GetActionResult(request)
    348 
    +            result = stub.GetActionResult(request)
    330 349 
             except grpc.RpcError as e:
    331 350 
                 if e.code() != grpc.StatusCode.NOT_FOUND:
    332 351 
                     raise SandboxError("Failed to query action cache: {} ({})"
    333 352 
                                        .format(e.code(), e.details()))
    334 353 
                 else:
    335 354 
                     return None
    355 
    +        else:
    356 
    +            self.info("Action result found in action cache")
    357 
    +            return result
    336 358
    337 359 
         def _create_command(self, command, working_directory, environment):
    338 360 
             # Creates a command proto

  • doc/source/format_project.rst
    ... ... @@ -238,6 +238,8 @@ using the `remote-execution` option:
    238 238 
           server-cert: server.crt
    239 239 
           client-cert: client.crt
    240 240 
           client-key: client.key
    241 
    +    action-service:
    242 
    +      url: http://bar.action.com:50052
    241 243
    242 244 
     The execution-service part of remote execution does not support encrypted
    243 245 
     connections yet, so the protocol must always be http.
    ... ... @@ -245,6 +247,10 @@ connections yet, so the protocol must always be http.
    245 247 
     storage-service specifies a remote CAS store and the parameters are the
    246 248 
     same as those used to specify an :ref:`artifact server <artifacts>`.
    247 249
    250 
    +The action-service specifies where built actions are cached, allowing
    251 
    +buildstream to check whether an action has already been executed and download it
    252 
    +if so. This is similar to the artifact cache but REAPI specified.
    253 
    +
    248 254 
     The storage service may be the same endpoint used for artifact
    249 255 
     caching. Remote execution cannot work without push access to the
    250 256 
     storage endpoint, so you must specify a client certificate and key,

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]