[Notes] [Git][BuildStream/buildstream][master] 8 commits: artifactcache: Move ArtifactCacheSpec code into CASRemoteSpec.



Title: GitLab

Jim MacArthur pushed to branch master at BuildStream / buildstream

Commits:

12 changed files:

Changes:

  • buildstream/_artifactcache/artifactcache.py
    ... ... @@ -21,7 +21,6 @@ import multiprocessing
    21 21
     import os
    
    22 22
     import signal
    
    23 23
     import string
    
    24
    -from collections import namedtuple
    
    25 24
     from collections.abc import Mapping
    
    26 25
     
    
    27 26
     from ..types import _KeyStrength
    
    ... ... @@ -31,7 +30,7 @@ from .. import _signals
    31 30
     from .. import utils
    
    32 31
     from .. import _yaml
    
    33 32
     
    
    34
    -from .cascache import CASCache, CASRemote
    
    33
    +from .cascache import CASRemote, CASRemoteSpec
    
    35 34
     
    
    36 35
     
    
    37 36
     CACHE_SIZE_FILE = "cache_size"
    
    ... ... @@ -45,48 +44,8 @@ CACHE_SIZE_FILE = "cache_size"
    45 44
     #     push (bool): Whether we should attempt to push artifacts to this cache,
    
    46 45
     #                  in addition to pulling from it.
    
    47 46
     #
    
    48
    -class ArtifactCacheSpec(namedtuple('ArtifactCacheSpec', 'url push server_cert client_key client_cert')):
    
    49
    -
    
    50
    -    # _new_from_config_node
    
    51
    -    #
    
    52
    -    # Creates an ArtifactCacheSpec() from a YAML loaded node
    
    53
    -    #
    
    54
    -    @staticmethod
    
    55
    -    def _new_from_config_node(spec_node, basedir=None):
    
    56
    -        _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert'])
    
    57
    -        url = _yaml.node_get(spec_node, str, 'url')
    
    58
    -        push = _yaml.node_get(spec_node, bool, 'push', default_value=False)
    
    59
    -        if not url:
    
    60
    -            provenance = _yaml.node_get_provenance(spec_node, 'url')
    
    61
    -            raise LoadError(LoadErrorReason.INVALID_DATA,
    
    62
    -                            "{}: empty artifact cache URL".format(provenance))
    
    63
    -
    
    64
    -        server_cert = _yaml.node_get(spec_node, str, 'server-cert', default_value=None)
    
    65
    -        if server_cert and basedir:
    
    66
    -            server_cert = os.path.join(basedir, server_cert)
    
    67
    -
    
    68
    -        client_key = _yaml.node_get(spec_node, str, 'client-key', default_value=None)
    
    69
    -        if client_key and basedir:
    
    70
    -            client_key = os.path.join(basedir, client_key)
    
    71
    -
    
    72
    -        client_cert = _yaml.node_get(spec_node, str, 'client-cert', default_value=None)
    
    73
    -        if client_cert and basedir:
    
    74
    -            client_cert = os.path.join(basedir, client_cert)
    
    75
    -
    
    76
    -        if client_key and not client_cert:
    
    77
    -            provenance = _yaml.node_get_provenance(spec_node, 'client-key')
    
    78
    -            raise LoadError(LoadErrorReason.INVALID_DATA,
    
    79
    -                            "{}: 'client-key' was specified without 'client-cert'".format(provenance))
    
    80
    -
    
    81
    -        if client_cert and not client_key:
    
    82
    -            provenance = _yaml.node_get_provenance(spec_node, 'client-cert')
    
    83
    -            raise LoadError(LoadErrorReason.INVALID_DATA,
    
    84
    -                            "{}: 'client-cert' was specified without 'client-key'".format(provenance))
    
    85
    -
    
    86
    -        return ArtifactCacheSpec(url, push, server_cert, client_key, client_cert)
    
    87
    -
    
    88
    -
    
    89
    -ArtifactCacheSpec.__new__.__defaults__ = (None, None, None)
    
    47
    +class ArtifactCacheSpec(CASRemoteSpec):
    
    48
    +    pass
    
    90 49
     
    
    91 50
     
    
    92 51
     # An ArtifactCache manages artifacts.
    
    ... ... @@ -99,7 +58,7 @@ class ArtifactCache():
    99 58
             self.context = context
    
    100 59
             self.extractdir = os.path.join(context.artifactdir, 'extract')
    
    101 60
     
    
    102
    -        self.cas = CASCache(context.artifactdir)
    
    61
    +        self.cas = context.get_cascache()
    
    103 62
     
    
    104 63
             self.global_remote_specs = []
    
    105 64
             self.project_remote_specs = {}
    
    ... ... @@ -792,34 +751,6 @@ class ArtifactCache():
    792 751
     
    
    793 752
             return message_digest
    
    794 753
     
    
    795
    -    # verify_digest_pushed():
    
    796
    -    #
    
    797
    -    # Check whether the object is already on the server in which case
    
    798
    -    # there is no need to upload it.
    
    799
    -    #
    
    800
    -    # Args:
    
    801
    -    #     project (Project): The current project
    
    802
    -    #     digest (Digest): The object digest.
    
    803
    -    #
    
    804
    -    def verify_digest_pushed(self, project, digest):
    
    805
    -
    
    806
    -        if self._has_push_remotes:
    
    807
    -            push_remotes = [r for r in self._remotes[project] if r.spec.push]
    
    808
    -        else:
    
    809
    -            push_remotes = []
    
    810
    -
    
    811
    -        if not push_remotes:
    
    812
    -            raise ArtifactError("verify_digest_pushed was called, but no remote artifact " +
    
    813
    -                                "servers are configured as push remotes.")
    
    814
    -
    
    815
    -        pushed = False
    
    816
    -
    
    817
    -        for remote in push_remotes:
    
    818
    -            if self.cas.verify_digest_on_remote(remote, digest):
    
    819
    -                pushed = True
    
    820
    -
    
    821
    -        return pushed
    
    822
    -
    
    823 754
         # link_key():
    
    824 755
         #
    
    825 756
         # Add a key for an existing artifact.
    

  • buildstream/_artifactcache/cascache.py
    ... ... @@ -17,6 +17,7 @@
    17 17
     #  Authors:
    
    18 18
     #        Jürg Billeter <juerg billeter codethink co uk>
    
    19 19
     
    
    20
    +from collections import namedtuple
    
    20 21
     import hashlib
    
    21 22
     import itertools
    
    22 23
     import io
    
    ... ... @@ -34,7 +35,8 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remo
    34 35
     from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
    
    35 36
     
    
    36 37
     from .. import utils
    
    37
    -from .._exceptions import CASError
    
    38
    +from .._exceptions import CASError, LoadError, LoadErrorReason
    
    39
    +from .. import _yaml
    
    38 40
     
    
    39 41
     
    
    40 42
     # The default limit for gRPC messages is 4 MiB.
    
    ... ... @@ -42,6 +44,50 @@ from .._exceptions import CASError
    42 44
     _MAX_PAYLOAD_BYTES = 1024 * 1024
    
    43 45
     
    
    44 46
     
    
    47
    +class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key client_cert')):
    
    48
    +
    
    49
    +    # _new_from_config_node
    
    50
    +    #
    
    51
    +    # Creates an CASRemoteSpec() from a YAML loaded node
    
    52
    +    #
    
    53
    +    @staticmethod
    
    54
    +    def _new_from_config_node(spec_node, basedir=None):
    
    55
    +        _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert'])
    
    56
    +        url = _yaml.node_get(spec_node, str, 'url')
    
    57
    +        push = _yaml.node_get(spec_node, bool, 'push', default_value=False)
    
    58
    +        if not url:
    
    59
    +            provenance = _yaml.node_get_provenance(spec_node, 'url')
    
    60
    +            raise LoadError(LoadErrorReason.INVALID_DATA,
    
    61
    +                            "{}: empty artifact cache URL".format(provenance))
    
    62
    +
    
    63
    +        server_cert = _yaml.node_get(spec_node, str, 'server-cert', default_value=None)
    
    64
    +        if server_cert and basedir:
    
    65
    +            server_cert = os.path.join(basedir, server_cert)
    
    66
    +
    
    67
    +        client_key = _yaml.node_get(spec_node, str, 'client-key', default_value=None)
    
    68
    +        if client_key and basedir:
    
    69
    +            client_key = os.path.join(basedir, client_key)
    
    70
    +
    
    71
    +        client_cert = _yaml.node_get(spec_node, str, 'client-cert', default_value=None)
    
    72
    +        if client_cert and basedir:
    
    73
    +            client_cert = os.path.join(basedir, client_cert)
    
    74
    +
    
    75
    +        if client_key and not client_cert:
    
    76
    +            provenance = _yaml.node_get_provenance(spec_node, 'client-key')
    
    77
    +            raise LoadError(LoadErrorReason.INVALID_DATA,
    
    78
    +                            "{}: 'client-key' was specified without 'client-cert'".format(provenance))
    
    79
    +
    
    80
    +        if client_cert and not client_key:
    
    81
    +            provenance = _yaml.node_get_provenance(spec_node, 'client-cert')
    
    82
    +            raise LoadError(LoadErrorReason.INVALID_DATA,
    
    83
    +                            "{}: 'client-cert' was specified without 'client-key'".format(provenance))
    
    84
    +
    
    85
    +        return CASRemoteSpec(url, push, server_cert, client_key, client_cert)
    
    86
    +
    
    87
    +
    
    88
    +CASRemoteSpec.__new__.__defaults__ = (None, None, None)
    
    89
    +
    
    90
    +
    
    45 91
     # A CASCache manages a CAS repository as specified in the Remote Execution API.
    
    46 92
     #
    
    47 93
     # Args:
    

  • buildstream/_context.py
    ... ... @@ -31,6 +31,7 @@ from ._exceptions import LoadError, LoadErrorReason, BstError
    31 31
     from ._message import Message, MessageType
    
    32 32
     from ._profile import Topics, profile_start, profile_end
    
    33 33
     from ._artifactcache import ArtifactCache
    
    34
    +from ._artifactcache.cascache import CASCache
    
    34 35
     from ._workspaces import Workspaces
    
    35 36
     from .plugin import _plugin_lookup
    
    36 37
     
    
    ... ... @@ -141,6 +142,7 @@ class Context():
    141 142
             self._workspaces = None
    
    142 143
             self._log_handle = None
    
    143 144
             self._log_filename = None
    
    145
    +        self._cascache = None
    
    144 146
     
    
    145 147
         # load()
    
    146 148
         #
    
    ... ... @@ -620,6 +622,11 @@ class Context():
    620 622
             if not os.environ.get('XDG_DATA_HOME'):
    
    621 623
                 os.environ['XDG_DATA_HOME'] = os.path.expanduser('~/.local/share')
    
    622 624
     
    
    625
    +    def get_cascache(self):
    
    626
    +        if self._cascache is None:
    
    627
    +            self._cascache = CASCache(self.artifactdir)
    
    628
    +        return self._cascache
    
    629
    +
    
    623 630
     
    
    624 631
     # _node_get_option_str()
    
    625 632
     #
    

  • buildstream/_project.py
    ... ... @@ -30,6 +30,7 @@ from ._profile import Topics, profile_start, profile_end
    30 30
     from ._exceptions import LoadError, LoadErrorReason
    
    31 31
     from ._options import OptionPool
    
    32 32
     from ._artifactcache import ArtifactCache
    
    33
    +from .sandbox import SandboxRemote
    
    33 34
     from ._elementfactory import ElementFactory
    
    34 35
     from ._sourcefactory import SourceFactory
    
    35 36
     from .plugin import CoreWarnings
    
    ... ... @@ -130,7 +131,7 @@ class Project():
    130 131
             self._shell_host_files = []   # A list of HostMount objects
    
    131 132
     
    
    132 133
             self.artifact_cache_specs = None
    
    133
    -        self.remote_execution_url = None
    
    134
    +        self.remote_execution_specs = None
    
    134 135
             self._sandbox = None
    
    135 136
             self._splits = None
    
    136 137
     
    
    ... ... @@ -493,9 +494,7 @@ class Project():
    493 494
             self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
    
    494 495
     
    
    495 496
             # Load remote-execution configuration for this project
    
    496
    -        remote_execution = _yaml.node_get(config, Mapping, 'remote-execution')
    
    497
    -        _yaml.node_validate(remote_execution, ['url'])
    
    498
    -        self.remote_execution_url = _yaml.node_get(remote_execution, str, 'url')
    
    497
    +        self.remote_execution_specs = SandboxRemote.specs_from_config_node(config, self.directory)
    
    499 498
     
    
    500 499
             # Load sandbox environment variables
    
    501 500
             self.base_environment = _yaml.node_get(config, Mapping, 'environment')
    

  • buildstream/data/projectconfig.yaml
    ... ... @@ -196,7 +196,4 @@ shell:
    196 196
     
    
    197 197
       # Command to run when `bst shell` does not provide a command
    
    198 198
       #
    
    199
    -  command: [ 'sh', '-i' ]
    
    200
    -
    
    201
    -remote-execution:
    
    202
    -  url: ""
    \ No newline at end of file
    199
    +  command: [ 'sh', '-i' ]
    \ No newline at end of file

  • buildstream/element.py
    ... ... @@ -255,9 +255,9 @@ class Element(Plugin):
    255 255
     
    
    256 256
             # Extract remote execution URL
    
    257 257
             if not self.__is_junction:
    
    258
    -            self.__remote_execution_url = project.remote_execution_url
    
    258
    +            self.__remote_execution_specs = project.remote_execution_specs
    
    259 259
             else:
    
    260
    -            self.__remote_execution_url = None
    
    260
    +            self.__remote_execution_specs = None
    
    261 261
     
    
    262 262
             # Extract Sandbox config
    
    263 263
             self.__sandbox_config = self.__extract_sandbox_config(meta)
    
    ... ... @@ -2171,7 +2171,7 @@ class Element(Plugin):
    2171 2171
         # supports it.
    
    2172 2172
         #
    
    2173 2173
         def __use_remote_execution(self):
    
    2174
    -        return self.__remote_execution_url and self.BST_VIRTUAL_DIRECTORY
    
    2174
    +        return self.__remote_execution_specs and self.BST_VIRTUAL_DIRECTORY
    
    2175 2175
     
    
    2176 2176
         # __sandbox():
    
    2177 2177
         #
    
    ... ... @@ -2207,13 +2207,13 @@ class Element(Plugin):
    2207 2207
                                         stdout=stdout,
    
    2208 2208
                                         stderr=stderr,
    
    2209 2209
                                         config=config,
    
    2210
    -                                    server_url=self.__remote_execution_url,
    
    2210
    +                                    specs=self.__remote_execution_specs,
    
    2211 2211
                                         bare_directory=bare_directory,
    
    2212 2212
                                         allow_real_directory=False)
    
    2213 2213
                 yield sandbox
    
    2214 2214
     
    
    2215 2215
             elif directory is not None and os.path.exists(directory):
    
    2216
    -            if allow_remote and self.__remote_execution_url:
    
    2216
    +            if allow_remote and self.__remote_execution_specs:
    
    2217 2217
                     self.warn("Artifact {} is configured to use remote execution but element plugin does not support it."
    
    2218 2218
                               .format(self.name), detail="Element plugin '{kind}' does not support virtual directories."
    
    2219 2219
                               .format(kind=self.get_kind()), warning_token="remote-failure")
    

  • buildstream/sandbox/_sandboxremote.py
    ... ... @@ -20,6 +20,7 @@
    20 20
     
    
    21 21
     import os
    
    22 22
     import shlex
    
    23
    +from collections import namedtuple
    
    23 24
     from urllib.parse import urlparse
    
    24 25
     from functools import partial
    
    25 26
     
    
    ... ... @@ -33,7 +34,13 @@ from .. import _signals
    33 34
     from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    
    34 35
     from .._protos.google.rpc import code_pb2
    
    35 36
     from .._exceptions import SandboxError
    
    37
    +from .. import _yaml
    
    36 38
     from .._protos.google.longrunning import operations_pb2, operations_pb2_grpc
    
    39
    +from .._artifactcache.cascache import CASRemote, CASRemoteSpec
    
    40
    +
    
    41
    +
    
    42
    +class RemoteExecutionSpec(namedtuple('RemoteExecutionSpec', 'exec_service storage_service')):
    
    43
    +    pass
    
    37 44
     
    
    38 45
     
    
    39 46
     # SandboxRemote()
    
    ... ... @@ -46,18 +53,70 @@ class SandboxRemote(Sandbox):
    46 53
         def __init__(self, *args, **kwargs):
    
    47 54
             super().__init__(*args, **kwargs)
    
    48 55
     
    
    49
    -        url = urlparse(kwargs['server_url'])
    
    50
    -        if not url.scheme or not url.hostname or not url.port:
    
    51
    -            raise SandboxError("Configured remote URL '{}' does not match the expected layout. "
    
    52
    -                               .format(kwargs['server_url']) +
    
    53
    -                               "It should be of the form <protocol>://<domain name>:<port>.")
    
    54
    -        elif url.scheme != 'http':
    
    55
    -            raise SandboxError("Configured remote '{}' uses an unsupported protocol. "
    
    56
    -                               "Only plain HTTP is currenlty supported (no HTTPS).")
    
    56
    +        config = kwargs['specs']  # This should be a RemoteExecutionSpec
    
    57
    +        if config is None:
    
    58
    +            return
    
    59
    +
    
    60
    +        self.storage_url = config.storage_service['url']
    
    61
    +        self.exec_url = config.exec_service['url']
    
    57 62
     
    
    58
    -        self.server_url = '{}:{}'.format(url.hostname, url.port)
    
    63
    +        self.storage_remote_spec = CASRemoteSpec(self.storage_url, push=True,
    
    64
    +                                                 server_cert=config.storage_service['server-cert'],
    
    65
    +                                                 client_key=config.storage_service['client-key'],
    
    66
    +                                                 client_cert=config.storage_service['client-cert'])
    
    59 67
             self.operation_name = None
    
    60 68
     
    
    69
    +    @staticmethod
    
    70
    +    def specs_from_config_node(config_node, basedir):
    
    71
    +
    
    72
    +        def require_node(config, keyname):
    
    73
    +            val = config.get(keyname)
    
    74
    +            if val is None:
    
    75
    +                provenance = _yaml.node_get_provenance(remote_config, key=keyname)
    
    76
    +                raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
    
    77
    +                                      "{}: '{}' was not present in the remote "
    
    78
    +                                      "execution configuration (remote-execution). "
    
    79
    +                                      .format(str(provenance), keyname))
    
    80
    +            return val
    
    81
    +
    
    82
    +        remote_config = config_node.get("remote-execution", None)
    
    83
    +        if remote_config is None:
    
    84
    +            return None
    
    85
    +
    
    86
    +        # Maintain some backwards compatibility with older configs, in which 'url' was the only valid key for
    
    87
    +        # remote-execution.
    
    88
    +
    
    89
    +        tls_keys = ['client-key', 'client-cert', 'server-cert']
    
    90
    +
    
    91
    +        _yaml.node_validate(remote_config, ['execution-service', 'storage-service', 'url'])
    
    92
    +        remote_exec_service_config = require_node(remote_config, 'execution-service')
    
    93
    +        remote_exec_storage_config = require_node(remote_config, 'storage-service')
    
    94
    +
    
    95
    +        _yaml.node_validate(remote_exec_service_config, ['url'])
    
    96
    +        _yaml.node_validate(remote_exec_storage_config, ['url'] + tls_keys)
    
    97
    +
    
    98
    +        if 'url' in remote_config:
    
    99
    +            if 'execution-service' not in remote_config:
    
    100
    +                remote_config['execution-service'] = {'url': remote_config['url']}
    
    101
    +            else:
    
    102
    +                provenance = _yaml.node_get_provenance(remote_config, key='url')
    
    103
    +                raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
    
    104
    +                                      "{}: 'url' and 'execution-service' keys were found in the remote "
    
    105
    +                                      "execution configuration (remote-execution). "
    
    106
    +                                      "You can only specify one of these."
    
    107
    +                                      .format(str(provenance)))
    
    108
    +
    
    109
    +        for key in tls_keys:
    
    110
    +            if key not in remote_exec_storage_config:
    
    111
    +                provenance = _yaml.node_get_provenance(remote_config, key='storage-service')
    
    112
    +                raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
    
    113
    +                                      "{}: The keys {} are necessary for the storage-service section of "
    
    114
    +                                      "remote-execution configuration. Your config is missing '{}'."
    
    115
    +                                      .format(str(provenance), tls_keys, key))
    
    116
    +
    
    117
    +        spec = RemoteExecutionSpec(remote_config['execution-service'], remote_config['storage-service'])
    
    118
    +        return spec
    
    119
    +
    
    61 120
         def run_remote_command(self, command, input_root_digest, working_directory, environment):
    
    62 121
             # Sends an execution request to the remote execution server.
    
    63 122
             #
    
    ... ... @@ -75,12 +134,13 @@ class SandboxRemote(Sandbox):
    75 134
                                                           output_directories=[self._output_directory],
    
    76 135
                                                           platform=None)
    
    77 136
             context = self._get_context()
    
    78
    -        cascache = context.artifactcache
    
    137
    +        cascache = context.get_cascache()
    
    138
    +        casremote = CASRemote(self.storage_remote_spec)
    
    139
    +
    
    79 140
             # Upload the Command message to the remote CAS server
    
    80
    -        command_digest = cascache.push_message(self._get_project(), remote_command)
    
    81
    -        if not command_digest or not cascache.verify_digest_pushed(self._get_project(), command_digest):
    
    141
    +        command_digest = cascache.push_message(casremote, remote_command)
    
    142
    +        if not command_digest or not cascache.verify_digest_on_remote(casremote, command_digest):
    
    82 143
                 raise SandboxError("Failed pushing build command to remote CAS.")
    
    83
    -
    
    84 144
             # Create and send the action.
    
    85 145
             action = remote_execution_pb2.Action(command_digest=command_digest,
    
    86 146
                                                  input_root_digest=input_root_digest,
    
    ... ... @@ -88,12 +148,21 @@ class SandboxRemote(Sandbox):
    88 148
                                                  do_not_cache=False)
    
    89 149
     
    
    90 150
             # Upload the Action message to the remote CAS server
    
    91
    -        action_digest = cascache.push_message(self._get_project(), action)
    
    92
    -        if not action_digest or not cascache.verify_digest_pushed(self._get_project(), action_digest):
    
    151
    +        action_digest = cascache.push_message(casremote, action)
    
    152
    +        if not action_digest or not cascache.verify_digest_on_remote(casremote, action_digest):
    
    93 153
                 raise SandboxError("Failed pushing build action to remote CAS.")
    
    94 154
     
    
    95 155
             # Next, try to create a communication channel to the BuildGrid server.
    
    96
    -        channel = grpc.insecure_channel(self.server_url)
    
    156
    +        url = urlparse(self.exec_url)
    
    157
    +        if not url.port:
    
    158
    +            raise SandboxError("You must supply a protocol and port number in the execution-service url, "
    
    159
    +                               "for example: http://buildservice:50051.")
    
    160
    +        if url.scheme == 'http':
    
    161
    +            channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
    
    162
    +        else:
    
    163
    +            raise SandboxError("Remote execution currently only supports the 'http' protocol "
    
    164
    +                               "and '{}' was supplied.".format(url.scheme))
    
    165
    +
    
    97 166
             stub = remote_execution_pb2_grpc.ExecutionStub(channel)
    
    98 167
             request = remote_execution_pb2.ExecuteRequest(action_digest=action_digest,
    
    99 168
                                                           skip_cache_lookup=False)
    
    ... ... @@ -119,7 +188,7 @@ class SandboxRemote(Sandbox):
    119 188
                     status_code = e.code()
    
    120 189
                     if status_code == grpc.StatusCode.UNAVAILABLE:
    
    121 190
                         raise SandboxError("Failed contacting remote execution server at {}."
    
    122
    -                                       .format(self.server_url))
    
    191
    +                                       .format(self.exec_url))
    
    123 192
     
    
    124 193
                     elif status_code in (grpc.StatusCode.INVALID_ARGUMENT,
    
    125 194
                                          grpc.StatusCode.FAILED_PRECONDITION,
    
    ... ... @@ -190,9 +259,11 @@ class SandboxRemote(Sandbox):
    190 259
                 raise SandboxError("Output directory structure had no digest attached.")
    
    191 260
     
    
    192 261
             context = self._get_context()
    
    193
    -        cascache = context.artifactcache
    
    262
    +        cascache = context.get_cascache()
    
    263
    +        casremote = CASRemote(self.storage_remote_spec)
    
    264
    +
    
    194 265
             # Now do a pull to ensure we have the necessary parts.
    
    195
    -        dir_digest = cascache.pull_tree(self._get_project(), tree_digest)
    
    266
    +        dir_digest = cascache.pull_tree(casremote, tree_digest)
    
    196 267
             if dir_digest is None or not dir_digest.hash or not dir_digest.size_bytes:
    
    197 268
                 raise SandboxError("Output directory structure pulling from remote failed.")
    
    198 269
     
    
    ... ... @@ -218,18 +289,23 @@ class SandboxRemote(Sandbox):
    218 289
             # Upload sources
    
    219 290
             upload_vdir = self.get_virtual_directory()
    
    220 291
     
    
    292
    +        cascache = self._get_context().get_cascache()
    
    221 293
             if isinstance(upload_vdir, FileBasedDirectory):
    
    222 294
                 # Make a new temporary directory to put source in
    
    223
    -            upload_vdir = CasBasedDirectory(self._get_context().artifactcache.cas, ref=None)
    
    295
    +            upload_vdir = CasBasedDirectory(cascache, ref=None)
    
    224 296
                 upload_vdir.import_files(self.get_virtual_directory()._get_underlying_directory())
    
    225 297
     
    
    226 298
             upload_vdir.recalculate_hash()
    
    227 299
     
    
    228
    -        context = self._get_context()
    
    229
    -        cascache = context.artifactcache
    
    300
    +        casremote = CASRemote(self.storage_remote_spec)
    
    230 301
             # Now, push that key (without necessarily needing a ref) to the remote.
    
    231
    -        cascache.push_directory(self._get_project(), upload_vdir)
    
    232
    -        if not cascache.verify_digest_pushed(self._get_project(), upload_vdir.ref):
    
    302
    +
    
    303
    +        try:
    
    304
    +            cascache.push_directory(casremote, upload_vdir)
    
    305
    +        except grpc.RpcError as e:
    
    306
    +            raise SandboxError("Failed to push source directory to remote: {}".format(e)) from e
    
    307
    +
    
    308
    +        if not cascache.verify_digest_on_remote(casremote, upload_vdir.ref):
    
    233 309
                 raise SandboxError("Failed to verify that source has been pushed to the remote artifact cache.")
    
    234 310
     
    
    235 311
             # Now transmit the command to execute
    

  • doc/source/format_project.rst
    ... ... @@ -201,10 +201,10 @@ with an artifact share.
    201 201
       #
    
    202 202
       artifacts:
    
    203 203
         # A remote cache from which to download prebuilt artifacts
    
    204
    -    - url: https://foo.com/artifacts:11001
    
    204
    +    - url: https://foo.com:11001
    
    205 205
           server.cert: server.crt
    
    206 206
         # A remote cache from which to upload/download built/prebuilt artifacts
    
    207
    -    - url: https://foo.com/artifacts:11002
    
    207
    +    - url: https://foo.com:11002
    
    208 208
           server-cert: server.crt
    
    209 209
           client-cert: client.crt
    
    210 210
           client-key: client.key
    
    ... ... @@ -231,10 +231,24 @@ using the `remote-execution` option:
    231 231
       remote-execution:
    
    232 232
     
    
    233 233
         # A url defining a remote execution server
    
    234
    -    url: http://buildserver.example.com:50051
    
    234
    +    execution-service:
    
    235
    +      url: http://buildserver.example.com:50051
    
    236
    +    storage-service:
    
    237
    +    - url: https://foo.com:11002/
    
    238
    +      server-cert: server.crt
    
    239
    +      client-cert: client.crt
    
    240
    +      client-key: client.key
    
    241
    +
    
    242
    +The execution-service part of remote execution does not support encrypted
    
    243
    +connections yet, so the protocol must always be http.
    
    244
    +
    
    245
    +storage-service specifies a remote CAS store and the parameters are the
    
    246
    +same as those used to specify an :ref:`artifact server <artifacts>`.
    
    235 247
     
    
    236
    -The url should contain a hostname and port separated by ':'. Only plain HTTP is
    
    237
    -currently suported (no HTTPS).
    
    248
    +The storage service may be the same endpoint used for artifact
    
    249
    +caching. Remote execution cannot work without push access to the
    
    250
    +storage endpoint, so you must specify a client certificate and key,
    
    251
    +and a server certificate.
    
    238 252
     
    
    239 253
     The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis.
    
    240 254
     
    

  • tests/sandboxes/remote-exec-config.py
    1
    +import pytest
    
    2
    +
    
    3
    +import itertools
    
    4
    +import os
    
    5
    +
    
    6
    +from buildstream import _yaml
    
    7
    +from buildstream._exceptions import ErrorDomain, LoadErrorReason
    
    8
    +
    
    9
    +from tests.testutils.runcli import cli
    
    10
    +
    
    11
    +DATA_DIR = os.path.join(
    
    12
    +    os.path.dirname(os.path.realpath(__file__)),
    
    13
    +    "remote-exec-config"
    
    14
    +)
    
    15
    +
    
    16
    +# Tests that we get a useful error message when supplying invalid
    
    17
    +# remote execution configurations.
    
    18
    +
    
    19
    +
    
    20
    +# Assert that if both 'url' (the old style) and 'execution-service' (the new style)
    
    21
    +# are used at once, a LoadError results.
    
    22
    +@pytest.mark.datafiles(DATA_DIR)
    
    23
    +def test_old_and_new_configs(cli, datafiles):
    
    24
    +    project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
    
    25
    +
    
    26
    +    project_conf = {
    
    27
    +        'name': 'test',
    
    28
    +
    
    29
    +        'remote-execution': {
    
    30
    +            'url': 'https://cache.example.com:12345',
    
    31
    +            'execution-service': {
    
    32
    +                'url': 'http://localhost:8088'
    
    33
    +            },
    
    34
    +            'storage-service': {
    
    35
    +                'url': 'http://charactron:11001',
    
    36
    +            }
    
    37
    +        }
    
    38
    +    }
    
    39
    +    project_conf_file = os.path.join(project, 'project.conf')
    
    40
    +    _yaml.dump(project_conf, project_conf_file)
    
    41
    +
    
    42
    +    # Use `pull` here to ensure we try to initialize the remotes, triggering the error
    
    43
    +    #
    
    44
    +    # This does not happen for a simple `bst show`.
    
    45
    +    result = cli.run(project=project, args=['pull', 'element.bst'])
    
    46
    +    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one")
    
    47
    +
    
    48
    +
    
    49
    +# Assert that if either the client key or client cert is specified
    
    50
    +# without specifying its counterpart, we get a comprehensive LoadError
    
    51
    +# instead of an unhandled exception.
    
    52
    +@pytest.mark.datafiles(DATA_DIR)
    
    53
    +@pytest.mark.parametrize('config_key, config_value', [
    
    54
    +    ('client-cert', 'client.crt'),
    
    55
    +    ('client-key', 'client.key')
    
    56
    +])
    
    57
    +def test_missing_certs(cli, datafiles, config_key, config_value):
    
    58
    +    project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
    
    59
    +
    
    60
    +    project_conf = {
    
    61
    +        'name': 'test',
    
    62
    +
    
    63
    +        'remote-execution': {
    
    64
    +            'execution-service': {
    
    65
    +                'url': 'http://localhost:8088'
    
    66
    +            },
    
    67
    +            'storage-service': {
    
    68
    +                'url': 'http://charactron:11001',
    
    69
    +                config_key: config_value,
    
    70
    +            }
    
    71
    +        }
    
    72
    +    }
    
    73
    +    project_conf_file = os.path.join(project, 'project.conf')
    
    74
    +    _yaml.dump(project_conf, project_conf_file)
    
    75
    +
    
    76
    +    # Use `pull` here to ensure we try to initialize the remotes, triggering the error
    
    77
    +    #
    
    78
    +    # This does not happen for a simple `bst show`.
    
    79
    +    result = cli.run(project=project, args=['show', 'element.bst'])
    
    80
    +    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "Your config is missing")
    
    81
    +
    
    82
    +
    
    83
    +# Assert that if incomplete information is supplied we get a sensible error message.
    
    84
    +@pytest.mark.datafiles(DATA_DIR)
    
    85
    +def test_empty_config(cli, datafiles):
    
    86
    +    project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
    
    87
    +
    
    88
    +    project_conf = {
    
    89
    +        'name': 'test',
    
    90
    +
    
    91
    +        'remote-execution': {
    
    92
    +        }
    
    93
    +    }
    
    94
    +    project_conf_file = os.path.join(project, 'project.conf')
    
    95
    +    _yaml.dump(project_conf, project_conf_file)
    
    96
    +
    
    97
    +    # Use `pull` here to ensure we try to initialize the remotes, triggering the error
    
    98
    +    #
    
    99
    +    # This does not happen for a simple `bst show`.
    
    100
    +    result = cli.run(project=project, args=['pull', 'element.bst'])
    
    101
    +    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one")

  • tests/sandboxes/remote-exec-config/missing-certs/certificates/client.crt

  • tests/sandboxes/remote-exec-config/missing-certs/certificates/client.key

  • tests/sandboxes/remote-exec-config/missing-certs/element.bst
    1
    +kind: autotools



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]