GNOME.org
 

[Notes] [Git][BuildStream/buildstream][jmac/remote_execution_split] 4 commits: _sandboxremote: Remove references to artifact cache.

Title: GitLab

Jim MacArthur pushed to branch jmac/remote_execution_split at BuildStream / buildstream

Commits:

10 changed files:

Changes:

  • buildstream/_artifactcache/artifactcache.py
    ... ... @@ -751,34 +751,6 @@ class ArtifactCache():
    751 751
    752 752 
             return message_digest
    753 753
    754 
    -    # verify_digest_pushed():
    755 
    -    #
    756 
    -    # Check whether the object is already on the server in which case
    757 
    -    # there is no need to upload it.
    758 
    -    #
    759 
    -    # Args:
    760 
    -    #     project (Project): The current project
    761 
    -    #     digest (Digest): The object digest.
    762 
    -    #
    763 
    -    def verify_digest_pushed(self, project, digest):
    764 
    -
    765 
    -        if self._has_push_remotes:
    766 
    -            push_remotes = [r for r in self._remotes[project] if r.spec.push]
    767 
    -        else:
    768 
    -            push_remotes = []
    769 
    -
    770 
    -        if not push_remotes:
    771 
    -            raise ArtifactError("verify_digest_pushed was called, but no remote artifact " +
    772 
    -                                "servers are configured as push remotes.")
    773 
    -
    774 
    -        pushed = False
    775 
    -
    776 
    -        for remote in push_remotes:
    777 
    -            if self.cas.verify_digest_on_remote(remote, digest):
    778 
    -                pushed = True
    779 
    -
    780 
    -        return pushed
    781 
    -
    782 754 
         # link_key():
    783 755 
         #
    784 756 
         # Add a key for an existing artifact.

  • buildstream/_project.py
    ... ... @@ -30,6 +30,7 @@ from ._profile import Topics, profile_start, profile_end
    30 30 
     from ._exceptions import LoadError, LoadErrorReason
    31 31 
     from ._options import OptionPool
    32 32 
     from ._artifactcache import ArtifactCache
    33 
    +from .sandbox import SandboxRemote
    33 34 
     from ._elementfactory import ElementFactory
    34 35 
     from ._sourcefactory import SourceFactory
    35 36 
     from .plugin import CoreWarnings
    ... ... @@ -130,7 +131,7 @@ class Project():
    130 131 
             self._shell_host_files = []   # A list of HostMount objects
    131 132
    132 133 
             self.artifact_cache_specs = None
    133 
    -        self.remote_execution_url = None
    134 
    +        self.remote_execution_specs = None
    134 135 
             self._sandbox = None
    135 136 
             self._splits = None
    136 137
    ... ... @@ -493,9 +494,7 @@ class Project():
    493 494 
             self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
    494 495
    495 496 
             # Load remote-execution configuration for this project
    496 
    -        remote_execution = _yaml.node_get(config, Mapping, 'remote-execution')
    497 
    -        _yaml.node_validate(remote_execution, ['url'])
    498 
    -        self.remote_execution_url = _yaml.node_get(remote_execution, str, 'url')
    497 
    +        self.remote_execution_specs = SandboxRemote.specs_from_config_node(config, self.directory)
    499 498
    500 499 
             # Load sandbox environment variables
    501 500 
             self.base_environment = _yaml.node_get(config, Mapping, 'environment')

  • buildstream/data/projectconfig.yaml
    ... ... @@ -196,7 +196,4 @@ shell:
    196 196
    197 197 
       # Command to run when `bst shell` does not provide a command
    198 198 
       #
    199 
    -  command: [ 'sh', '-i' ]
    200 
    -
    201 
    -remote-execution:
    202 
    -  url: ""
    \ No newline at end of file
    199 
    +  command: [ 'sh', '-i' ]
    \ No newline at end of file

  • buildstream/element.py
    ... ... @@ -250,9 +250,9 @@ class Element(Plugin):
    250 250
    251 251 
             # Extract remote execution URL
    252 252 
             if not self.__is_junction:
    253 
    -            self.__remote_execution_url = project.remote_execution_url
    253 
    +            self.__remote_execution_specs = project.remote_execution_specs
    254 254 
             else:
    255 
    -            self.__remote_execution_url = None
    255 
    +            self.__remote_execution_specs = None
    256 256
    257 257 
             # Extract Sandbox config
    258 258 
             self.__sandbox_config = self.__extract_sandbox_config(meta)
    ... ... @@ -2125,7 +2125,7 @@ class Element(Plugin):
    2125 2125 
         # supports it.
    2126 2126 
         #
    2127 2127 
         def __use_remote_execution(self):
    2128 
    -        return self.__remote_execution_url and self.BST_VIRTUAL_DIRECTORY
    2128 
    +        return self.__remote_execution_specs and self.BST_VIRTUAL_DIRECTORY
    2129 2129
    2130 2130 
         # __sandbox():
    2131 2131 
         #
    ... ... @@ -2160,13 +2160,13 @@ class Element(Plugin):
    2160 2160 
                                         stdout=stdout,
    2161 2161 
                                         stderr=stderr,
    2162 2162 
                                         config=config,
    2163 
    -                                    server_url=self.__remote_execution_url,
    2163 
    +                                    specs=self.__remote_execution_specs,
    2164 2164 
                                         bare_directory=bare_directory,
    2165 2165 
                                         allow_real_directory=False)
    2166 2166 
                 yield sandbox
    2167 2167
    2168 2168 
             elif directory is not None and os.path.exists(directory):
    2169 
    -            if allow_remote and self.__remote_execution_url:
    2169 
    +            if allow_remote and self.__remote_execution_specs:
    2170 2170 
                     self.warn("Artifact {} is configured to use remote execution but element plugin does not support it."
    2171 2171 
                               .format(self.name), detail="Element plugin '{kind}' does not support virtual directories."
    2172 2172 
                               .format(kind=self.get_kind()), warning_token="remote-failure")

  • buildstream/sandbox/_sandboxremote.py
    ... ... @@ -19,6 +19,7 @@
    19 19 
     #        Jim MacArthur <jim macarthur codethink co uk>
    20 20
    21 21 
     import os
    22 
    +from collections import namedtuple
    22 23 
     from urllib.parse import urlparse
    23 24 
     from functools import partial
    24 25
    ... ... @@ -31,7 +32,13 @@ from .. import _signals
    31 32 
     from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    32 33 
     from .._protos.google.rpc import code_pb2
    33 34 
     from .._exceptions import SandboxError
    35 
    +from .. import _yaml
    34 36 
     from .._protos.google.longrunning import operations_pb2, operations_pb2_grpc
    37 
    +from .._artifactcache.cascache import CASRemote, CASRemoteSpec
    38 
    +
    39 
    +
    40 
    +class RemoteExecutionSpec(namedtuple('RemoteExecutionSpec', 'exec_service storage_service')):
    41 
    +    pass
    35 42
    36 43
    37 44 
     # SandboxRemote()
    ... ... @@ -44,18 +51,70 @@ class SandboxRemote(Sandbox):
    44 51 
         def __init__(self, *args, **kwargs):
    45 52 
             super().__init__(*args, **kwargs)
    46 53
    47 
    -        url = urlparse(kwargs['server_url'])
    48 
    -        if not url.scheme or not url.hostname or not url.port:
    49 
    -            raise SandboxError("Configured remote URL '{}' does not match the expected layout. "
    50 
    -                               .format(kwargs['server_url']) +
    51 
    -                               "It should be of the form <protocol>://<domain name>:<port>.")
    52 
    -        elif url.scheme != 'http':
    53 
    -            raise SandboxError("Configured remote '{}' uses an unsupported protocol. "
    54 
    -                               "Only plain HTTP is currenlty supported (no HTTPS).")
    54 
    +        config = kwargs['specs']  # This should be a RemoteExecutionSpec
    55 
    +        if config is None:
    56 
    +            return
    57 
    +
    58 
    +        self.storage_url = config.storage_service['url']
    59 
    +        self.exec_url = config.exec_service['url']
    55 60
    56 
    -        self.server_url = '{}:{}'.format(url.hostname, url.port)
    61 
    +        self.storage_remote_spec = CASRemoteSpec(self.storage_url, push=True,
    62 
    +                                                 server_cert=config.storage_service['server-cert'],
    63 
    +                                                 client_key=config.storage_service['client-key'],
    64 
    +                                                 client_cert=config.storage_service['client-cert'])
    57 65 
             self.operation_name = None
    58 66
    67 
    +    @staticmethod
    68 
    +    def specs_from_config_node(config_node, basedir):
    69 
    +
    70 
    +        def require_node(config, keyname):
    71 
    +            val = config.get(keyname)
    72 
    +            if val is None:
    73 
    +                provenance = _yaml.node_get_provenance(remote_config, key=keyname)
    74 
    +                raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
    75 
    +                                      "{}: '{}' was not present in the remote "
    76 
    +                                      "execution configuration (remote-execution). "
    77 
    +                                      .format(str(provenance), keyname))
    78 
    +            return val
    79 
    +
    80 
    +        remote_config = config_node.get("remote-execution", None)
    81 
    +        if remote_config is None:
    82 
    +            return None
    83 
    +
    84 
    +        # Maintain some backwards compatibility with older configs, in which 'url' was the only valid key for
    85 
    +        # remote-execution.
    86 
    +
    87 
    +        tls_keys = ['client-key', 'client-cert', 'server-cert']
    88 
    +
    89 
    +        _yaml.node_validate(remote_config, ['execution-service', 'storage-service', 'url'])
    90 
    +        remote_exec_service_config = require_node(remote_config, 'execution-service')
    91 
    +        remote_exec_storage_config = require_node(remote_config, 'storage-service')
    92 
    +
    93 
    +        _yaml.node_validate(remote_exec_service_config, ['url'])
    94 
    +        _yaml.node_validate(remote_exec_storage_config, ['url'] + tls_keys)
    95 
    +
    96 
    +        if 'url' in remote_config:
    97 
    +            if 'execution-service' not in remote_config:
    98 
    +                remote_config['execution-service'] = {'url': remote_config['url']}
    99 
    +            else:
    100 
    +                provenance = _yaml.node_get_provenance(remote_config, key='url')
    101 
    +                raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
    102 
    +                                      "{}: 'url' and 'execution-service' keys were found in the remote "
    103 
    +                                      "execution configuration (remote-execution). "
    104 
    +                                      "You can only specify one of these."
    105 
    +                                      .format(str(provenance)))
    106 
    +
    107 
    +        for key in tls_keys:
    108 
    +            if key not in remote_exec_storage_config:
    109 
    +                provenance = _yaml.node_get_provenance(remote_config, key='storage-service')
    110 
    +                raise _yaml.LoadError(_yaml.LoadErrorReason.INVALID_DATA,
    111 
    +                                      "{}: The keys {} are necessary for the storage-service section of "
    112 
    +                                      "remote-execution configuration. Your config is missing '{}'."
    113 
    +                                      .format(str(provenance), tls_keys, key))
    114 
    +
    115 
    +        spec = RemoteExecutionSpec(remote_config['execution-service'], remote_config['storage-service'])
    116 
    +        return spec
    117 
    +
    59 118 
         def run_remote_command(self, command, input_root_digest, working_directory, environment):
    60 119 
             # Sends an execution request to the remote execution server.
    61 120 
             #
    ... ... @@ -73,12 +132,13 @@ class SandboxRemote(Sandbox):
    73 132 
                                                           output_directories=[self._output_directory],
    74 133 
                                                           platform=None)
    75 134 
             context = self._get_context()
    76 
    -        cascache = context.artifactcache
    135 
    +        cascache = context.get_cascache()
    136 
    +        casremote = CASRemote(self.storage_remote_spec)
    137 
    +
    77 138 
             # Upload the Command message to the remote CAS server
    78 
    -        command_digest = cascache.push_message(self._get_project(), remote_command)
    79 
    -        if not command_digest or not cascache.verify_digest_pushed(self._get_project(), command_digest):
    139 
    +        command_digest = cascache.push_message(casremote, remote_command)
    140 
    +        if not command_digest or not cascache.verify_digest_on_remote(casremote, command_digest):
    80 141 
                 raise SandboxError("Failed pushing build command to remote CAS.")
    81 
    -
    82 142 
             # Create and send the action.
    83 143 
             action = remote_execution_pb2.Action(command_digest=command_digest,
    84 144 
                                                  input_root_digest=input_root_digest,
    ... ... @@ -86,12 +146,21 @@ class SandboxRemote(Sandbox):
    86 146 
                                                  do_not_cache=False)
    87 147
    88 148 
             # Upload the Action message to the remote CAS server
    89 
    -        action_digest = cascache.push_message(self._get_project(), action)
    90 
    -        if not action_digest or not cascache.verify_digest_pushed(self._get_project(), action_digest):
    149 
    +        action_digest = cascache.push_message(casremote, action)
    150 
    +        if not action_digest or not cascache.verify_digest_on_remote(casremote, action_digest):
    91 151 
                 raise SandboxError("Failed pushing build action to remote CAS.")
    92 152
    93 153 
             # Next, try to create a communication channel to the BuildGrid server.
    94 
    -        channel = grpc.insecure_channel(self.server_url)
    154 
    +        url = urlparse(self.exec_url)
    155 
    +        if not url.port:
    156 
    +            raise SandboxError("You must supply a protocol and port number in the execution-service url, "
    157 
    +                               "for example: http://buildservice:50051.")
    158 
    +        if url.scheme == 'http':
    159 
    +            channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
    160 
    +        else:
    161 
    +            raise SandboxError("Remote execution currently only supports the 'http' protocol "
    162 
    +                               "and '{}' was supplied.".format(url.scheme))
    163 
    +
    95 164 
             stub = remote_execution_pb2_grpc.ExecutionStub(channel)
    96 165 
             request = remote_execution_pb2.ExecuteRequest(action_digest=action_digest,
    97 166 
                                                           skip_cache_lookup=False)
    ... ... @@ -117,7 +186,7 @@ class SandboxRemote(Sandbox):
    117 186 
                     status_code = e.code()
    118 187 
                     if status_code == grpc.StatusCode.UNAVAILABLE:
    119 188 
                         raise SandboxError("Failed contacting remote execution server at {}."
    120 
    -                                       .format(self.server_url))
    189 
    +                                       .format(self.exec_url))
    121 190
    122 191 
                     elif status_code in (grpc.StatusCode.INVALID_ARGUMENT,
    123 192 
                                          grpc.StatusCode.FAILED_PRECONDITION,
    ... ... @@ -188,9 +257,11 @@ class SandboxRemote(Sandbox):
    188 257 
                 raise SandboxError("Output directory structure had no digest attached.")
    189 258
    190 259 
             context = self._get_context()
    191 
    -        cascache = context.artifactcache
    260 
    +        cascache = context.get_cascache()
    261 
    +        casremote = CASRemote(self.storage_remote_spec)
    262 
    +
    192 263 
             # Now do a pull to ensure we have the necessary parts.
    193 
    -        dir_digest = cascache.pull_tree(self._get_project(), tree_digest)
    264 
    +        dir_digest = cascache.pull_tree(casremote, tree_digest)
    194 265 
             if dir_digest is None or not dir_digest.hash or not dir_digest.size_bytes:
    195 266 
                 raise SandboxError("Output directory structure pulling from remote failed.")
    196 267
    ... ... @@ -216,18 +287,23 @@ class SandboxRemote(Sandbox):
    216 287 
             # Upload sources
    217 288 
             upload_vdir = self.get_virtual_directory()
    218 289
    290 
    +        cascache = self._get_context().get_cascache()
    219 291 
             if isinstance(upload_vdir, FileBasedDirectory):
    220 292 
                 # Make a new temporary directory to put source in
    221 
    -            upload_vdir = CasBasedDirectory(self._get_context().artifactcache.cas, ref=None)
    293 
    +            upload_vdir = CasBasedDirectory(cascache, ref=None)
    222 294 
                 upload_vdir.import_files(self.get_virtual_directory()._get_underlying_directory())
    223 295
    224 296 
             upload_vdir.recalculate_hash()
    225 297
    226 
    -        context = self._get_context()
    227 
    -        cascache = context.artifactcache
    298 
    +        casremote = CASRemote(self.storage_remote_spec)
    228 299 
             # Now, push that key (without necessarily needing a ref) to the remote.
    229 
    -        cascache.push_directory(self._get_project(), upload_vdir)
    230 
    -        if not cascache.verify_digest_pushed(self._get_project(), upload_vdir.ref):
    300 
    +
    301 
    +        try:
    302 
    +            cascache.push_directory(casremote, upload_vdir)
    303 
    +        except grpc._channel._Rendezvous as e:
    304 
    +            raise SandboxError("Failed to push source directory to remote: {}".format(e)) from e
    305 
    +
    306 
    +        if not cascache.verify_digest_on_remote(casremote, upload_vdir.ref):
    231 307 
                 raise SandboxError("Failed to verify that source has been pushed to the remote artifact cache.")
    232 308
    233 309 
             # Fallback to the sandbox default settings for

  • doc/source/format_project.rst
    ... ... @@ -231,10 +231,24 @@ using the `remote-execution` option:
    231 231 
       remote-execution:
    232 232
    233 233 
         # A url defining a remote execution server
    234 
    -    url: http://buildserver.example.com:50051
    234 
    +    execution-service:
    235 
    +      url: http://buildserver.example.com:50051
    236 
    +    storage-service:
    237 
    +    - url: https://foo.com/artifacts:11002
    238 
    +      server-cert: server.crt
    239 
    +      client-cert: client.crt
    240 
    +      client-key: client.key
    241 
    +
    242 
    +The execution-server part of remote execution does not support encrypted
    243 
    +connections yet, so the protocol must always be http.
    244 
    +
    245 
    +storage-server specifies a remote CAS store and the parameters are the
    246 
    +same as those used to specify an :ref:`artifact server <artifacts>`.
    235 247
    236 
    -The url should contain a hostname and port separated by ':'. Only plain HTTP is
    237 
    -currently suported (no HTTPS).
    248 
    +The storage server may be the same server used for artifact
    249 
    +caching. Remote execution cannot work without push access to the
    250 
    +storage server, so you must specify a client certificate and key, and
    251 
    +a server certificate.
    238 252
    239 253 
     The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis.
    240 254

  • tests/sandboxes/remote-exec-config.py
    1 
    +import pytest
    2 
    +
    3 
    +import itertools
    4 
    +import os
    5 
    +
    6 
    +from buildstream import _yaml
    7 
    +from buildstream._exceptions import ErrorDomain, LoadErrorReason
    8 
    +
    9 
    +from tests.testutils.runcli import cli
    10 
    +
    11 
    +DATA_DIR = os.path.join(
    12 
    +    os.path.dirname(os.path.realpath(__file__)),
    13 
    +    "remote-exec-config"
    14 
    +)
    15 
    +
    16 
    +# Tests that we get a useful error message when supplying invalid
    17 
    +# remote execution configurations.
    18 
    +
    19 
    +
    20 
    +# Assert that if both 'url' (the old style) and 'execution-service' (the new style)
    21 
    +# are used at once, a LoadError results.
    22 
    +@pytest.mark.datafiles(DATA_DIR)
    23 
    +def test_old_and_new_configs(cli, datafiles):
    24 
    +    project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
    25 
    +
    26 
    +    project_conf = {
    27 
    +        'name': 'test',
    28 
    +
    29 
    +        'remote-execution': {
    30 
    +            'url': 'https://cache.example.com:12345',
    31 
    +            'execution-service': {
    32 
    +                'url': 'http://localhost:8088'
    33 
    +            },
    34 
    +            'storage-service': {
    35 
    +                'url': 'http://charactron:11001',
    36 
    +            }
    37 
    +        }
    38 
    +    }
    39 
    +    project_conf_file = os.path.join(project, 'project.conf')
    40 
    +    _yaml.dump(project_conf, project_conf_file)
    41 
    +
    42 
    +    # Use `pull` here to ensure we try to initialize the remotes, triggering the error
    43 
    +    #
    44 
    +    # This does not happen for a simple `bst show`.
    45 
    +    result = cli.run(project=project, args=['pull', 'element.bst'])
    46 
    +    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one")
    47 
    +
    48 
    +
    49 
    +# Assert that if either the client key or client cert is specified
    50 
    +# without specifying its counterpart, we get a comprehensive LoadError
    51 
    +# instead of an unhandled exception.
    52 
    +@pytest.mark.datafiles(DATA_DIR)
    53 
    +@pytest.mark.parametrize('config_key, config_value', [
    54 
    +    ('client-cert', 'client.crt'),
    55 
    +    ('client-key', 'client.key')
    56 
    +])
    57 
    +def test_missing_certs(cli, datafiles, config_key, config_value):
    58 
    +    project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
    59 
    +
    60 
    +    project_conf = {
    61 
    +        'name': 'test',
    62 
    +
    63 
    +        'remote-execution': {
    64 
    +            'execution-service': {
    65 
    +                'url': 'http://localhost:8088'
    66 
    +            },
    67 
    +            'storage-service': {
    68 
    +                'url': 'http://charactron:11001',
    69 
    +                config_key: config_value,
    70 
    +            }
    71 
    +        }
    72 
    +    }
    73 
    +    project_conf_file = os.path.join(project, 'project.conf')
    74 
    +    _yaml.dump(project_conf, project_conf_file)
    75 
    +
    76 
    +    # Use `pull` here to ensure we try to initialize the remotes, triggering the error
    77 
    +    #
    78 
    +    # This does not happen for a simple `bst show`.
    79 
    +    result = cli.run(project=project, args=['show', 'element.bst'])
    80 
    +    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "Your config is missing")
    81 
    +
    82 
    +
    83 
    +# Assert that if incomplete information is supplied we get a sensible error message.
    84 
    +@pytest.mark.datafiles(DATA_DIR)
    85 
    +def test_empty_config(cli, datafiles):
    86 
    +    project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
    87 
    +
    88 
    +    project_conf = {
    89 
    +        'name': 'test',
    90 
    +
    91 
    +        'remote-execution': {
    92 
    +        }
    93 
    +    }
    94 
    +    project_conf_file = os.path.join(project, 'project.conf')
    95 
    +    _yaml.dump(project_conf, project_conf_file)
    96 
    +
    97 
    +    # Use `pull` here to ensure we try to initialize the remotes, triggering the error
    98 
    +    #
    99 
    +    # This does not happen for a simple `bst show`.
    100 
    +    result = cli.run(project=project, args=['pull', 'element.bst'])
    101 
    +    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one")

  • tests/sandboxes/remote-exec-config/missing-certs/certificates/client.crt

  • tests/sandboxes/remote-exec-config/missing-certs/certificates/client.key

  • tests/sandboxes/remote-exec-config/missing-certs/element.bst
    1 
    +kind: autotools

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]