GNOME.org
 

[Notes] [Git][BuildStream/buildstream][valentindavid/remote_execution_configuration] 3 commits: Add support for user remote execution configuration

Title: GitLab

Valentin David pushed to branch valentindavid/remote_execution_configuration at BuildStream / buildstream

Commits:

8 changed files:

Changes:

  • buildstream/_context.py
    ... ... @@ -34,6 +34,7 @@ from ._artifactcache import ArtifactCache
    34 34 
     from ._artifactcache.cascache import CASCache
    35 35 
     from ._workspaces import Workspaces, WorkspaceProjectCache, WORKSPACE_PROJECT_FILE
    36 36 
     from .plugin import _plugin_lookup
    37 
    +from .sandbox import SandboxRemote
    37 38
    38 39
    39 40 
     # Context()
    ... ... @@ -72,6 +73,9 @@ class Context():
    72 73 
             # The locations from which to push and pull prebuilt artifacts
    73 74 
             self.artifact_cache_specs = None
    74 75
    76 
    +        # The global remote execution configuration
    77 
    +        self.remote_execution_specs = None
    78 
    +
    75 79 
             # The directory to store build logs
    76 80 
             self.logdir = None
    77 81
    ... ... @@ -117,6 +121,9 @@ class Context():
    117 121 
             # Whether or not to attempt to pull build trees globally
    118 122 
             self.pull_buildtrees = None
    119 123
    124 
    +        # Which configured remote execution server to use
    125 
    +        self.remote_execution = 'any'
    126 
    +
    120 127 
             # Boolean, whether to offer to create a project for the user, if we are
    121 128 
             # invoked outside of a directory where we can resolve the project.
    122 129 
             self.prompt_auto_init = None
    ... ... @@ -191,7 +198,7 @@ class Context():
    191 198 
             _yaml.node_validate(defaults, [
    192 199 
                 'sourcedir', 'builddir', 'artifactdir', 'logdir',
    193 200 
                 'scheduler', 'artifacts', 'logging', 'projects',
    194 
    -            'cache', 'prompt', 'workspacedir',
    201 
    +            'cache', 'prompt', 'workspacedir', 'remote-execution'
    195 202 
             ])
    196 203
    197 204 
             for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
    ... ... @@ -216,6 +223,8 @@ class Context():
    216 223 
             # Load artifact share configuration
    217 224 
             self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
    218 225
    226 
    +        self.remote_execution_specs = SandboxRemote.specs_from_config_node(defaults)
    227 
    +
    219 228 
             # Load pull build trees configuration
    220 229 
             self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
    221 230
    ... ... @@ -277,7 +286,8 @@ class Context():
    277 286 
             # Shallow validation of overrides, parts of buildstream which rely
    278 287 
             # on the overrides are expected to validate elsewhere.
    279 288 
             for _, overrides in _yaml.node_items(self._project_overrides):
    280 
    -            _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror'])
    289 
    +            _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror',
    290 
    +                                            'remote-execution'])
    281 291
    282 292 
             profile_end(Topics.LOAD_CONTEXT, 'load')
    283 293

  • buildstream/_frontend/app.py
    ... ... @@ -183,7 +183,8 @@ class App():
    183 183 
                 'builders': 'sched_builders',
    184 184 
                 'pushers': 'sched_pushers',
    185 185 
                 'network_retries': 'sched_network_retries',
    186 
    -            'pull_buildtrees': 'pull_buildtrees'
    186 
    +            'pull_buildtrees': 'pull_buildtrees',
    187 
    +            'remote_execution': 'remote_execution'
    187 188 
             }
    188 189 
             for cli_option, context_attr in override_map.items():
    189 190 
                 option_value = self._main_options.get(cli_option)

  • buildstream/_frontend/cli.py
    ... ... @@ -239,6 +239,9 @@ def print_version(ctx, param, value):
    239 239 
                   help="The mirror to fetch from first, before attempting other mirrors")
    240 240 
     @click.option('--pull-buildtrees', is_flag=True, default=None,
    241 241 
                   help="Include an element's build tree when pulling remote element artifacts")
    242 
    +@click.option('--remote-execution', default='any',
    243 
    +              type=click.Choice(['any', 'user', 'project', 'none']),
    244 
    +              help='Select which remote execution server to use')
    242 245 
     @click.pass_context
    243 246 
     def cli(context, **kwargs):
    244 247 
         """Build and manipulate BuildStream projects

  • buildstream/_project.py
    ... ... @@ -507,7 +507,27 @@ class Project():
    507 507 
             self.artifact_cache_specs = ArtifactCache.specs_from_config_node(config, self.directory)
    508 508
    509 509 
             # Load remote-execution configuration for this project
    510 
    -        self.remote_execution_specs = SandboxRemote.specs_from_config_node(config, self.directory)
    510 
    +        project_specs = SandboxRemote.specs_from_config_node(config, self.directory)
    511 
    +        override_specs = SandboxRemote.specs_from_config_node(
    512 
    +            self._context.get_overrides(self.name), self.directory)
    513 
    +
    514 
    +        if self._context.remote_execution == 'any':
    515 
    +            if override_specs is not None:
    516 
    +                self.remote_execution_specs = override_specs
    517 
    +            elif project_specs is not None:
    518 
    +                self.remote_execution_specs = project_specs
    519 
    +            else:
    520 
    +                self.remote_execution_specs = self._context.remote_execution_specs
    521 
    +        elif self._context.remote_execution == 'user':
    522 
    +            if override_specs is not None:
    523 
    +                self.remote_execution_specs = override_specs
    524 
    +            else:
    525 
    +                self.remote_execution_specs = self._context.remote_execution_specs
    526 
    +        elif self._context.remote_execution == 'project':
    527 
    +            self.remote_execution_specs = project_specs
    528 
    +        else:
    529 
    +            assert self._context.remote_execution == 'none'
    530 
    +            self.remote_execution_specs = None
    511 531
    512 532 
             # Load sandbox environment variables
    513 533 
             self.base_environment = _yaml.node_get(config, Mapping, 'environment')

  • buildstream/sandbox/_sandboxremote.py
    ... ... @@ -68,10 +68,32 @@ class SandboxRemote(Sandbox):
    68 68 
             self.storage_url = config.storage_service['url']
    69 69 
             self.exec_url = config.exec_service['url']
    70 70
    71 
    +        exec_certs = {}
    72 
    +        for key in ['client-cert', 'client-key', 'server-cert']:
    73 
    +            if key in config.exec_service:
    74 
    +                with open(resolve_path(config.exec_service[key]), 'rb') as f:
    75 
    +                    exec_certs[key] = f.read()
    76 
    +
    77 
    +        self.exec_credentials = grpc.ssl_channel_credentials(
    78 
    +            root_certificates=exec_certs.get('server-cert'),
    79 
    +            private_key=exec_certs.get('client-key'),
    80 
    +            certificate_chain=exec_certs.get('client-cert'))
    81 
    +
    82 
    +        action_certs = {}
    83 
    +        for key in ['client-cert', 'client-key', 'server-cert']:
    84 
    +            if key in config.action_service:
    85 
    +                with open(resolve_path(config.exec_service[key]), 'rb') as f:
    86 
    +                    action_certs[key] = f.read()
    87 
    +
    71 88 
             if config.action_service:
    72 89 
                 self.action_url = config.action_service['url']
    90 
    +            self.action_credentials = grpc.ssl_channel_credentials(
    91 
    +                root_certificates=action_certs.get('server-cert'),
    92 
    +                private_key=action_certs.get('client-key'),
    93 
    +                certificate_chain=action_certs.get('client-cert'))
    73 94 
             else:
    74 95 
                 self.action_url = None
    96 
    +            self.action_credentials = None
    75 97
    76 98 
             self.server_instance = config.exec_service.get('instance', None)
    77 99 
             self.storage_instance = config.storage_service.get('instance', None)
    ... ... @@ -89,7 +111,7 @@ class SandboxRemote(Sandbox):
    89 111 
             self._get_context().message(Message(None, MessageType.INFO, msg))
    90 112
    91 113 
         @staticmethod
    92 
    -    def specs_from_config_node(config_node, basedir):
    114 
    +    def specs_from_config_node(config_node, basedir=None):
    93 115
    94 116 
             def require_node(config, keyname):
    95 117 
                 val = config.get(keyname)
    ... ... @@ -117,7 +139,7 @@ class SandboxRemote(Sandbox):
    117 139 
             remote_exec_storage_config = require_node(remote_config, 'storage-service')
    118 140 
             remote_exec_action_config = remote_config.get('action-cache-service', {})
    119 141
    120 
    -        _yaml.node_validate(remote_exec_service_config, ['url', 'instance'])
    142 
    +        _yaml.node_validate(remote_exec_service_config, ['url', 'instance'] + tls_keys)
    121 143 
             _yaml.node_validate(remote_exec_storage_config, ['url', 'instance'] + tls_keys)
    122 144 
             if remote_exec_action_config:
    123 145 
                 _yaml.node_validate(remote_exec_action_config, ['url'])
    ... ... @@ -304,6 +326,8 @@ class SandboxRemote(Sandbox):
    304 326 
                                    "for example: http://buildservice:50051.")
    305 327 
             if url.scheme == 'http':
    306 328 
                 channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
    329 
    +        elif url.scheme == 'https':
    330 
    +            channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.exec_credentials)
    307 331 
             else:
    308 332 
                 raise SandboxError("Remote execution currently only supports the 'http' protocol "
    309 333 
                                    "and '{}' was supplied.".format(url.scheme))
    ... ... @@ -361,11 +385,11 @@ class SandboxRemote(Sandbox):
    361 385 
             if not url.port:
    362 386 
                 raise SandboxError("You must supply a protocol and port number in the action-cache-service url, "
    363 387 
                                    "for example: http://buildservice:50051.")
    364 
    -        if not url.scheme == "http":
    365 
    -            raise SandboxError("Currently only support http for the action cache"
    366 
    -                               "and {} was supplied".format(url.scheme))
    388 
    +        if url.scheme == 'http':
    389 
    +            channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
    390 
    +        elif url.scheme == 'https':
    391 
    +            channel = grpc.secure_channel('{}:{}'.format(url.hostname, url.port), self.action_credentials)
    367 392
    368 
    -        channel = grpc.insecure_channel('{}:{}'.format(url.hostname, url.port))
    369 393 
             request = remote_execution_pb2.GetActionResultRequest(action_digest=action_digest)
    370 394 
             stub = remote_execution_pb2_grpc.ActionCacheStub(channel)
    371 395 
             try:

  • doc/source/format_project.rst
    ... ... @@ -218,6 +218,7 @@ The use of ports are required to distinguish between pull only access and
    218 218 
     push/pull access. For information regarding the server/client certificates
    219 219 
     and keys, please see: :ref:`Key pair for the server <server_authentication>`.
    220 220
    221 
    +.. _project_remote_execution:
    221 222
    222 223 
     Remote execution
    223 224 
     ~~~~~~~~~~~~~~~~
    ... ... @@ -243,9 +244,6 @@ using the `remote-execution` option:
    243 244 
         action-cache-service:
    244 245 
           url: http://bar.action.com:50052
    245 246
    246 
    -The execution-service part of remote execution does not support encrypted
    247 
    -connections yet, so the protocol must always be http.
    248 
    -
    249 247 
     storage-service specifies a remote CAS store and the parameters are the
    250 248 
     same as those used to specify an :ref:`artifact server <artifacts>`.
    251 249
    ... ... @@ -268,6 +266,9 @@ instance names.
    268 266
    269 267 
     The Remote Execution API can be found via https://github.com/bazelbuild/remote-apis.
    270 268
    269 
    +Remote execution configuration can be also provided in the `user
    270 
    +configuration <user_config_remote_execution>`.
    271 
    +
    271 272 
     .. _project_essentials_mirrors:
    272 273
    273 274 
     Mirrors

  • doc/source/using_config.rst
    ... ... @@ -100,6 +100,54 @@ pull only access and push/pull access. For information regarding this and the
    100 100 
     server/client certificates and keys, please see:
    101 101 
     :ref:`Key pair for the server <server_authentication>`.
    102 102
    103 
    +.. _user_config_remote_execution:
    104 
    +
    105 
    +Remote execution
    106 
    +~~~~~~~~~~~~~~~~
    107 
    +
    108 
    +The same configuration for :ref:`remote execution <project_remote_execution>`
    109 
    +in ``project.conf`` can be provided in the user configuation.
    110 
    +
    111 
    +There is only one remote execution configuration used per project.
    112 
    +
    113 
    +The project overrides will be taken in priority. The global
    114 
    +configuration will be used as fallback.
    115 
    +
    116 
    +1. Global remote execution fallback:
    117 
    +
    118 
    +.. code:: yaml
    119 
    +
    120 
    +  remote-execution:
    121 
    +    execution-service:
    122 
    +      url: http://execution.fallback.example.com:50051
    123 
    +      instance-name: main
    124 
    +    storage-service:
    125 
    +      url: https://storage.fallback.example.com:11002/
    126 
    +      server-cert: /keys/server.crt
    127 
    +      client-cert: /keys/client.crt
    128 
    +      client-key: /keys/client.key
    129 
    +      instance-name: main
    130 
    +    action-cache-service:
    131 
    +      url: http://action.flalback.example.com:50052
    132 
    +
    133 
    +2. Project override:
    134 
    +
    135 
    +.. code:: yaml
    136 
    +
    137 
    +  projects:
    138 
    +    some_project:
    139 
    +      remote-execution:
    140 
    +        execution-service:
    141 
    +          url: http://execution.some_project.example.com:50051
    142 
    +          instance-name: main
    143 
    +        storage-service:
    144 
    +          url: https://storage.some_project.example.com:11002/
    145 
    +          server-cert: /some_project_keys/server.crt
    146 
    +          client-cert: /some_project_keys/client.crt
    147 
    +          client-key: /some_project_keys/client.key
    148 
    +          instance-name: main
    149 
    +        action-cache-service:
    150 
    +          url: http://action.some_project.example.com:50052
    103 151
    104 152
    105 153 
     Strict build plan

  • tests/completions/completions.py
    ... ... @@ -42,6 +42,7 @@ MAIN_OPTIONS = [
    42 42 
         "--option ",
    43 43 
         "--on-error ",
    44 44 
         "--pull-buildtrees ",
    45 
    +    "--remote-execution ",
    45 46 
         "--pushers ",
    46 47 
         "--strict ",
    47 48 
         "--verbose ",

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]