GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][finn/80-remote-parser] 3 commits: Added remote parser.

Title: GitLab

finn pushed to branch finn/80-remote-parser at BuildGrid / buildgrid

Commits:

6 changed files:

Changes:

  • buildgrid/_app/commands/cmd_server.py
    ... ... @@ -53,10 +53,16 @@ def start(context, config):
    53 53 
         insecure_mode = server_settings['insecure-mode']
    54 54
    55 55 
         credentials = None
    56 
    +    credentials_settings = server_settings.get('credentials')
    56 57 
         if not insecure_mode:
    57 
    -        server_key = server_settings['tls-server-key']
    58 
    -        server_cert = server_settings['tls-server-cert']
    59 
    -        client_certs = server_settings['tls-client-certs']
    58 
    +        if not credentials_settings:
    59 
    +            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    60 
    +                       "Set `insecure-mode: false` in order to deactivate TLS encryption.\n", err=True)
    61 
    +            sys.exit(-1)
    62 
    +
    63 
    +        server_key = credentials_settings['tls-server-key']
    64 
    +        server_cert = credentials_settings['tls-server-cert']
    65 
    +        client_certs = credentials_settings['tls-client-certs']
    60 66 
             credentials = context.load_server_credentials(server_key, server_cert, client_certs)
    61 67
    62 68 
             if not credentials:

  • buildgrid/_app/settings/__init__.py

  • buildgrid/_app/settings/cas.yml
    1 
    +server:
    2 
    +  port: 50052
    3 
    +  insecure-mode: true
    4 
    +  tls-server-key: null
    5 
    +  tls-server-cert: null
    6 
    +  tls-client-certs: null
    7 
    +
    8 
    +description: |
    9 
    +  Just a CAS.
    10 
    +
    11 
    +instances:
    12 
    +  - name: main
    13 
    +    description: |
    14 
    +      The main server
    15 
    +
    16 
    +    storages:
    17 
    +        - !disk-storage &main-storage
    18 
    +          path: ~/cas/
    19 
    +
    20 
    +    services:
    21 
    +      - !cas
    22 
    +        storage: *main-storage

  • buildgrid/_app/settings/default.yml
    1 1 
     server:
    2 2 
       port: 50051
    3 
    +  insecure-mode: true
    3 4 
       tls-server-key: null
    4 5 
       tls-server-cert: null
    5 6 
       tls-client-certs: null
    6 
    -  insecure-mode: true
    7 7
    8 8 
     description: |
    9 9 
       A single default instance

  • buildgrid/_app/settings/parser.py
    ... ... @@ -14,7 +14,11 @@
    14 14
    15 15
    16 16 
     import os
    17 
    +import sys
    18 
    +from urllib.parse import urlparse
    17 19
    20 
    +import click
    21 
    +import grpc
    18 22 
     import yaml
    19 23
    20 24 
     from buildgrid.server.controller import ExecutionController
    ... ... @@ -22,9 +26,12 @@ from buildgrid.server.actioncache.storage import ActionCache
    22 26 
     from buildgrid.server.cas.instance import ByteStreamInstance, ContentAddressableStorageInstance
    23 27 
     from buildgrid.server.cas.storage.disk import DiskStorage
    24 28 
     from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
    29 
    +from buildgrid.server.cas.storage.remote import RemoteStorage
    25 30 
     from buildgrid.server.cas.storage.s3 import S3Storage
    26 31 
     from buildgrid.server.cas.storage.with_cache import WithCacheStorage
    27 32
    33 
    +from ..cli import Context
    34 
    +
    28 35
    29 36 
     class YamlFactory(yaml.YAMLObject):
    30 37 
         @classmethod
    ... ... @@ -58,6 +65,45 @@ class S3(YamlFactory):
    58 65 
             return S3Storage(bucket, endpoint_url=endpoint)
    59 66
    60 67
    68 
    +class Remote(YamlFactory):
    69 
    +
    70 
    +    yaml_tag = u'!remote-storage'
    71 
    +
    72 
    +    def __new__(cls, url, credentials=None):
    73 
    +        # TODO: Context could be passed into the parser.
    74 
    +        context = Context()
    75 
    +
    76 
    +        url = urlparse(url)
    77 
    +        remote = '{}:{}'.format(url.hostname, url.port or 50051)
    78 
    +
    79 
    +        channel = None
    80 
    +        if url.scheme == 'http':
    81 
    +            channel = grpc.insecure_channel(remote)
    82 
    +
    83 
    +        else:
    84 
    +            if not credentials:
    85 
    +                click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    86 
    +                           "Set remote url scheme to `http` in order to deactivate" +
    87 
    +                           "TLS encryption.\n", err=True)
    88 
    +                sys.exit(-1)
    89 
    +
    90 
    +            client_key = credentials['tls-client-key']
    91 
    +            client_cert = credentials['tls-client-cert']
    92 
    +            server_cert = credentials['tls-server-cert']
    93 
    +            credentials = context.load_client_credentials(client_key,
    94 
    +                                                          client_cert,
    95 
    +                                                          server_cert)
    96 
    +            if not credentials:
    97 
    +                click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    98 
    +                           "Set remote url scheme to `http` in order to deactivate" +
    99 
    +                           "TLS encryption.\n", err=True)
    100 
    +                sys.exit(-1)
    101 
    +
    102 
    +            channel = grpc.secure_channel(remote, credentials)
    103 
    +
    104 
    +        return RemoteStorage(channel)
    105 
    +
    106 
    +
    61 107 
     class WithCache(YamlFactory):
    62 108
    63 109 
         yaml_tag = u'!with-cache-storage'
    ... ... @@ -118,6 +164,7 @@ def get_parser():
    118 164 
         yaml.SafeLoader.add_constructor(Disk.yaml_tag, Disk.from_yaml)
    119 165 
         yaml.SafeLoader.add_constructor(LRU.yaml_tag, LRU.from_yaml)
    120 166 
         yaml.SafeLoader.add_constructor(S3.yaml_tag, S3.from_yaml)
    167 
    +    yaml.SafeLoader.add_constructor(Remote.yaml_tag, Remote.from_yaml)
    121 168 
         yaml.SafeLoader.add_constructor(WithCache.yaml_tag, WithCache.from_yaml)
    122 169 
         yaml.SafeLoader.add_constructor(CAS.yaml_tag, CAS.from_yaml)
    123 170 
         yaml.SafeLoader.add_constructor(ByteStream.yaml_tag, ByteStream.from_yaml)

  • buildgrid/_app/settings/remote-storage.yml
    1 
    +server:
    2 
    +  port: 50051
    3 
    +  insecure-mode: true
    4 
    +  tls-server-key: null
    5 
    +  tls-server-cert: null
    6 
    +  tls-client-certs: null
    7 
    +
    8 
    +
    9 
    +description: |
    10 
    +  A single default instance with remote storage.
    11 
    +
    12 
    +instances:
    13 
    +  - name: main
    14 
    +    description: |
    15 
    +      The main server
    16 
    +
    17 
    +    storages:
    18 
    +        - !remote-storage &main-storage
    19 
    +          url: "https://localhost:50052"
    20 
    +          credentials:
    21 
    +            tls-client-key: null
    22 
    +            tls-client-cert: null
    23 
    +            tls-server-cert: null
    24 
    +
    25 
    +    services:
    26 
    +      - !action-cache &main-action
    27 
    +        storage: *main-storage
    28 
    +        max_cached_refs: 256
    29 
    +        allow_updates: true
    30 
    +
    31 
    +      - !execution
    32 
    +        storage: *main-storage
    33 
    +        action_cache: *main-action
    34 
    +
    35 
    +      - !cas
    36 
    +        storage: *main-storage
    37 
    +
    38 
    +      - !bytestream
    39 
    +        storage: *main-storage

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]