[Notes] [Git][BuildGrid/buildgrid][finn/80-remote-parser] 3 commits: Added remote parser.



Title: GitLab

finn pushed to branch finn/80-remote-parser at BuildGrid / buildgrid

Commits:

6 changed files:

Changes:

  • buildgrid/_app/commands/cmd_server.py
    ... ... @@ -53,10 +53,16 @@ def start(context, config):
    53 53
         insecure_mode = server_settings['insecure-mode']
    
    54 54
     
    
    55 55
         credentials = None
    
    56
    +    credentials_settings = server_settings.get('credentials')
    
    56 57
         if not insecure_mode:
    
    57
    -        server_key = server_settings['tls-server-key']
    
    58
    -        server_cert = server_settings['tls-server-cert']
    
    59
    -        client_certs = server_settings['tls-client-certs']
    
    58
    +        if not credentials_settings:
    
    59
    +            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    
    60
    +                       "Set `insecure-mode: false` in order to deactivate TLS encryption.\n", err=True)
    
    61
    +            sys.exit(-1)
    
    62
    +
    
    63
    +        server_key = credentials_settings['tls-server-key']
    
    64
    +        server_cert = credentials_settings['tls-server-cert']
    
    65
    +        client_certs = credentials_settings['tls-client-certs']
    
    60 66
             credentials = context.load_server_credentials(server_key, server_cert, client_certs)
    
    61 67
     
    
    62 68
             if not credentials:
    

  • buildgrid/_app/settings/__init__.py

  • buildgrid/_app/settings/cas.yml
    1
    +server:
    
    2
    +  port: 50052
    
    3
    +  insecure-mode: true
    
    4
    +  tls-server-key: null
    
    5
    +  tls-server-cert: null
    
    6
    +  tls-client-certs: null
    
    7
    +
    
    8
    +description: |
    
    9
    +  Just a CAS.
    
    10
    +
    
    11
    +instances:
    
    12
    +  - name: main
    
    13
    +    description: |
    
    14
    +      The main server
    
    15
    +
    
    16
    +    storages:
    
    17
    +        - !disk-storage &main-storage
    
    18
    +          path: ~/cas/
    
    19
    +
    
    20
    +    services:
    
    21
    +      - !cas
    
    22
    +        storage: *main-storage

  • buildgrid/_app/settings/default.yml
    1 1
     server:
    
    2 2
       port: 50051
    
    3
    +  insecure-mode: true
    
    3 4
       tls-server-key: null
    
    4 5
       tls-server-cert: null
    
    5 6
       tls-client-certs: null
    
    6
    -  insecure-mode: true
    
    7 7
     
    
    8 8
     description: |
    
    9 9
       A single default instance
    

  • buildgrid/_app/settings/parser.py
    ... ... @@ -14,7 +14,11 @@
    14 14
     
    
    15 15
     
    
    16 16
     import os
    
    17
    +import sys
    
    18
    +from urllib.parse import urlparse
    
    17 19
     
    
    20
    +import click
    
    21
    +import grpc
    
    18 22
     import yaml
    
    19 23
     
    
    20 24
     from buildgrid.server.controller import ExecutionController
    
    ... ... @@ -22,9 +26,12 @@ from buildgrid.server.actioncache.storage import ActionCache
    22 26
     from buildgrid.server.cas.instance import ByteStreamInstance, ContentAddressableStorageInstance
    
    23 27
     from buildgrid.server.cas.storage.disk import DiskStorage
    
    24 28
     from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
    
    29
    +from buildgrid.server.cas.storage.remote import RemoteStorage
    
    25 30
     from buildgrid.server.cas.storage.s3 import S3Storage
    
    26 31
     from buildgrid.server.cas.storage.with_cache import WithCacheStorage
    
    27 32
     
    
    33
    +from ..cli import Context
    
    34
    +
    
    28 35
     
    
    29 36
     class YamlFactory(yaml.YAMLObject):
    
    30 37
         @classmethod
    
    ... ... @@ -58,6 +65,45 @@ class S3(YamlFactory):
    58 65
             return S3Storage(bucket, endpoint_url=endpoint)
    
    59 66
     
    
    60 67
     
    
    68
    +class Remote(YamlFactory):
    
    69
    +
    
    70
    +    yaml_tag = u'!remote-storage'
    
    71
    +
    
    72
    +    def __new__(cls, url, credentials=None):
    
    73
    +        # TODO: Context could be passed into the parser.
    
    74
    +        context = Context()
    
    75
    +
    
    76
    +        url = urlparse(url)
    
    77
    +        remote = '{}:{}'.format(url.hostname, url.port or 50051)
    
    78
    +
    
    79
    +        channel = None
    
    80
    +        if url.scheme == 'http':
    
    81
    +            channel = grpc.insecure_channel(remote)
    
    82
    +
    
    83
    +        else:
    
    84
    +            if not credentials:
    
    85
    +                click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    
    86
    +                           "Set remote url scheme to `http` in order to deactivate" +
    
    87
    +                           "TLS encryption.\n", err=True)
    
    88
    +                sys.exit(-1)
    
    89
    +
    
    90
    +            client_key = credentials['tls-client-key']
    
    91
    +            client_cert = credentials['tls-client-cert']
    
    92
    +            server_cert = credentials['tls-server-cert']
    
    93
    +            credentials = context.load_client_credentials(client_key,
    
    94
    +                                                          client_cert,
    
    95
    +                                                          server_cert)
    
    96
    +            if not credentials:
    
    97
    +                click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    
    98
    +                           "Set remote url scheme to `http` in order to deactivate" +
    
    99
    +                           "TLS encryption.\n", err=True)
    
    100
    +                sys.exit(-1)
    
    101
    +
    
    102
    +            channel = grpc.secure_channel(remote, credentials)
    
    103
    +
    
    104
    +        return RemoteStorage(channel)
    
    105
    +
    
    106
    +
    
    61 107
     class WithCache(YamlFactory):
    
    62 108
     
    
    63 109
         yaml_tag = u'!with-cache-storage'
    
    ... ... @@ -118,6 +164,7 @@ def get_parser():
    118 164
         yaml.SafeLoader.add_constructor(Disk.yaml_tag, Disk.from_yaml)
    
    119 165
         yaml.SafeLoader.add_constructor(LRU.yaml_tag, LRU.from_yaml)
    
    120 166
         yaml.SafeLoader.add_constructor(S3.yaml_tag, S3.from_yaml)
    
    167
    +    yaml.SafeLoader.add_constructor(Remote.yaml_tag, Remote.from_yaml)
    
    121 168
         yaml.SafeLoader.add_constructor(WithCache.yaml_tag, WithCache.from_yaml)
    
    122 169
         yaml.SafeLoader.add_constructor(CAS.yaml_tag, CAS.from_yaml)
    
    123 170
         yaml.SafeLoader.add_constructor(ByteStream.yaml_tag, ByteStream.from_yaml)
    

  • buildgrid/_app/settings/remote-storage.yml
    1
    +server:
    
    2
    +  port: 50051
    
    3
    +  insecure-mode: false
    
    4
    +  tls-server-key: null
    
    5
    +  tls-server-cert: null
    
    6
    +  tls-client-certs: null
    
    7
    +
    
    8
    +
    
    9
    +description: |
    
    10
    +  A single default instance with remote storage.
    
    11
    +
    
    12
    +instances:
    
    13
    +  - name: main
    
    14
    +    description: |
    
    15
    +      The main server
    
    16
    +
    
    17
    +    storages:
    
    18
    +        - !remote-storage &main-storage
    
    19
    +          url: "https://localhost:50052"
    
    20
    +          credentials:
    
    21
    +            tls-client-key: null
    
    22
    +            tls-client-cert: null
    
    23
    +            tls-server-cert: null
    
    24
    +
    
    25
    +    services:
    
    26
    +      - !action-cache &main-action
    
    27
    +        storage: *main-storage
    
    28
    +        max_cached_refs: 256
    
    29
    +        allow_updates: true
    
    30
    +
    
    31
    +      - !execution
    
    32
    +        storage: *main-storage
    
    33
    +        action_cache: *main-action
    
    34
    +
    
    35
    +      - !cas
    
    36
    +        storage: *main-storage
    
    37
    +
    
    38
    +      - !bytestream
    
    39
    +        storage: *main-storage



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]