[Notes] [Git][BuildGrid/buildgrid][finn/separate-services] Added yml parser



Title: GitLab

finn pushed to branch finn/separate-services at BuildGrid / buildgrid

Commits:

13 changed files:

Changes:

  • buildgrid/_app/cli.py
    ... ... @@ -174,7 +174,7 @@ class BuildGridCLI(click.MultiCommand):
    174 174
                 mod = __import__(name='buildgrid._app.commands.cmd_{}'.format(name),
    
    175 175
                                  fromlist=['cli'])
    
    176 176
             except ImportError:
    
    177
    -            return None
    
    177
    +            raise
    
    178 178
             return mod.cli
    
    179 179
     
    
    180 180
     
    

  • buildgrid/_app/commands/cmd_server.py
    ... ... @@ -26,17 +26,15 @@ import sys
    26 26
     
    
    27 27
     import click
    
    28 28
     
    
    29
    -from buildgrid.server import buildgrid_server
    
    29
    +from buildgrid.server.execution.instance import ExecutionInstance
    
    30
    +from buildgrid.server.actioncache.storage import ActionCache
    
    31
    +from buildgrid.server.cas.instance import ByteStreamInstance, ContentAddressableStorageInstance
    
    30 32
     from buildgrid.server.cas.storage.disk import DiskStorage
    
    31 33
     from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
    
    32
    -from buildgrid.server.cas.storage.s3 import S3Storage
    
    33
    -from buildgrid.server.cas.storage.with_cache import WithCacheStorage
    
    34
    -from buildgrid.server.actioncache.storage import ActionCache
    
    35 34
     
    
    36 35
     from ..cli import pass_context
    
    37
    -
    
    38
    -_SIZE_PREFIXES = {'k': 2 ** 10, 'm': 2 ** 20, 'g': 2 ** 30, 't': 2 ** 40}
    
    39
    -
    
    36
    +from ..settings import parser
    
    37
    +from ..server import BuildGridServer
    
    40 38
     
    
    41 39
     @click.group(name='server', short_help="Start a local server instance.")
    
    42 40
     @pass_context
    
    ... ... @@ -45,72 +43,26 @@ def cli(context):
    45 43
     
    
    46 44
     
    
    47 45
     @cli.command('start', short_help="Setup a new server instance.")
    
    48
    -@click.argument('instances', nargs=-1, type=click.STRING)
    
    49
    -@click.option('--port', type=click.INT, default='50051', show_default=True,
    
    50
    -              help="The port number to be listened.")
    
    51
    -@click.option('--server-key', type=click.Path(exists=True, dir_okay=False), default=None,
    
    52
    -              help="Private server key for TLS (PEM-encoded)")
    
    53
    -@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    
    54
    -              help="Public server certificate for TLS (PEM-encoded)")
    
    55
    -@click.option('--client-certs', type=click.Path(exists=True, dir_okay=False), default=None,
    
    56
    -              help="Public client certificates for TLS (PEM-encoded, one single file)")
    
    57
    -@click.option('--allow-insecure', type=click.BOOL, is_flag=True,
    
    58
    -              help="Whether or not to allow unencrypted connections.")
    
    59
    -@click.option('--allow-update-action-result/--forbid-update-action-result',
    
    60
    -              'allow_uar', default=True, show_default=True,
    
    61
    -              help="Whether or not to allow clients to manually edit the action cache.")
    
    62
    -@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
    
    63
    -              help="Maximum number of actions to keep in the ActionCache.")
    
    64
    -@click.option('--cas', type=click.Choice(('lru', 's3', 'disk', 'with-cache')),
    
    65
    -              help="The CAS storage type to use.")
    
    66
    -@click.option('--cas-cache', type=click.Choice(('lru', 's3', 'disk')),
    
    67
    -              help="For --cas=with-cache, the CAS storage to use as the cache.")
    
    68
    -@click.option('--cas-fallback', type=click.Choice(('lru', 's3', 'disk')),
    
    69
    -              help="For --cas=with-cache, the CAS storage to use as the fallback.")
    
    70
    -@click.option('--cas-lru-size', type=click.STRING,
    
    71
    -              help="For --cas=lru, the LRU cache's memory limit.")
    
    72
    -@click.option('--cas-s3-bucket', type=click.STRING,
    
    73
    -              help="For --cas=s3, the bucket name.")
    
    74
    -@click.option('--cas-s3-endpoint', type=click.STRING,
    
    75
    -              help="For --cas=s3, the endpoint URI.")
    
    76
    -@click.option('--cas-disk-directory', type=click.Path(file_okay=False, dir_okay=True, writable=True),
    
    77
    -              help="For --cas=disk, the folder to store CAS blobs in.")
    
    46
    +@click.argument('yml', type=click.Path(file_okay=True, dir_okay=False, writable=False))
    
    78 47
     @pass_context
    
    79
    -def start(context, port, allow_insecure, server_key, server_cert, client_certs,
    
    80
    -          instances, max_cached_actions, allow_uar, cas, **cas_args):
    
    81
    -    """Setups a new server instance."""
    
    82
    -    credentials = None
    
    83
    -    if not allow_insecure:
    
    84
    -        credentials = context.load_server_credentials(server_key, server_cert, client_certs)
    
    85
    -    if not credentials and not allow_insecure:
    
    86
    -        click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    
    87
    -                   "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    
    88
    -        sys.exit(-1)
    
    89
    -
    
    90
    -    context.credentials = credentials
    
    91
    -    context.port = port
    
    92
    -
    
    93
    -    context.logger.info("BuildGrid server booting up")
    
    94
    -    context.logger.info("Starting on port {}".format(port))
    
    95
    -
    
    96
    -    cas_storage = _make_cas_storage(context, cas, cas_args)
    
    97
    -
    
    98
    -    if cas_storage is None:
    
    99
    -        context.logger.info("Running without CAS - action cache will be unavailable")
    
    100
    -        action_cache = None
    
    101
    -
    
    102
    -    else:
    
    103
    -        action_cache = ActionCache(cas_storage, max_cached_actions, allow_uar)
    
    104
    -
    
    105
    -    if instances is None:
    
    106
    -        instances = ['main']
    
    107
    -
    
    108
    -    server = buildgrid_server.BuildGridServer(port=context.port,
    
    109
    -                                              credentials=context.credentials,
    
    110
    -                                              instances=instances,
    
    111
    -                                              cas_storage=cas_storage,
    
    112
    -                                              action_cache=action_cache)
    
    48
    +def start(context, yml):
    
    49
    +    with open(yml) as f:
    
    50
    +        settings = parser.get_parser().safe_load(f)
    
    51
    +
    
    52
    +    #print(settings)
    
    53
    +
    
    54
    +    server_settings = settings['server']
    
    55
    +
    
    56
    +    instances = settings['instances']
    
    57
    +
    
    58
    +    execution_instances = _instance_maker(instances, ExecutionInstance)
    
    59
    +
    
    60
    +    port = server_settings['port']
    
    61
    +    server = BuildGridServer(port=port,
    
    62
    +                             execution_instances=execution_instances)
    
    63
    +
    
    113 64
         loop = asyncio.get_event_loop()
    
    65
    +
    
    114 66
         try:
    
    115 67
             server.start()
    
    116 68
             loop.run_forever()
    
    ... ... @@ -123,53 +75,16 @@ def start(context, port, allow_insecure, server_key, server_cert, client_certs,
    123 75
             loop.close()
    
    124 76
     
    
    125 77
     
    
    126
    -def _make_cas_storage(context, cas_type, cas_args):
    
    127
    -    """Returns the storage provider corresponding to the given `cas_type`,
    
    128
    -    or None if the provider cannot be created.
    
    129
    -    """
    
    130
    -    if cas_type == "lru":
    
    131
    -        if cas_args["cas_lru_size"] is None:
    
    132
    -            context.logger.error("--cas-lru-size is required for LRU CAS")
    
    133
    -            return None
    
    134
    -        try:
    
    135
    -            size = _parse_size(cas_args["cas_lru_size"])
    
    136
    -        except ValueError:
    
    137
    -            context.logger.error('Invalid LRU size "{0}"'.format(cas_args["cas_lru_size"]))
    
    138
    -            return None
    
    139
    -        return LRUMemoryCache(size)
    
    140
    -    elif cas_type == "s3":
    
    141
    -        if cas_args["cas_s3_bucket"] is None:
    
    142
    -            context.logger.error("--cas-s3-bucket is required for S3 CAS")
    
    143
    -            return None
    
    144
    -        if cas_args["cas_s3_endpoint"] is not None:
    
    145
    -            return S3Storage(cas_args["cas_s3_bucket"],
    
    146
    -                             endpoint_url=cas_args["cas_s3_endpoint"])
    
    147
    -        return S3Storage(cas_args["cas_s3_bucket"])
    
    148
    -    elif cas_type == "disk":
    
    149
    -        if cas_args["cas_disk_directory"] is None:
    
    150
    -            context.logger.error("--cas-disk-directory is required for disk CAS")
    
    151
    -            return None
    
    152
    -        return DiskStorage(cas_args["cas_disk_directory"])
    
    153
    -    elif cas_type == "with-cache":
    
    154
    -        cache = _make_cas_storage(context, cas_args["cas_cache"], cas_args)
    
    155
    -        fallback = _make_cas_storage(context, cas_args["cas_fallback"], cas_args)
    
    156
    -        if cache is None:
    
    157
    -            context.logger.error("Missing cache provider for --cas=with-cache")
    
    158
    -            return None
    
    159
    -        elif fallback is None:
    
    160
    -            context.logger.error("Missing fallback provider for --cas=with-cache")
    
    161
    -            return None
    
    162
    -        return WithCacheStorage(cache, fallback)
    
    163
    -    elif cas_type is None:
    
    164
    -        return None
    
    165
    -    return None
    
    166
    -
    
    167
    -
    
    168
    -def _parse_size(size):
    
    169
    -    """Convert a string containing a size in bytes (e.g. '2GB') to a number."""
    
    170
    -    size = size.lower()
    
    171
    -    if size[-1] == 'b':
    
    172
    -        size = size[:-1]
    
    173
    -    if size[-1] in _SIZE_PREFIXES:
    
    174
    -        return int(size[:-1]) * _SIZE_PREFIXES[size[-1]]
    
    175
    -    return int(size)
    78
    +# Turn away now if you want to keep your eyes
    
    79
    +def _instance_maker(instances, service_type):
    
    80
    +    # I did warn you
    
    81
    +    # Really should map this properly
    
    82
    +    made = {}
    
    83
    +
    
    84
    +    for instance in instances:
    
    85
    +        services = instance['services']
    
    86
    +        instance_name = instance['name']
    
    87
    +        for service in services:
    
    88
    +            if type(service) == service_type:
    
    89
    +                made[instance_name] = service
    
    90
    +    return made

  • buildgrid/_app/server.py
    1
    +# Copyright (C) 2018 Bloomberg LP
    
    2
    +#
    
    3
    +# Licensed under the Apache License, Version 2.0 (the "License");
    
    4
    +# you may not use this file except in compliance with the License.
    
    5
    +# You may obtain a copy of the License at
    
    6
    +#
    
    7
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    
    8
    +#
    
    9
    +# Unless required by applicable law or agreed to in writing, software
    
    10
    +# distributed under the License is distributed on an "AS IS" BASIS,
    
    11
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    
    12
    +# See the License for the specific language governing permissions and
    
    13
    +# limitations under the License.
    
    14
    +
    
    15
    +
    
    16
    +"""
    
    17
    +BuildGridServer
    
    18
    +==============
    
    19
    +
    
    20
    +Creates the user a local BuildGrid server.
    
    21
    +"""
    
    22
    +
    
    23
    +from concurrent import futures
    
    24
    +
    
    25
    +import grpc
    
    26
    +
    
    27
    +from buildgrid.server.cas.service import ByteStreamService, ContentAddressableStorageService
    
    28
    +from buildgrid.server.actioncache.service import ActionCacheService
    
    29
    +from buildgrid.server.execution.service import ExecutionService
    
    30
    +from buildgrid.server.operations.service import OperationsService
    
    31
    +from buildgrid.server.bots.service import BotsService
    
    32
    +
    
    33
    +
    
    34
    +class BuildGridServer:
    
    35
    +
    
    36
    +    def __init__(self, port=50051, credentials=None, execution_instances=None,
    
    37
    +                 bots_interface_instances = None, operations_instances=None,
    
    38
    +                 reference_storage_instances = None,
    
    39
    +                 max_workers=10, action_cache_instances=None, cas_instances=None):
    
    40
    +
    
    41
    +        address = '[::]:{0}'.format(port)
    
    42
    +
    
    43
    +        server = grpc.server(futures.ThreadPoolExecutor(max_workers))
    
    44
    +
    
    45
    +        if credentials is not None:
    
    46
    +            server.add_secure_port(address, credentials)
    
    47
    +
    
    48
    +        else:
    
    49
    +            server.add_insecure_port(address)
    
    50
    +
    
    51
    +        if execution_instances:
    
    52
    +            service = ExecutionService(execution_instances)
    
    53
    +            service.add_self_to_server(server)
    
    54
    +
    
    55
    +        if bots_interface_instances:
    
    56
    +            service = BotsService(bots_interface_instances)
    
    57
    +            service.add_self_to_server(server)
    
    58
    +
    
    59
    +        self._server = server
    
    60
    +
    
    61
    +    def start(self):
    
    62
    +        self._server.start()
    
    63
    +
    
    64
    +    def stop(self):
    
    65
    +        self._server.stop(0)

  • buildgrid/_app/settings/default.yml
    1
    +server:
    
    2
    +  port: 50001
    
    3
    +  tls-server-key: null
    
    4
    +  tls-server-cert: null
    
    5
    +  tls-client-certs: null
    
    6
    +  insecure-mode: true
    
    7
    +
    
    8
    +description: |
    
    9
    +  A single default instance
    
    10
    +
    
    11
    +instances:
    
    12
    +  - name: main
    
    13
    +    description: |
    
    14
    +      The main server
    
    15
    +
    
    16
    +    storages:
    
    17
    +        - !disk-storage &main-storage
    
    18
    +          path: ~/
    
    19
    +
    
    20
    +        - !lru-storage &main-lru-storage
    
    21
    +          size: 10mb
    
    22
    +
    
    23
    +    services:
    
    24
    +      - !action-cache &main-action
    
    25
    +        storage: *main-storage
    
    26
    +        max_cached_refs: 256
    
    27
    +        allow_updates: true
    
    28
    +
    
    29
    +      - !execution
    
    30
    +        storage: *main-storage
    
    31
    +        action_cache: *main-action
    
    32
    +
    
    33
    +      - !cas
    
    34
    +        storage: *main-storage
    
    35
    +
    
    36
    +      - !bytestream
    
    37
    +        storage: *main-storage

  • buildgrid/_app/settings/parser.py
    1
    +# Copyright (C) 2018 Bloomberg LP
    
    2
    +#
    
    3
    +# Licensed under the Apache License, Version 2.0 (the "License");
    
    4
    +# you may not use this file except in compliance with the License.
    
    5
    +# You may obtain a copy of the License at
    
    6
    +#
    
    7
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    
    8
    +#
    
    9
    +# Unless required by applicable law or agreed to in writing, software
    
    10
    +# distributed under the License is distributed on an "AS IS" BASIS,
    
    11
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    
    12
    +# See the License for the specific language governing permissions and
    
    13
    +# limitations under the License.
    
    14
    +
    
    15
    +
    
    16
    +import sys
    
    17
    +
    
    18
    +import yaml
    
    19
    +
    
    20
    +from buildgrid.server.scheduler import Scheduler
    
    21
    +from buildgrid.server.execution.instance import ExecutionInstance
    
    22
    +from buildgrid.server.actioncache.storage import ActionCache
    
    23
    +from buildgrid.server.cas.instance import ByteStreamInstance, ContentAddressableStorageInstance
    
    24
    +from buildgrid.server.cas.storage.disk import DiskStorage
    
    25
    +from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
    
    26
    +
    
    27
    +
    
    28
    +class YamlFactory(yaml.YAMLObject):
    
    29
    +    @classmethod
    
    30
    +    def from_yaml(cls, loader, node):
    
    31
    +        values = loader.construct_mapping(node, deep=True)
    
    32
    +        return cls(**values)
    
    33
    +
    
    34
    +
    
    35
    +class Disk(YamlFactory):
    
    36
    +
    
    37
    +    yaml_tag = u'!disk-storage'
    
    38
    +
    
    39
    +    def __new__(cls, path):
    
    40
    +        return DiskStorage(path)
    
    41
    +
    
    42
    +
    
    43
    +class LRU(YamlFactory):
    
    44
    +
    
    45
    +    yaml_tag = u'!lru-storage'
    
    46
    +
    
    47
    +    def __new__(cls, size):
    
    48
    +        return LRUMemoryCache(_parse_size(size))
    
    49
    +
    
    50
    +
    
    51
    +class Execution(YamlFactory):
    
    52
    +
    
    53
    +    yaml_tag = u'!execution'
    
    54
    +
    
    55
    +    def __new__(cls, storage, action_cache=None):
    
    56
    +        scheduler = Scheduler(storage)
    
    57
    +        return ExecutionInstance(scheduler, storage)
    
    58
    +
    
    59
    +
    
    60
    +class Action(YamlFactory):
    
    61
    +
    
    62
    +    yaml_tag = u'!action-cache'
    
    63
    +
    
    64
    +    def __new__(cls, storage, max_cached_refs=0, allow_updates=True):
    
    65
    +        return ActionCache(storage, max_cached_refs, allow_updates)
    
    66
    +
    
    67
    +
    
    68
    +class CAS(YamlFactory):
    
    69
    +
    
    70
    +    yaml_tag = u'!cas'
    
    71
    +
    
    72
    +    def __new__(cls, storage):
    
    73
    +        return ContentAddressableStorageInstance(storage)
    
    74
    +
    
    75
    +
    
    76
    +class ByteStream(YamlFactory):
    
    77
    +
    
    78
    +    yaml_tag = u'!bytestream'
    
    79
    +
    
    80
    +    def __new__(cls, storage):
    
    81
    +        return ByteStreamInstance(storage)
    
    82
    +
    
    83
    +
    
    84
    +def _parse_size(size):
    
    85
    +    """Convert a string containing a size in bytes (e.g. '2GB') to a number."""
    
    86
    +    _size_prefixes = {'k': 2 ** 10, 'm': 2 ** 20, 'g': 2 ** 30, 't': 2 ** 40}
    
    87
    +    size = size.lower()
    
    88
    +
    
    89
    +    if size[-1] == 'b':
    
    90
    +        size = size[:-1]
    
    91
    +    if size[-1] in _size_prefixes:
    
    92
    +        return int(size[:-1]) * _size_prefixes[size[-1]]
    
    93
    +    return int(size)
    
    94
    +
    
    95
    +def get_parser():
    
    96
    +
    
    97
    +    yaml.SafeLoader.add_constructor(Execution.yaml_tag, Execution.from_yaml)
    
    98
    +    yaml.SafeLoader.add_constructor(Execution.yaml_tag, Execution.from_yaml)
    
    99
    +    yaml.SafeLoader.add_constructor(Action.yaml_tag, Action.from_yaml)
    
    100
    +    yaml.SafeLoader.add_constructor(Disk.yaml_tag, Disk.from_yaml)
    
    101
    +    yaml.SafeLoader.add_constructor(LRU.yaml_tag, LRU.from_yaml)
    
    102
    +    yaml.SafeLoader.add_constructor(CAS.yaml_tag, CAS.from_yaml)
    
    103
    +    yaml.SafeLoader.add_constructor(ByteStream.yaml_tag, ByteStream.from_yaml)
    
    104
    +
    
    105
    +    return yaml

  • buildgrid/server/bots/service.py
    ... ... @@ -83,6 +83,9 @@ class BotsService(bots_pb2_grpc.BotsServicer):
    83 83
             context.set_code(grpc.StatusCode.UNIMPLEMENTED)
    
    84 84
             return Empty()
    
    85 85
     
    
    86
    +    def add_self_to_server(self, server):
    
    87
    +        bots_pb2_grpc.add_BotsServicer_to_server(self, server)
    
    88
    +
    
    86 89
         def _get_instance(self, name):
    
    87 90
             try:
    
    88 91
                 return self._instances[name]
    

  • buildgrid/server/buildgrid_server.py deleted
    1
    -# Copyright (C) 2018 Bloomberg LP
    
    2
    -#
    
    3
    -# Licensed under the Apache License, Version 2.0 (the "License");
    
    4
    -# you may not use this file except in compliance with the License.
    
    5
    -# You may obtain a copy of the License at
    
    6
    -#
    
    7
    -#  <http://www.apache.org/licenses/LICENSE-2.0>
    
    8
    -#
    
    9
    -# Unless required by applicable law or agreed to in writing, software
    
    10
    -# distributed under the License is distributed on an "AS IS" BASIS,
    
    11
    -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    
    12
    -# See the License for the specific language governing permissions and
    
    13
    -# limitations under the License.
    
    14
    -
    
    15
    -
    
    16
    -"""
    
    17
    -BuildGridServer
    
    18
    -==============
    
    19
    -
    
    20
    -Creates the user a local server BuildGrid server.
    
    21
    -"""
    
    22
    -
    
    23
    -from concurrent import futures
    
    24
    -
    
    25
    -import grpc
    
    26
    -
    
    27
    -from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
    
    28
    -from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc
    
    29
    -from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2_grpc
    
    30
    -from buildgrid._protos.google.longrunning import operations_pb2_grpc
    
    31
    -
    
    32
    -from .instance import BuildGridInstance
    
    33
    -from .cas.service import ByteStreamService, ContentAddressableStorageService
    
    34
    -from .actioncache.service import ActionCacheService
    
    35
    -from .execution.service import ExecutionService
    
    36
    -from .operations.service import OperationsService
    
    37
    -from .bots.service import BotsService
    
    38
    -
    
    39
    -
    
    40
    -class BuildGridServer:
    
    41
    -
    
    42
    -    def __init__(self, port=50051, credentials=None, instances=None,
    
    43
    -                 max_workers=10, action_cache=None, cas_storage=None):
    
    44
    -        address = '[::]:{0}'.format(port)
    
    45
    -
    
    46
    -        self._server = grpc.server(futures.ThreadPoolExecutor(max_workers))
    
    47
    -
    
    48
    -        if credentials is not None:
    
    49
    -            self._server.add_secure_port(address, credentials)
    
    50
    -        else:
    
    51
    -            self._server.add_insecure_port(address)
    
    52
    -
    
    53
    -        if cas_storage is not None:
    
    54
    -            cas_service = ContentAddressableStorageService(cas_storage)
    
    55
    -            remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server(cas_service,
    
    56
    -                                                                                      self._server)
    
    57
    -            bytestream_pb2_grpc.add_ByteStreamServicer_to_server(ByteStreamService(cas_storage),
    
    58
    -                                                                 self._server)
    
    59
    -        if action_cache is not None:
    
    60
    -            action_cache_service = ActionCacheService(action_cache)
    
    61
    -            remote_execution_pb2_grpc.add_ActionCacheServicer_to_server(action_cache_service,
    
    62
    -                                                                        self._server)
    
    63
    -
    
    64
    -        buildgrid_instances = {}
    
    65
    -        if not instances:
    
    66
    -            buildgrid_instances["main"] = BuildGridInstance(action_cache, cas_storage)
    
    67
    -        else:
    
    68
    -            for name in instances:
    
    69
    -                buildgrid_instances[name] = BuildGridInstance(action_cache, cas_storage)
    
    70
    -
    
    71
    -        bots_pb2_grpc.add_BotsServicer_to_server(BotsService(buildgrid_instances),
    
    72
    -                                                 self._server)
    
    73
    -        remote_execution_pb2_grpc.add_ExecutionServicer_to_server(ExecutionService(buildgrid_instances),
    
    74
    -                                                                  self._server)
    
    75
    -        operations_pb2_grpc.add_OperationsServicer_to_server(OperationsService(buildgrid_instances),
    
    76
    -                                                             self._server)
    
    77
    -
    
    78
    -    def start(self):
    
    79
    -        self._server.start()
    
    80
    -
    
    81
    -    def stop(self):
    
    82
    -        self._server.stop(0)

  • buildgrid/server/cas/service.py
    ... ... @@ -27,13 +27,13 @@ import logging
    27 27
     import grpc
    
    28 28
     
    
    29 29
     from buildgrid._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
    
    30
    -from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2
    
    31
    -from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc as re_pb2_grpc
    
    30
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    
    31
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc
    
    32 32
     
    
    33 33
     from .._exceptions import InvalidArgumentError, NotFoundError, OutOfRangeError
    
    34 34
     
    
    35 35
     
    
    36
    -class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServicer):
    
    36
    +class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressableStorageServicer):
    
    37 37
     
    
    38 38
         def __init__(self, instances):
    
    39 39
             self.logger = logging.getLogger(__name__)
    
    ... ... @@ -49,7 +49,7 @@ class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServ
    49 49
                 context.set_details(str(e))
    
    50 50
                 context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
    
    51 51
     
    
    52
    -        return re_pb2.FindMissingBlobsResponse()
    
    52
    +        return remote_execution_pb2.FindMissingBlobsResponse()
    
    53 53
     
    
    54 54
         def BatchUpdateBlobs(self, request, context):
    
    55 55
             try:
    
    ... ... @@ -61,7 +61,10 @@ class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServ
    61 61
                 context.set_details(str(e))
    
    62 62
                 context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
    
    63 63
     
    
    64
    -        return re_pb2.BatchReadBlobsResponse()
    
    64
    +        return remote_execution_pb2.BatchReadBlobsResponse()
    
    65
    +
    
    66
    +    def add_self_to_server(self, server):
    
    67
    +        remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server(self, server)
    
    65 68
     
    
    66 69
         def _get_instance(self, instance_name):
    
    67 70
             try:
    
    ... ... @@ -140,6 +143,9 @@ class ByteStreamService(bytestream_pb2_grpc.ByteStreamServicer):
    140 143
     
    
    141 144
             return bytestream_pb2.WriteResponse()
    
    142 145
     
    
    146
    +    def add_self_to_server(self, server):
    
    147
    +        bytestream_pb2_grpc.add_ByteStreamServicer_to_server(self, server)
    
    148
    +
    
    143 149
         def _get_instance(self, instance_name):
    
    144 150
             try:
    
    145 151
                 return self._instances[instance_name]
    

  • buildgrid/server/execution/service.py
    ... ... @@ -85,6 +85,9 @@ class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
    85 85
                 context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
    
    86 86
                 yield operations_pb2.Operation()
    
    87 87
     
    
    88
    +    def add_self_to_server(self, server):
    
    89
    +        remote_execution_pb2_grpc.add_ExecutionServicer_to_server(self, server)
    
    90
    +
    
    88 91
         def _get_instance(self, name):
    
    89 92
             try:
    
    90 93
                 return self._instances[name]
    

  • buildgrid/server/instance.py deleted
    1
    -# Copyright (C) 2018 Bloomberg LP
    
    2
    -#
    
    3
    -# Licensed under the Apache License, Version 2.0 (the "License");
    
    4
    -# you may not use this file except in compliance with the License.
    
    5
    -# You may obtain a copy of the License at
    
    6
    -#
    
    7
    -#  <http://www.apache.org/licenses/LICENSE-2.0>
    
    8
    -#
    
    9
    -# Unless required by applicable law or agreed to in writing, software
    
    10
    -# distributed under the License is distributed on an "AS IS" BASIS,
    
    11
    -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    
    12
    -# See the License for the specific language governing permissions and
    
    13
    -# limitations under the License.
    
    14
    -
    
    15
    -
    
    16
    -"""
    
    17
    -BuildGrid Instance
    
    18
    -==================
    
    19
    -
    
    20
    -An instance of the BuildGrid server.
    
    21
    -
    
    22
    -Contains scheduler, execution instance and an interface to the bots.
    
    23
    -"""
    
    24
    -
    
    25
    -
    
    26
    -import logging
    
    27
    -
    
    28
    -from .execution.instance import ExecutionInstance
    
    29
    -from .scheduler import Scheduler
    
    30
    -from .bots.instance import BotsInterface
    
    31
    -
    
    32
    -
    
    33
    -class BuildGridInstance(ExecutionInstance, BotsInterface):
    
    34
    -
    
    35
    -    def __init__(self, action_cache=None, cas_storage=None):
    
    36
    -        scheduler = Scheduler(action_cache)
    
    37
    -
    
    38
    -        self.logger = logging.getLogger(__name__)
    
    39
    -
    
    40
    -        ExecutionInstance.__init__(self, scheduler, cas_storage)
    
    41
    -        BotsInterface.__init__(self, scheduler)
    
    42
    -
    
    43
    -    def stream_operation_updates(self, message_queue, operation_name):
    
    44
    -        operation = message_queue.get()
    
    45
    -        while not operation.done:
    
    46
    -            yield operation
    
    47
    -            operation = message_queue.get()
    
    48
    -        yield operation
    
    49
    -
    
    50
    -    def cancel_operation(self, name):
    
    51
    -        # TODO: Cancel leases
    
    52
    -        raise NotImplementedError("Cancelled operations not supported")

  • buildgrid/server/operations/service.py
    ... ... @@ -29,7 +29,6 @@ from buildgrid._protos.google.longrunning import operations_pb2_grpc, operations
    29 29
     
    
    30 30
     from .._exceptions import InvalidArgumentError
    
    31 31
     
    
    32
    -
    
    33 32
     class OperationsService(operations_pb2_grpc.OperationsServicer):
    
    34 33
     
    
    35 34
         def __init__(self, instances):
    
    ... ... @@ -114,6 +113,9 @@ class OperationsService(operations_pb2_grpc.OperationsServicer):
    114 113
     
    
    115 114
             return Empty()
    
    116 115
     
    
    116
    +    def add_self_to_server(self, server):
    
    117
    +        operations_pb2_grpc.add_OperationsServicer_to_server(self, server)
    
    118
    +
    
    117 119
         def _get_operation_name(self, name):
    
    118 120
             return name.split("/")[-1]
    
    119 121
     
    

  • buildgrid/server/referencestorage/service.py
    ... ... @@ -51,3 +51,6 @@ class ReferenceStorageService(buildstream_pb2_grpc.ReferenceStorageServicer):
    51 51
         def Status(self, request, context):
    
    52 52
             allow_updates = self._reference_cache.allow_updates
    
    53 53
             return buildstream_pb2.StatusResponse(allow_updates=allow_updates)
    
    54
    +
    
    55
    +    def add_self_to_server(self, server):
    
    56
    +        buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server(self, server)

  • setup.py
    ... ... @@ -114,6 +114,7 @@ setup(
    114 114
             'protobuf',
    
    115 115
             'grpcio',
    
    116 116
             'Click',
    
    117
    +        'pyaml',
    
    117 118
             'boto3 < 1.8.0',
    
    118 119
             'botocore < 1.11.0',
    
    119 120
             'xdg',
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]