GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][finn/separate-services] Added yml parser

Title: GitLab

finn pushed to branch finn/separate-services at BuildGrid / buildgrid

Commits:

5 changed files:

Changes:

  • buildgrid/_app/commands/cmd_server.py
    ... ... @@ -26,17 +26,9 @@ import sys
    26 26
    27 27 
     import click
    28 28
    29 
    -from buildgrid.server import buildgrid_server
    30 
    -from buildgrid.server.cas.storage.disk import DiskStorage
    31 
    -from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
    32 
    -from buildgrid.server.cas.storage.s3 import S3Storage
    33 
    -from buildgrid.server.cas.storage.with_cache import WithCacheStorage
    34 
    -from buildgrid.server.actioncache.storage import ActionCache
    35 
    -
    29 
    +from ..settings import parser
    36 30 
     from ..cli import pass_context
    37 31
    38 
    -_SIZE_PREFIXES = {'k': 2 ** 10, 'm': 2 ** 20, 'g': 2 ** 30, 't': 2 ** 40}
    39 
    -
    40 32
    41 33 
     @click.group(name='server', short_help="Start a local server instance.")
    42 34 
     @pass_context
    ... ... @@ -45,131 +37,12 @@ def cli(context):
    45 37
    46 38
    47 39 
     @cli.command('start', short_help="Setup a new server instance.")
    48 
    -@click.argument('instances', nargs=-1, type=click.STRING)
    49 
    -@click.option('--port', type=click.INT, default='50051', show_default=True,
    50 
    -              help="The port number to be listened.")
    51 
    -@click.option('--server-key', type=click.Path(exists=True, dir_okay=False), default=None,
    52 
    -              help="Private server key for TLS (PEM-encoded)")
    53 
    -@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    54 
    -              help="Public server certificate for TLS (PEM-encoded)")
    55 
    -@click.option('--client-certs', type=click.Path(exists=True, dir_okay=False), default=None,
    56 
    -              help="Public client certificates for TLS (PEM-encoded, one single file)")
    57 
    -@click.option('--allow-insecure', type=click.BOOL, is_flag=True,
    58 
    -              help="Whether or not to allow unencrypted connections.")
    59 
    -@click.option('--allow-update-action-result/--forbid-update-action-result',
    60 
    -              'allow_uar', default=True, show_default=True,
    61 
    -              help="Whether or not to allow clients to manually edit the action cache.")
    62 
    -@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
    63 
    -              help="Maximum number of actions to keep in the ActionCache.")
    64 
    -@click.option('--cas', type=click.Choice(('lru', 's3', 'disk', 'with-cache')),
    65 
    -              help="The CAS storage type to use.")
    66 
    -@click.option('--cas-cache', type=click.Choice(('lru', 's3', 'disk')),
    67 
    -              help="For --cas=with-cache, the CAS storage to use as the cache.")
    68 
    -@click.option('--cas-fallback', type=click.Choice(('lru', 's3', 'disk')),
    69 
    -              help="For --cas=with-cache, the CAS storage to use as the fallback.")
    70 
    -@click.option('--cas-lru-size', type=click.STRING,
    71 
    -              help="For --cas=lru, the LRU cache's memory limit.")
    72 
    -@click.option('--cas-s3-bucket', type=click.STRING,
    73 
    -              help="For --cas=s3, the bucket name.")
    74 
    -@click.option('--cas-s3-endpoint', type=click.STRING,
    75 
    -              help="For --cas=s3, the endpoint URI.")
    76 
    -@click.option('--cas-disk-directory', type=click.Path(file_okay=False, dir_okay=True, writable=True),
    77 
    -              help="For --cas=disk, the folder to store CAS blobs in.")
    40 
    +@click.argument('yml', type=click.Path(file_okay=True, dir_okay=False, writable=False))
    78 41 
     @pass_context
    79 
    -def start(context, port, allow_insecure, server_key, server_cert, client_certs,
    80 
    -          instances, max_cached_actions, allow_uar, cas, **cas_args):
    81 
    -    """Setups a new server instance."""
    82 
    -    credentials = None
    83 
    -    if not allow_insecure:
    84 
    -        credentials = context.load_server_credentials(server_key, server_cert, client_certs)
    85 
    -    if not credentials and not allow_insecure:
    86 
    -        click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    87 
    -                   "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    88 
    -        sys.exit(-1)
    89 
    -
    90 
    -    context.credentials = credentials
    91 
    -    context.port = port
    92 
    -
    93 
    -    context.logger.info("BuildGrid server booting up")
    94 
    -    context.logger.info("Starting on port {}".format(port))
    95 
    -
    96 
    -    cas_storage = _make_cas_storage(context, cas, cas_args)
    97 
    -
    98 
    -    if cas_storage is None:
    99 
    -        context.logger.info("Running without CAS - action cache will be unavailable")
    100 
    -        action_cache = None
    101 
    -
    102 
    -    else:
    103 
    -        action_cache = ActionCache(cas_storage, max_cached_actions, allow_uar)
    104 
    -
    105 
    -    if instances is None:
    106 
    -        instances = ['main']
    107 
    -
    108 
    -    server = buildgrid_server.BuildGridServer(port=context.port,
    109 
    -                                              credentials=context.credentials,
    110 
    -                                              instances=instances,
    111 
    -                                              cas_storage=cas_storage,
    112 
    -                                              action_cache=action_cache)
    113 
    -    loop = asyncio.get_event_loop()
    114 
    -    try:
    115 
    -        server.start()
    116 
    -        loop.run_forever()
    117 
    -
    118 
    -    except KeyboardInterrupt:
    119 
    -        pass
    120 
    -
    121 
    -    finally:
    122 
    -        server.stop()
    123 
    -        loop.close()
    124 
    -
    125 
    -
    126 
    -def _make_cas_storage(context, cas_type, cas_args):
    127 
    -    """Returns the storage provider corresponding to the given `cas_type`,
    128 
    -    or None if the provider cannot be created.
    129 
    -    """
    130 
    -    if cas_type == "lru":
    131 
    -        if cas_args["cas_lru_size"] is None:
    132 
    -            context.logger.error("--cas-lru-size is required for LRU CAS")
    133 
    -            return None
    134 
    -        try:
    135 
    -            size = _parse_size(cas_args["cas_lru_size"])
    136 
    -        except ValueError:
    137 
    -            context.logger.error('Invalid LRU size "{0}"'.format(cas_args["cas_lru_size"]))
    138 
    -            return None
    139 
    -        return LRUMemoryCache(size)
    140 
    -    elif cas_type == "s3":
    141 
    -        if cas_args["cas_s3_bucket"] is None:
    142 
    -            context.logger.error("--cas-s3-bucket is required for S3 CAS")
    143 
    -            return None
    144 
    -        if cas_args["cas_s3_endpoint"] is not None:
    145 
    -            return S3Storage(cas_args["cas_s3_bucket"],
    146 
    -                             endpoint_url=cas_args["cas_s3_endpoint"])
    147 
    -        return S3Storage(cas_args["cas_s3_bucket"])
    148 
    -    elif cas_type == "disk":
    149 
    -        if cas_args["cas_disk_directory"] is None:
    150 
    -            context.logger.error("--cas-disk-directory is required for disk CAS")
    151 
    -            return None
    152 
    -        return DiskStorage(cas_args["cas_disk_directory"])
    153 
    -    elif cas_type == "with-cache":
    154 
    -        cache = _make_cas_storage(context, cas_args["cas_cache"], cas_args)
    155 
    -        fallback = _make_cas_storage(context, cas_args["cas_fallback"], cas_args)
    156 
    -        if cache is None:
    157 
    -            context.logger.error("Missing cache provider for --cas=with-cache")
    158 
    -            return None
    159 
    -        elif fallback is None:
    160 
    -            context.logger.error("Missing fallback provider for --cas=with-cache")
    161 
    -            return None
    162 
    -        return WithCacheStorage(cache, fallback)
    163 
    -    elif cas_type is None:
    164 
    -        return None
    165 
    -    return None
    42 
    +def start(context, yml):
    43 
    +    with open(yml) as f:
    44 
    +        settings = parser.get_parser().safe_load(f)
    166 45
    46 
    +    print(settings)
    167 47
    168 
    -def _parse_size(size):
    169 
    -    """Convert a string containing a size in bytes (e.g. '2GB') to a number."""
    170 
    -    size = size.lower()
    171 
    -    if size[-1] == 'b':
    172 
    -        size = size[:-1]
    173 
    -    if size[-1] in _SIZE_PREFIXES:
    174 
    -        return int(size[:-1]) * _SIZE_PREFIXES[size[-1]]
    175 
    -    return int(size)
    48 
    +    print(settings['instances'][0]['services'][1]._storage)

  • buildgrid/_app/settings/default.yml
    1 
    +server:
    2 
    +  port: 50001
    3 
    +  tls-server-key: null
    4 
    +  tls-server-cert: null
    5 
    +  tls-client-certs: null
    6 
    +  insecure-mode: true
    7 
    +
    8 
    +description: |
    9 
    +  A single default instance
    10 
    +
    11 
    +instances:
    12 
    +  - name: main
    13 
    +    description: |
    14 
    +      The main server
    15 
    +
    16 
    +    storages:
    17 
    +        - !disk-storage &main-storage
    18 
    +          path: ~/
    19 
    +
    20 
    +        - !lru-storage &main-lru-storage
    21 
    +          size: 10mb
    22 
    +
    23 
    +    services:
    24 
    +      - !action-cache &main-action
    25 
    +        storage: *main-storage
    26 
    +        max_cached_refs: 256
    27 
    +        allow_updates: true
    28 
    +
    29 
    +      - !execution
    30 
    +        storage: *main-storage
    31 
    +        action_cache: *main-action
    32 
    +
    33 
    +      - !cas
    34 
    +        storage: *main-storage
    35 
    +
    36 
    +      - !bytestream
    37 
    +        storage: *main-storage

  • buildgrid/_app/settings/parser.py
    1 
    +# Copyright (C) 2018 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +
    16 
    +import sys
    17 
    +
    18 
    +import yaml
    19 
    +
    20 
    +from buildgrid.server.scheduler import Scheduler
    21 
    +from buildgrid.server.execution.instance import ExecutionInstance
    22 
    +from buildgrid.server.actioncache.storage import ActionCache
    23 
    +from buildgrid.server.cas.instance import ByteStreamInstance, ContentAddressableStorageInstance
    24 
    +from buildgrid.server.cas.storage.disk import DiskStorage
    25 
    +from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
    26 
    +
    27 
    +
    28 
    +class YamlFactory(yaml.YAMLObject):
    29 
    +    @classmethod
    30 
    +    def from_yaml(cls, loader, node):
    31 
    +        values = loader.construct_mapping(node, deep=True)
    32 
    +        return cls(**values)
    33 
    +
    34 
    +
    35 
    +class Disk(YamlFactory):
    36 
    +
    37 
    +    yaml_tag = u'!disk-storage'
    38 
    +
    39 
    +    def __new__(cls, path):
    40 
    +        return DiskStorage(path)
    41 
    +
    42 
    +
    43 
    +class LRU(YamlFactory):
    44 
    +
    45 
    +    yaml_tag = u'!lru-storage'
    46 
    +
    47 
    +    def __new__(cls, size):
    48 
    +        return LRUMemoryCache(_parse_size(size))
    49 
    +
    50 
    +
    51 
    +class Execution(YamlFactory):
    52 
    +
    53 
    +    yaml_tag = u'!execution'
    54 
    +
    55 
    +    def __new__(cls, storage, action_cache=None):
    56 
    +        scheduler = Scheduler(storage)
    57 
    +        return ExecutionInstance(scheduler, storage)
    58 
    +
    59 
    +
    60 
    +class Action(YamlFactory):
    61 
    +
    62 
    +    yaml_tag = u'!action-cache'
    63 
    +
    64 
    +    def __new__(cls, storage, max_cached_refs=0, allow_updates=True):
    65 
    +        return ActionCache(storage, max_cached_refs, allow_updates)
    66 
    +
    67 
    +
    68 
    +class CAS(YamlFactory):
    69 
    +
    70 
    +    yaml_tag = u'!cas'
    71 
    +
    72 
    +    def __new__(cls, storage):
    73 
    +        return ContentAddressableStorageInstance(storage)
    74 
    +
    75 
    +
    76 
    +class ByteStream(YamlFactory):
    77 
    +
    78 
    +    yaml_tag = u'!bytestream'
    79 
    +
    80 
    +    def __new__(cls, storage):
    81 
    +        return ByteStreamInstance(storage)
    82 
    +
    83 
    +
    84 
    +def _parse_size(size):
    85 
    +    """Convert a string containing a size in bytes (e.g. '2GB') to a number."""
    86 
    +    _size_prefixes = {'k': 2 ** 10, 'm': 2 ** 20, 'g': 2 ** 30, 't': 2 ** 40}
    87 
    +    size = size.lower()
    88 
    +
    89 
    +    if size[-1] == 'b':
    90 
    +        size = size[:-1]
    91 
    +    if size[-1] in _size_prefixes:
    92 
    +        return int(size[:-1]) * _size_prefixes[size[-1]]
    93 
    +    return int(size)
    94 
    +
    95 
    +def get_parser():
    96 
    +
    97 
    +    yaml.SafeLoader.add_constructor(Execution.yaml_tag, Execution.from_yaml)
    98 
    +    yaml.SafeLoader.add_constructor(Execution.yaml_tag, Execution.from_yaml)
    99 
    +    yaml.SafeLoader.add_constructor(Action.yaml_tag, Action.from_yaml)
    100 
    +    yaml.SafeLoader.add_constructor(Disk.yaml_tag, Disk.from_yaml)
    101 
    +    yaml.SafeLoader.add_constructor(LRU.yaml_tag, LRU.from_yaml)
    102 
    +    yaml.SafeLoader.add_constructor(CAS.yaml_tag, CAS.from_yaml)
    103 
    +    yaml.SafeLoader.add_constructor(ByteStream.yaml_tag, ByteStream.from_yaml)
    104 
    +
    105 
    +    return yaml

  • buildgrid/server/instance.py
    ... ... @@ -25,21 +25,17 @@ Contains scheduler, execution instance and an interface to the bots.
    25 25
    26 26 
     import logging
    27 27
    28 
    -from .execution.instance import ExecutionInstance
    29 
    -from .scheduler import Scheduler
    30 
    -from .bots.instance import BotsInterface
    28 
    +class BuildGridInstance:
    31 29
    30 
    +    def __init__(self, execution=None, action_cache=None, cas=None, capabilities=None):
    32 31
    33 
    -class BuildGridInstance(ExecutionInstance, BotsInterface):
    34 
    -
    35 
    -    def __init__(self, action_cache=None, cas_storage=None):
    36 
    -        scheduler = Scheduler(action_cache)
    32 
    +        self.execution=execution
    33 
    +        self.action_cache=action_cache
    34 
    +        self.cas=cas
    35 
    +        self.capabilities=capabilities
    37 36
    38 37 
             self.logger = logging.getLogger(__name__)
    39 38
    40 
    -        ExecutionInstance.__init__(self, scheduler, cas_storage)
    41 
    -        BotsInterface.__init__(self, scheduler)
    42 
    -
    43 39 
         def stream_operation_updates(self, message_queue, operation_name):
    44 40 
             operation = message_queue.get()
    45 41 
             while not operation.done:

  • setup.py
    ... ... @@ -114,6 +114,7 @@ setup(
    114 114 
             'protobuf',
    115 115 
             'grpcio',
    116 116 
             'Click',
    117 
    +        'pyaml',
    117 118 
             'boto3 < 1.8.0',
    118 119 
             'botocore < 1.11.0',
    119 120 
             'xdg',

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]