GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][mablanch/63-tls-encryption] 5 commits: setup.py: Avoid the boto3 1.8.x release series

Title: GitLab

Martin Blanchard pushed to branch mablanch/63-tls-encryption at BuildGrid / buildgrid

Commits:

11 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -28,10 +28,10 @@ before_script:
    28 28 
     .run-dummy-job-template: &dummy-job
    29 29 
       stage: test
    30 30 
       script:
    31 
    -    - ${BGD} server start &
    31 
    +    - ${BGD} server start --allow-insecure &
    32 32 
         - sleep 1 # Allow server to boot
    33 
    -    - ${BGD} bot --host=0.0.0.0 dummy &
    34 
    -    - ${BGD} execute --host=0.0.0.0 request-dummy --wait-for-completion
    33 
    +    - ${BGD} bot dummy &
    34 
    +    - ${BGD} execute request-dummy --wait-for-completion
    35 35
    36 36 
     tests-debian-stretch:
    37 37 
       <<: *linux-tests

  • buildgrid/_app/cli.py
    ... ... @@ -25,6 +25,9 @@ import os
    25 25 
     import logging
    26 26
    27 27 
     import click
    28 
    +import grpc
    29 
    +
    30 
    +from buildgrid.utils import read_file
    28 31
    29 32 
     from . import _logging
    30 33
    ... ... @@ -35,7 +38,114 @@ class Context:
    35 38
    36 39 
         def __init__(self):
    37 40 
             self.verbose = False
    38 
    -        self.home = os.getcwd()
    41 
    +
    42 
    +        self.user_home = os.getcwd()
    43 
    +
    44 
    +        self.user_cache_home = os.environ.get('XDG_CACHE_HOME')
    45 
    +        if not self.user_cache_home:
    46 
    +            self.user_cache_home = os.path.expanduser('~/.cache')
    47 
    +        self.cache_home = os.path.join(self.user_cache_home, 'buildgrid')
    48 
    +
    49 
    +        self.user_config_home = os.environ.get('XDG_CONFIG_HOME')
    50 
    +        if not self.user_config_home:
    51 
    +            self.user_config_home = os.path.expanduser('~/.config')
    52 
    +        self.config_home = os.path.join(self.user_config_home, 'buildgrid')
    53 
    +
    54 
    +        self.user_data_home = os.environ.get('XDG_DATA_HOME')
    55 
    +        if not self.user_data_home:
    56 
    +            self.user_data_home = os.path.expanduser('~/.local/share')
    57 
    +        self.data_home = os.path.join(self.user_data_home, 'buildgrid')
    58 
    +
    59 
    +    def load_client_credentials(self, client_key=None, client_cert=None,
    60 
    +                                server_cert=None, use_default_client_keys=False):
    61 
    +        """Looks-up and loads TLS client gRPC credentials.
    62 
    +
    63 
    +        Args:
    64 
    +            client_key(str): root certificate file path.
    65 
    +            client_cert(str): private key file path.
    66 
    +            server_cert(str): certificate chain file path.
    67 
    +            use_default_client_keys(bool, optional): whether or not to try
    68 
    +                loading client keys from default location. Defaults to False.
    69 
    +
    70 
    +        Returns:
    71 
    +            :obj:`ChannelCredentials`: The credentials for use for a
    72 
    +            TLS-encrypted gRPC client channel.
    73 
    +        """
    74 
    +        if not client_key or not os.path.exists(client_key):
    75 
    +            if use_default_client_keys:
    76 
    +                client_key = os.path.join(self.config_home, 'client.key')
    77 
    +            else:
    78 
    +                client_key = None
    79 
    +
    80 
    +        if not client_cert or not os.path.exists(client_cert):
    81 
    +            if use_default_client_keys:
    82 
    +                client_cert = os.path.join(self.config_home, 'client.crt')
    83 
    +            else:
    84 
    +                client_cert = None
    85 
    +
    86 
    +        if not server_cert or not os.path.exists(server_cert):
    87 
    +            server_cert = os.path.join(self.config_home, 'server.crt')
    88 
    +            if not os.path.exists(server_cert):
    89 
    +                return None
    90 
    +
    91 
    +        server_cert_pem = read_file(server_cert)
    92 
    +        if client_key and os.path.exists(client_key):
    93 
    +            client_key_pem = read_file(client_key)
    94 
    +        else:
    95 
    +            client_key_pem = None
    96 
    +        if client_key_pem and client_cert and os.path.exists(client_cert):
    97 
    +            client_cert_pem = read_file(client_cert)
    98 
    +        else:
    99 
    +            client_cert_pem = None
    100 
    +
    101 
    +        return grpc.ssl_channel_credentials(root_certificates=server_cert_pem,
    102 
    +                                            private_key=client_key_pem,
    103 
    +                                            certificate_chain=client_cert_pem)
    104 
    +
    105 
    +    def load_server_credentials(self, server_key=None, server_cert=None,
    106 
    +                                client_certs=None, use_default_client_certs=False):
    107 
    +        """Looks-up and loads TLS server gRPC credentials.
    108 
    +
    109 
    +        Every private and public keys are expected to be PEM-encoded.
    110 
    +
    111 
    +        Args:
    112 
    +            server_key(str): private server key file path.
    113 
    +            server_cert(str): public server certificate file path.
    114 
    +            client_certs(str): public client certificates file path.
    115 
    +            use_default_client_certs(bool, optional): whether or not to try
    116 
    +                loading public client certificates from default location.
    117 
    +                Defaults to False.
    118 
    +
    119 
    +        Returns:
    120 
    +            :obj:`ServerCredentials`: The credentials for use for a
    121 
    +            TLS-encrypted gRPC server channel.
    122 
    +        """
    123 
    +        if not server_key or not os.path.exists(server_key):
    124 
    +            server_key = os.path.join(self.config_home, 'server.key')
    125 
    +            if not os.path.exists(server_key):
    126 
    +                return None
    127 
    +
    128 
    +        if not server_cert or not os.path.exists(server_cert):
    129 
    +            server_cert = os.path.join(self.config_home, 'server.crt')
    130 
    +            if not os.path.exists(server_cert):
    131 
    +                return None
    132 
    +
    133 
    +        if not client_certs or not os.path.exists(client_certs):
    134 
    +            if use_default_client_certs:
    135 
    +                client_certs = os.path.join(self.config_home, 'client.crt')
    136 
    +            else:
    137 
    +                client_certs = None
    138 
    +
    139 
    +        server_key_pem = read_file(server_key)
    140 
    +        server_cert_pem = read_file(server_cert)
    141 
    +        if client_certs and os.path.exists(client_certs):
    142 
    +            client_certs_pem = read_file(client_certs)
    143 
    +        else:
    144 
    +            client_certs_pem = None
    145 
    +
    146 
    +        return grpc.ssl_server_credentials([(server_key_pem, server_cert_pem)],
    147 
    +                                           root_certificates=client_certs_pem,
    148 
    +                                           require_client_auth=bool(client_certs))
    39 149
    40 150
    41 151 
     pass_context = click.make_pass_decorator(Context, ensure=True)

  • buildgrid/_app/commands/cmd_bot.py
    ... ... @@ -21,8 +21,9 @@ Create a bot interface and request work
    21 21 
     """
    22 22
    23 23 
     import logging
    24 
    -
    25 24 
     from pathlib import Path, PurePath
    25 
    +import sys
    26 
    +from urllib.parse import urlparse
    26 27
    27 28 
     import click
    28 29 
     import grpc
    ... ... @@ -35,22 +36,39 @@ from ..cli import pass_context
    35 36
    36 37
    37 38 
     @click.group(name='bot', short_help="Create and register bot clients.")
    39 
    +@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    40 
    +              help="Remote execution server's URL (port defaults to 50051 if not specified).")
    41 
    +@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    42 
    +              help="Private client key for TLS (PEM-encoded)")
    43 
    +@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    44 
    +              help="Public client certificate for TLS (PEM-encoded)")
    45 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    46 
    +              help="Public server certificate for TLS (PEM-encoded)")
    38 47 
     @click.option('--parent', type=click.STRING, default='main', show_default=True,
    39 48 
                   help="Targeted farm resource.")
    40 
    -@click.option('--port', type=click.INT, default='50051', show_default=True,
    41 
    -              help="Remote server's port number.")
    42 
    -@click.option('--host', type=click.STRING, default='localhost', show_default=True,
    43 
    -              help="Renote server's hostname.")
    44 49 
     @pass_context
    45 
    -def cli(context, host, port, parent):
    46 
    -    channel = grpc.insecure_channel('{}:{}'.format(host, port))
    47 
    -    interface = bot_interface.BotInterface(channel)
    50 
    +def cli(context, remote, parent, client_key, client_cert, server_cert):
    51 
    +    url = urlparse(remote)
    48 52
    49 
    -    context.logger = logging.getLogger(__name__)
    50 
    -    context.logger.info("Starting on port {}".format(port))
    51 
    -    context.channel = channel
    53 
    +    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    52 54 
         context.parent = parent
    53 55
    56 
    +    if url.scheme == 'http':
    57 
    +        context.channel = grpc.insecure_channel(context.remote)
    58 
    +    else:
    59 
    +        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    60 
    +        if not credentials:
    61 
    +            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    62 
    +                       "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    63 
    +            sys.exit(-1)
    64 
    +
    65 
    +        context.channel = grpc.secure_channel(context.remote, credentials)
    66 
    +
    67 
    +    context.logger = logging.getLogger(__name__)
    68 
    +    context.logger.debug("Starting for remote {}".format(context.remote))
    69 
    +
    70 
    +    interface = bot_interface.BotInterface(context.channel)
    71 
    +
    54 72 
         worker = Worker()
    55 73 
         worker.add_device(Device())
    56 74

  • buildgrid/_app/commands/cmd_cas.py
    ... ... @@ -21,6 +21,9 @@ Request work to be executed and monitor status of jobs.
    21 21 
     """
    22 22
    23 23 
     import logging
    24 
    +import sys
    25 
    +from urllib.parse import urlparse
    26 
    +
    24 27 
     import click
    25 28 
     import grpc
    26 29
    ... ... @@ -31,20 +34,36 @@ from ..cli import pass_context
    31 34
    32 35
    33 36 
     @click.group(name='cas', short_help="Interact with the CAS server.")
    37 
    +@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    38 
    +              help="Remote execution server's URL (port defaults to 50051 if no specified).")
    39 
    +@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    40 
    +              help="Private client key for TLS (PEM-encoded)")
    41 
    +@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    42 
    +              help="Public client certificate for TLS (PEM-encoded)")
    43 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    44 
    +              help="Public server certificate for TLS (PEM-encoded)")
    34 45 
     @click.option('--instance-name', type=click.STRING, default='main', show_default=True,
    35 46 
                   help="Targeted farm instance name.")
    36 
    -@click.option('--port', type=click.INT, default='50051', show_default=True,
    37 
    -              help="Remote server's port number.")
    38 
    -@click.option('--host', type=click.STRING, default='localhost', show_default=True,
    39 
    -              help="Remote server's hostname.")
    40 47 
     @pass_context
    41 
    -def cli(context, instance_name, host, port):
    42 
    -    context.logger = logging.getLogger(__name__)
    43 
    -    context.logger.info("Starting on port {}".format(port))
    48 
    +def cli(context, remote, instance_name, client_key, client_cert, server_cert):
    49 
    +    url = urlparse(remote)
    44 50
    51 
    +    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    45 52 
         context.instance_name = instance_name
    46 
    -    context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
    47 
    -    context.port = port
    53 
    +
    54 
    +    if url.scheme == 'http':
    55 
    +        context.channel = grpc.insecure_channel(context.remote)
    56 
    +    else:
    57 
    +        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    58 
    +        if not credentials:
    59 
    +            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    60 
    +                       "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    61 
    +            sys.exit(-1)
    62 
    +
    63 
    +        context.channel = grpc.secure_channel(context.remote, credentials)
    64 
    +
    65 
    +    context.logger = logging.getLogger(__name__)
    66 
    +    context.logger.debug("Starting for remote {}".format(context.remote))
    48 67
    49 68
    50 69 
     @cli.command('upload-files', short_help="Upload files to the CAS server.")

  • buildgrid/_app/commands/cmd_execute.py
    ... ... @@ -22,8 +22,11 @@ Request work to be executed and monitor status of jobs.
    22 22
    23 23 
     import errno
    24 24 
     import logging
    25 
    -import stat
    26 25 
     import os
    26 
    +import stat
    27 
    +import sys
    28 
    +from urllib.parse import urlparse
    29 
    +
    27 30 
     import click
    28 31 
     import grpc
    29 32
    ... ... @@ -36,20 +39,36 @@ from ..cli import pass_context
    36 39
    37 40
    38 41 
     @click.group(name='execute', short_help="Execute simple operations.")
    39 
    -@click.option('--instance-name', type=click.STRING, default='main',
    40 
    -              show_default=True, help="Targeted farm instance name.")
    41 
    -@click.option('--port', type=click.INT, default='50051', show_default=True,
    42 
    -              help="Remote server's port number.")
    43 
    -@click.option('--host', type=click.STRING, default='localhost', show_default=True,
    44 
    -              help="Remote server's hostname.")
    42 
    +@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    43 
    +              help="Remote execution server's URL (port defaults to 50051 if no specified).")
    44 
    +@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    45 
    +              help="Private client key for TLS (PEM-encoded)")
    46 
    +@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    47 
    +              help="Public client certificate for TLS (PEM-encoded)")
    48 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    49 
    +              help="Public server certificate for TLS (PEM-encoded)")
    50 
    +@click.option('--instance-name', type=click.STRING, default='main', show_default=True,
    51 
    +              help="Targeted farm instance name.")
    45 52 
     @pass_context
    46 
    -def cli(context, instance_name, host, port):
    47 
    -    context.logger = logging.getLogger(__name__)
    48 
    -    context.logger.info("Starting on port {}".format(port))
    53 
    +def cli(context, remote, instance_name, client_key, client_cert, server_cert):
    54 
    +    url = urlparse(remote)
    49 55
    56 
    +    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    50 57 
         context.instance_name = instance_name
    51 
    -    context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
    52 
    -    context.port = port
    58 
    +
    59 
    +    if url.scheme == 'http':
    60 
    +        context.channel = grpc.insecure_channel(context.remote)
    61 
    +    else:
    62 
    +        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    63 
    +        if not credentials:
    64 
    +            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    65 
    +                       "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    66 
    +            sys.exit(-1)
    67 
    +
    68 
    +        context.channel = grpc.secure_channel(context.remote, credentials)
    69 
    +
    70 
    +    context.logger = logging.getLogger(__name__)
    71 
    +    context.logger.debug("Starting for remote {}".format(context.remote))
    53 72
    54 73
    55 74 
     @cli.command('request-dummy', short_help="Send a dummy action.")

  • buildgrid/_app/commands/cmd_server.py
    ... ... @@ -22,6 +22,7 @@ Create a BuildGrid server.
    22 22
    23 23 
     import asyncio
    24 24 
     import logging
    25 
    +import sys
    25 26
    26 27 
     import click
    27 28
    ... ... @@ -41,18 +42,25 @@ _SIZE_PREFIXES = {'k': 2 ** 10, 'm': 2 ** 20, 'g': 2 ** 30, 't': 2 ** 40}
    41 42 
     @pass_context
    42 43 
     def cli(context):
    43 44 
         context.logger = logging.getLogger(__name__)
    44 
    -    context.logger.info("BuildGrid server booting up")
    45 45
    46 46
    47 47 
     @cli.command('start', short_help="Setup a new server instance.")
    48 48 
     @click.argument('instances', nargs=-1, type=click.STRING)
    49 49 
     @click.option('--port', type=click.INT, default='50051', show_default=True,
    50 50 
                   help="The port number to be listened.")
    51 
    -@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
    52 
    -              help="Maximum number of actions to keep in the ActionCache.")
    51 
    +@click.option('--server-key', type=click.Path(exists=True, dir_okay=False), default=None,
    52 
    +              help="Private server key for TLS (PEM-encoded)")
    53 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    54 
    +              help="Public server certificate for TLS (PEM-encoded)")
    55 
    +@click.option('--client-certs', type=click.Path(exists=True, dir_okay=False), default=None,
    56 
    +              help="Public client certificates for TLS (PEM-encoded)")
    57 
    +@click.option('--allow-insecure', type=click.BOOL, is_flag=True,
    58 
    +              help="Whether or not to allow unencrypted connections.")
    53 59 
     @click.option('--allow-update-action-result/--forbid-update-action-result',
    54 60 
                   'allow_uar', default=True, show_default=True,
    55 61 
                   help="Whether or not to allow clients to manually edit the action cache.")
    62 
    +@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
    63 
    +              help="Maximum number of actions to keep in the ActionCache.")
    56 64 
     @click.option('--cas', type=click.Choice(('lru', 's3', 'disk', 'with-cache')),
    57 65 
                   help="The CAS storage type to use.")
    58 66 
     @click.option('--cas-cache', type=click.Choice(('lru', 's3', 'disk')),
    ... ... @@ -68,9 +76,21 @@ def cli(context):
    68 76 
     @click.option('--cas-disk-directory', type=click.Path(file_okay=False, dir_okay=True, writable=True),
    69 77 
                   help="For --cas=disk, the folder to store CAS blobs in.")
    70 78 
     @pass_context
    71 
    -def start(context, instances, port, max_cached_actions, allow_uar, cas, **cas_args):
    72 
    -    """ Starts a BuildGrid server.
    73 
    -    """
    79 
    +def start(context, port, allow_insecure, server_key, server_cert, client_certs,
    80 
    +          instances, max_cached_actions, allow_uar, cas, **cas_args):
    81 
    +    """Setups a new server instance."""
    82 
    +    credentials = None
    83 
    +    if not allow_insecure:
    84 
    +        credentials = context.load_server_credentials(server_key, server_cert, client_certs)
    85 
    +    if not credentials and not allow_insecure:
    86 
    +        click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    87 
    +                   "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    88 
    +        sys.exit(-1)
    89 
    +
    90 
    +    context.credentials = credentials
    91 
    +    context.port = port
    92 
    +
    93 
    +    context.logger.info("BuildGrid server booting up")
    74 94 
         context.logger.info("Starting on port {}".format(port))
    75 95
    76 96 
         cas_storage = _make_cas_storage(context, cas, cas_args)
    ... ... @@ -85,8 +105,9 @@ def start(context, instances, port, max_cached_actions, allow_uar, cas, **cas_ar
    85 105 
         if instances is None:
    86 106 
             instances = ['main']
    87 107
    88 
    -    server = buildgrid_server.BuildGridServer(port,
    89 
    -                                              instances,
    108 
    +    server = buildgrid_server.BuildGridServer(port=context.port,
    109 
    +                                              credentials=context.credentials,
    110 
    +                                              instances=instances,
    90 111 
                                                   cas_storage=cas_storage,
    91 112 
                                                   action_cache=action_cache)
    92 113 
         loop = asyncio.get_event_loop()

  • buildgrid/server/buildgrid_server.py
    ... ... @@ -40,11 +40,16 @@ from .worker.bots_service import BotsService
    40 40
    41 41 
     class BuildGridServer:
    42 42
    43 
    -    def __init__(self, port='50051', instances=None, max_workers=10, action_cache=None, cas_storage=None):
    44 
    -        port = '[::]:{0}'.format(port)
    43 
    +    def __init__(self, port=50051, credentials=None, instances=None,
    44 
    +                 max_workers=10, action_cache=None, cas_storage=None):
    45 
    +        address = '[::]:{0}'.format(port)
    45 46
    46 47 
             self._server = grpc.server(futures.ThreadPoolExecutor(max_workers))
    47 
    -        self._server.add_insecure_port(port)
    48 
    +
    49 
    +        if credentials is not None:
    50 
    +            self._server.add_secure_port(address, credentials)
    51 
    +        else:
    52 
    +            self._server.add_insecure_port(address)
    48 53
    49 54 
             if cas_storage is not None:
    50 55 
                 cas_service = ContentAddressableStorageService(cas_storage)

  • buildgrid/server/scheduler.py
    ... ... @@ -90,7 +90,7 @@ class Scheduler:
    90 90 
             job.update_execute_stage(ExecuteStage.COMPLETED)
    91 91 
             self.jobs[name] = job
    92 92 
             if not job.do_not_cache and self._action_cache is not None:
    93 
    -            self._action_cache.put_action_result(job.action_digest, result)
    93 
    +            self._action_cache.update_action_result(job.action_digest, result)
    94 94
    95 95 
         def get_operations(self):
    96 96 
             response = operations_pb2.ListOperationsResponse()

  • docs/source/using_dummy_build.rst
    ... ... @@ -8,7 +8,7 @@ In one terminal, start a server:
    8 8
    9 9 
     .. code-block:: sh
    10 10
    11 
    -   bgd server start
    11 
    +   bgd server start --allow-insecure
    12 12
    13 13 
     In another terminal, send a request for work:
    14 14

  • docs/source/using_simple_build.rst
    ... ... @@ -27,7 +27,7 @@ Now start a BuildGrid server, passing it a directory it can write a CAS to:
    27 27
    28 28 
     .. code-block:: sh
    29 29
    30 
    -   bgd server start --cas disk --cas-cache disk --cas-disk-directory /path/to/empty/directory
    30 
    +   bgd server start --allow-insecure --cas disk --cas-cache disk --cas-disk-directory /path/to/empty/directory
    31 31
    32 32 
     Start the following bot session:
    33 33

  • setup.py
    ... ... @@ -114,8 +114,8 @@ setup(
    114 114 
             'protobuf',
    115 115 
             'grpcio',
    116 116 
             'Click',
    117 
    -        'boto3',
    118 
    -        'botocore',
    117 
    +        'boto3 < 1.8.0',
    118 
    +        'botocore < 1.11.0',
    119 119 
         ],
    120 120 
         entry_points={
    121 121 
             'console_scripts': [

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]