GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][mablanch/63-tls-encryption] 2 commits: Add server-side gRPC TLS support

Title: GitLab

Martin Blanchard pushed to branch mablanch/63-tls-encryption at BuildGrid / buildgrid

Commits:

8 changed files:

Changes:

  • buildgrid/_app/cli.py
    ... ... @@ -25,6 +25,9 @@ import os
    25 25 
     import logging
    26 26
    27 27 
     import click
    28 
    +import grpc
    29 
    +
    30 
    +from buildgrid.utils import read_file
    28 31
    29 32 
     from . import _logging
    30 33
    ... ... @@ -35,7 +38,114 @@ class Context:
    35 38
    36 39 
         def __init__(self):
    37 40 
             self.verbose = False
    38 
    -        self.home = os.getcwd()
    41 
    +
    42 
    +        self.user_home = os.getcwd()
    43 
    +
    44 
    +        self.user_cache_home = os.environ.get('XDG_CACHE_HOME')
    45 
    +        if not self.user_cache_home:
    46 
    +            self.user_cache_home = os.path.expanduser('~/.cache')
    47 
    +        self.cache_home = os.path.join(self.user_cache_home, 'buildgrid')
    48 
    +
    49 
    +        self.user_config_home = os.environ.get('XDG_CONFIG_HOME')
    50 
    +        if not self.user_config_home:
    51 
    +            self.user_config_home = os.path.expanduser('~/.config')
    52 
    +        self.config_home = os.path.join(self.user_config_home, 'buildgrid')
    53 
    +
    54 
    +        self.user_data_home = os.environ.get('XDG_DATA_HOME')
    55 
    +        if not self.user_data_home:
    56 
    +            self.user_data_home = os.path.expanduser('~/.local/share')
    57 
    +        self.data_home = os.path.join(self.user_data_home, 'buildgrid')
    58 
    +
    59 
    +    def load_client_credentials(self, client_key=None, client_cert=None,
    60 
    +                                server_cert=None, use_default_client_keys=False):
    61 
    +        """Looks-up and loads TLS client gRPC credentials.
    62 
    +
    63 
    +        Args:
    64 
    +            client_key(str): root certificate file path.
    65 
    +            client_cert(str): private key file path.
    66 
    +            server_cert(str): certificate chain file path.
    67 
    +            use_default_client_keys(bool, optional): whether or not to try
    68 
    +                loading client keys from default location. Defaults to False.
    69 
    +
    70 
    +        Returns:
    71 
    +            :obj:`ChannelCredentials`: The credentials for use for a
    72 
    +            TLS-encrypted gRPC client channel.
    73 
    +        """
    74 
    +        if not client_key or not os.path.exists(client_key):
    75 
    +            if use_default_client_keys:
    76 
    +                client_key = os.path.join(self.config_home, 'client.key')
    77 
    +            else:
    78 
    +                client_key = None
    79 
    +
    80 
    +        if not client_cert or not os.path.exists(client_cert):
    81 
    +            if use_default_client_keys:
    82 
    +                client_cert = os.path.join(self.config_home, 'client.crt')
    83 
    +            else:
    84 
    +                client_cert = None
    85 
    +
    86 
    +        if not server_cert or not os.path.exists(server_cert):
    87 
    +            server_cert = os.path.join(self.config_home, 'server.crt')
    88 
    +            if not os.path.exists(server_cert):
    89 
    +                return None
    90 
    +
    91 
    +        server_cert_pem = read_file(server_cert)
    92 
    +        if client_key and os.path.exists(client_key):
    93 
    +            client_key_pem = read_file(client_key)
    94 
    +        else:
    95 
    +            client_key_pem = None
    96 
    +        if client_key_pem and client_cert and os.path.exists(client_cert):
    97 
    +            client_cert_pem = read_file(client_cert)
    98 
    +        else:
    99 
    +            client_cert_pem = None
    100 
    +
    101 
    +        return grpc.ssl_channel_credentials(root_certificates=server_cert_pem,
    102 
    +                                            private_key=client_key_pem,
    103 
    +                                            certificate_chain=client_cert_pem)
    104 
    +
    105 
    +    def load_server_credentials(self, server_key=None, server_cert=None,
    106 
    +                                client_certs=None, use_default_client_certs=False):
    107 
    +        """Looks-up and loads TLS server gRPC credentials.
    108 
    +
    109 
    +        Every private and publick keys are expected to be PEM-encoded.
    110 
    +
    111 
    +        Args:
    112 
    +            server_key(str): private server key file path.
    113 
    +            server_cert(str): public server certificate file path.
    114 
    +            client_certs(str): public client certificates file path.
    115 
    +            use_default_client_certs(bool, optional): whether or not to try
    116 
    +                loading public client certificates from default location.
    117 
    +                Defaults to False.
    118 
    +
    119 
    +        Returns:
    120 
    +            :obj:`ServerCredentials`: The credentials for use for a
    121 
    +            TLS-encrypted gRPC server channel.
    122 
    +        """
    123 
    +        if not server_key or not os.path.exists(server_key):
    124 
    +            server_key = os.path.join(self.config_home, 'server.key')
    125 
    +            if not os.path.exists(server_key):
    126 
    +                return None
    127 
    +
    128 
    +        if not server_cert or not os.path.exists(server_cert):
    129 
    +            server_cert = os.path.join(self.config_home, 'server.crt')
    130 
    +            if not os.path.exists(server_cert):
    131 
    +                return None
    132 
    +
    133 
    +        if not client_certs or not os.path.exists(client_certs):
    134 
    +            if use_default_client_certs:
    135 
    +                client_certs = os.path.join(self.config_home, 'client.crt')
    136 
    +            else:
    137 
    +                client_certs = None
    138 
    +
    139 
    +        server_key_pem = read_file(server_key)
    140 
    +        server_cert_pem = read_file(server_cert)
    141 
    +        if client_certs and os.path.exists(client_certs):
    142 
    +            client_certs_pem = read_file(client_certs)
    143 
    +        else:
    144 
    +            client_certs_pem = None
    145 
    +
    146 
    +        return grpc.ssl_server_credentials([(server_key_pem, server_cert_pem)],
    147 
    +                                           root_certificates=client_certs_pem,
    148 
    +                                           require_client_auth=bool(client_certs))
    39 149
    40 150
    41 151 
     pass_context = click.make_pass_decorator(Context, ensure=True)

  • buildgrid/_app/commands/cmd_bot.py
    ... ... @@ -21,8 +21,9 @@ Create a bot interface and request work
    21 21 
     """
    22 22
    23 23 
     import logging
    24 
    -
    25 24 
     from pathlib import Path, PurePath
    25 
    +import sys
    26 
    +from urllib.parse import urlparse
    26 27
    27 28 
     import click
    28 29 
     import grpc
    ... ... @@ -35,20 +36,37 @@ from ..cli import pass_context
    35 36
    36 37
    37 38 
     @click.group(name='bot', short_help="Create and register bot clients.")
    39 
    +@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    40 
    +              help="Remote execution server's URL (port defaults to 50051 if no specified).")
    41 
    +@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    42 
    +              help="Private client key for TLS (PEM-encoded)")
    43 
    +@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    44 
    +              help="Public client certificate for TLS (PEM-encoded)")
    45 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    46 
    +              help="Public server certificate for TLS (PEM-encoded)")
    38 47 
     @click.option('--parent', type=click.STRING, default='bgd_test', show_default=True,
    39 48 
                   help="Targeted farm resource.")
    40 
    -@click.option('--port', type=click.INT, default='50051', show_default=True,
    41 
    -              help="Remote server's port number.")
    42 
    -@click.option('--host', type=click.STRING, default='localhost', show_default=True,
    43 
    -              help="Renote server's hostname.")
    44 49 
     @pass_context
    45 
    -def cli(context, host, port, parent):
    46 
    -    channel = grpc.insecure_channel('{}:{}'.format(host, port))
    47 
    -    interface = bot_interface.BotInterface(channel)
    50 
    +def cli(context, remote, client_key, client_cert, server_cert, parent):
    51 
    +    url = urlparse(remote)
    52 
    +
    53 
    +    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    54 
    +
    55 
    +    if url.scheme == 'http':
    56 
    +        context.channel = grpc.insecure_channel(context.remote)
    57 
    +    else:
    58 
    +        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    59 
    +        if not credentials:
    60 
    +            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    61 
    +                       "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    62 
    +            sys.exit(-1)
    63 
    +
    64 
    +        context.channel = grpc.secure_channel(context.remote, credentials)
    48 65
    49 66 
         context.logger = logging.getLogger(__name__)
    50 
    -    context.logger.info("Starting on port {}".format(port))
    51 
    -    context.channel = channel
    67 
    +    context.logger.debug("Starting for remote {}".format(context.remote))
    68 
    +
    69 
    +    interface = bot_interface.BotInterface(context.channel)
    52 70
    53 71 
         worker = Worker()
    54 72 
         worker.add_device(Device())

  • buildgrid/_app/commands/cmd_cas.py
    ... ... @@ -21,6 +21,9 @@ Request work to be executed and monitor status of jobs.
    21 21 
     """
    22 22
    23 23 
     import logging
    24 
    +import sys
    25 
    +from urllib.parse import urlparse
    26 
    +
    24 27 
     import click
    25 28 
     import grpc
    26 29
    ... ... @@ -31,17 +34,33 @@ from ..cli import pass_context
    31 34
    32 35
    33 36 
     @click.group(name='cas', short_help="Interact with the CAS server.")
    34 
    -@click.option('--port', type=click.INT, default='50051', show_default=True,
    35 
    -              help="Remote server's port number.")
    36 
    -@click.option('--host', type=click.STRING, default='localhost', show_default=True,
    37 
    -              help="Remote server's hostname.")
    37 
    +@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    38 
    +              help="Remote execution server's URL (port defaults to 50051 if no specified).")
    39 
    +@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    40 
    +              help="Private client key for TLS (PEM-encoded)")
    41 
    +@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    42 
    +              help="Public client certificate for TLS (PEM-encoded)")
    43 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    44 
    +              help="Public server certificate for TLS (PEM-encoded)")
    38 45 
     @pass_context
    39 
    -def cli(context, host, port):
    40 
    -    context.logger = logging.getLogger(__name__)
    41 
    -    context.logger.info("Starting on port {}".format(port))
    46 
    +def cli(context, remote, client_key, client_cert, server_cert):
    47 
    +    url = urlparse(remote)
    48 
    +
    49 
    +    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    42 50
    43 
    -    context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
    44 
    -    context.port = port
    51 
    +    if url.scheme == 'http':
    52 
    +        context.channel = grpc.insecure_channel(context.remote)
    53 
    +    else:
    54 
    +        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    55 
    +        if not credentials:
    56 
    +            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    57 
    +                       "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    58 
    +            sys.exit(-1)
    59 
    +
    60 
    +        context.channel = grpc.secure_channel(context.remote, credentials)
    61 
    +
    62 
    +    context.logger = logging.getLogger(__name__)
    63 
    +    context.logger.debug("Starting for remote {}".format(context.remote))
    45 64
    46 65
    47 66 
     @cli.command('upload-files', short_help="Upload files to the CAS server.")

  • buildgrid/_app/commands/cmd_execute.py
    ... ... @@ -22,8 +22,11 @@ Request work to be executed and monitor status of jobs.
    22 22
    23 23 
     import errno
    24 24 
     import logging
    25 
    -import stat
    26 25 
     import os
    26 
    +import stat
    27 
    +import sys
    28 
    +from urllib.parse import urlparse
    29 
    +
    27 30 
     import click
    28 31 
     import grpc
    29 32
    ... ... @@ -36,17 +39,33 @@ from ..cli import pass_context
    36 39
    37 40
    38 41 
     @click.group(name='execute', short_help="Execute simple operations.")
    39 
    -@click.option('--port', type=click.INT, default='50051', show_default=True,
    40 
    -              help="Remote server's port number.")
    41 
    -@click.option('--host', type=click.STRING, default='localhost', show_default=True,
    42 
    -              help="Remote server's hostname.")
    42 
    +@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    43 
    +              help="Remote execution server's URL (port defaults to 50051 if no specified).")
    44 
    +@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    45 
    +              help="Private client key for TLS (PEM-encoded)")
    46 
    +@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    47 
    +              help="Public client certificate for TLS (PEM-encoded)")
    48 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    49 
    +              help="Public server certificate for TLS (PEM-encoded)")
    43 50 
     @pass_context
    44 
    -def cli(context, host, port):
    45 
    -    context.logger = logging.getLogger(__name__)
    46 
    -    context.logger.info("Starting on port {}".format(port))
    51 
    +def cli(context, remote, client_key, client_cert, server_cert):
    52 
    +    url = urlparse(remote)
    47 53
    48 
    -    context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
    49 
    -    context.port = port
    54 
    +    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    55 
    +
    56 
    +    if url.scheme == 'http':
    57 
    +        context.channel = grpc.insecure_channel(context.remote)
    58 
    +    else:
    59 
    +        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    60 
    +        if not credentials:
    61 
    +            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    62 
    +                       "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    63 
    +            sys.exit(-1)
    64 
    +
    65 
    +        context.channel = grpc.secure_channel(context.remote, credentials)
    66 
    +
    67 
    +    context.logger = logging.getLogger(__name__)
    68 
    +    context.logger.debug("Starting for remote {}".format(context.remote))
    50 69
    51 70
    52 71 
     @cli.command('request-dummy', short_help="Send a dummy action.")

  • buildgrid/_app/commands/cmd_server.py
    ... ... @@ -22,6 +22,7 @@ Create a BuildGrid server.
    22 22
    23 23 
     import asyncio
    24 24 
     import logging
    25 
    +import sys
    25 26
    26 27 
     import click
    27 28
    ... ... @@ -41,17 +42,24 @@ _SIZE_PREFIXES = {'k': 2 ** 10, 'm': 2 ** 20, 'g': 2 ** 30, 't': 2 ** 40}
    41 42 
     @pass_context
    42 43 
     def cli(context):
    43 44 
         context.logger = logging.getLogger(__name__)
    44 
    -    context.logger.info("BuildGrid server booting up")
    45 45
    46 46
    47 47 
     @cli.command('start', short_help="Setup a new server instance.")
    48 
    -@click.option('--port', type=click.INT, default='50051', show_default=True,
    48 
    +@click.option('--port', '-p', type=click.INT, default='50051', show_default=True,
    49 49 
                   help="The port number to be listened.")
    50 
    -@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
    51 
    -              help="Maximum number of actions to keep in the ActionCache.")
    50 
    +@click.option('--server-key', type=click.Path(exists=True, dir_okay=False), default=None,
    51 
    +              help="Private server key for TLS (PEM-encoded)")
    52 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    53 
    +              help="Public server certificate for TLS (PEM-encoded)")
    54 
    +@click.option('--client-certs', type=click.Path(exists=True, dir_okay=False), default=None,
    55 
    +              help="Public client certificates for TLS (PEM-encoded)")
    56 
    +@click.option('--allow-insecure', type=click.BOOL, is_flag=True,
    57 
    +              help="Whether or not to allow unencrypted connections.")
    52 58 
     @click.option('--allow-update-action-result/--forbid-update-action-result',
    53 59 
                   'allow_uar', default=True, show_default=True,
    54 60 
                   help="Whether or not to allow clients to manually edit the action cache.")
    61 
    +@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
    62 
    +              help="Maximum number of actions to keep in the ActionCache.")
    55 63 
     @click.option('--cas', type=click.Choice(('lru', 's3', 'disk', 'with-cache')),
    56 64 
                   help="The CAS storage type to use.")
    57 65 
     @click.option('--cas-cache', type=click.Choice(('lru', 's3', 'disk')),
    ... ... @@ -67,7 +75,21 @@ def cli(context):
    67 75 
     @click.option('--cas-disk-directory', type=click.Path(file_okay=False, dir_okay=True, writable=True),
    68 76 
                   help="For --cas=disk, the folder to store CAS blobs in.")
    69 77 
     @pass_context
    70 
    -def start(context, port, max_cached_actions, allow_uar, cas, **cas_args):
    78 
    +def start(context, port, allow_insecure, server_key, server_cert, client_certs,
    79 
    +          max_cached_actions, allow_uar, cas, **cas_args):
    80 
    +    """Setups a new server instance."""
    81 
    +    credentials = None
    82 
    +    if not allow_insecure:
    83 
    +        credentials = context.load_server_credentials(server_key, server_cert, client_certs)
    84 
    +    if not credentials and not allow_insecure:
    85 
    +        click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    86 
    +                   "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    87 
    +        sys.exit(-1)
    88 
    +
    89 
    +    context.credentials = credentials
    90 
    +    context.port = port
    91 
    +
    92 
    +    context.logger.info("BuildGrid server booting up")
    71 93 
         context.logger.info("Starting on port {}".format(port))
    72 94
    73 95 
         cas_storage = _make_cas_storage(context, cas, cas_args)
    ... ... @@ -79,7 +101,8 @@ def start(context, port, max_cached_actions, allow_uar, cas, **cas_args):
    79 101 
         else:
    80 102 
             action_cache = ActionCache(cas_storage, max_cached_actions, allow_uar)
    81 103
    82 
    -    server = build_grid_server.BuildGridServer(port,
    104 
    +    server = build_grid_server.BuildGridServer(port=context.port,
    105 
    +                                               credentials=context.credentials,
    83 106 
                                                    cas_storage=cas_storage,
    84 107 
                                                    action_cache=action_cache)
    85 108 
         loop = asyncio.get_event_loop()

  • buildgrid/server/build_grid_server.py
    ... ... @@ -42,14 +42,18 @@ from .worker.bots_interface import BotsInterface
    42 42
    43 43 
     class BuildGridServer:
    44 44
    45 
    -    def __init__(self, port='50051', max_workers=10, cas_storage=None, action_cache=None):
    45 
    +    def __init__(self, port='50051', credentials=None, max_workers=10, cas_storage=None, action_cache=None):
    46 46 
             port = '[::]:{0}'.format(port)
    47 47 
             scheduler = Scheduler(action_cache)
    48 48 
             bots_interface = BotsInterface(scheduler)
    49 49 
             execution_instance = ExecutionInstance(scheduler, cas_storage)
    50 50
    51 51 
             self._server = grpc.server(futures.ThreadPoolExecutor(max_workers))
    52 
    -        self._server.add_insecure_port(port)
    52 
    +
    53 
    +        if credentials is not None:
    54 
    +            self._server.add_secure_port(port, credentials)
    55 
    +        else:
    56 
    +            self._server.add_insecure_port(port)
    53 57
    54 58 
             bots_pb2_grpc.add_BotsServicer_to_server(BotsService(bots_interface),
    55 59 
                                                      self._server)

  • docs/source/using_dummy_build.rst
    ... ... @@ -8,7 +8,7 @@ In one terminal, start a server:
    8 8
    9 9 
     .. code-block:: sh
    10 10
    11 
    -   bgd server start
    11 
    +   bgd server start --allow-insecure
    12 12
    13 13 
     In another terminal, send a request for work:
    14 14

  • docs/source/using_simple_build.rst
    ... ... @@ -27,7 +27,7 @@ Now start a BuildGrid server, passing it a directory it can write a CAS to:
    27 27
    28 28 
     .. code-block:: sh
    29 29
    30 
    -   bgd server start --cas disk --cas-cache disk --cas-disk-directory /path/to/empty/directory
    30 
    +   bgd server start --allow-insecure --cas disk --cas-cache disk --cas-disk-directory /path/to/empty/directory
    31 31
    32 32 
     Start the following bot session:
    33 33

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]