GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][mablanch/61-bazel-support] 11 commits: Add server-side gRPC TLS encryption support

Title: GitLab

Martin Blanchard pushed to branch mablanch/61-bazel-support at BuildGrid / buildgrid

Commits:

16 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -28,10 +28,10 @@ before_script:
    28 28 
     .run-dummy-job-template: &dummy-job
    29 29 
       stage: test
    30 30 
       script:
    31 
    -    - ${BGD} server start &
    31 
    +    - ${BGD} server start --allow-insecure &
    32 32 
         - sleep 1 # Allow server to boot
    33 
    -    - ${BGD} bot --host=0.0.0.0 dummy &
    34 
    -    - ${BGD} execute --host=0.0.0.0 request-dummy --wait-for-completion
    33 
    +    - ${BGD} bot dummy &
    34 
    +    - ${BGD} execute request-dummy --wait-for-completion
    35 35
    36 36 
     tests-debian-stretch:
    37 37 
       <<: *linux-tests

  • buildgrid/_app/bots/buildbox.py
    ... ... @@ -16,13 +16,12 @@
    16 16 
     import os
    17 17 
     import subprocess
    18 18 
     import tempfile
    19 
    -import grpc
    20 19
    21 20 
     from google.protobuf import any_pb2
    22 21
    23 22 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    24 23 
     from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
    25 
    -from buildgrid.utils import read_file, parse_to_pb2_from_fetch
    24 
    +from buildgrid.utils import parse_to_pb2_from_fetch
    26 25
    27 26
    28 27 
     def work_buildbox(context, lease):
    ... ... @@ -32,18 +31,7 @@ def work_buildbox(context, lease):
    32 31 
         action_digest = remote_execution_pb2.Digest()
    33 32 
         action_digest_any.Unpack(action_digest)
    34 33
    35 
    -    cert_server = read_file(context.server_cert)
    36 
    -    cert_client = read_file(context.client_cert)
    37 
    -    key_client = read_file(context.client_key)
    38 
    -
    39 
    -    # create server credentials
    40 
    -    credentials = grpc.ssl_channel_credentials(root_certificates=cert_server,
    41 
    -                                               private_key=key_client,
    42 
    -                                               certificate_chain=cert_client)
    43 
    -
    44 
    -    channel = grpc.secure_channel('{}:{}'.format(context.remote, context.port), credentials)
    45 
    -
    46 
    -    stub = bytestream_pb2_grpc.ByteStreamStub(channel)
    34 
    +    stub = bytestream_pb2_grpc.ByteStreamStub(context.cas_channel)
    47 35
    48 36 
         action = remote_execution_pb2.Action()
    49 37 
         parse_to_pb2_from_fetch(action, stub, action_digest)
    ... ... @@ -66,13 +54,18 @@ def work_buildbox(context, lease):
    66 54
    67 55 
             with tempfile.NamedTemporaryFile(dir=os.path.join(casdir, 'tmp')) as output_digest_file:
    68 56 
                 command = ['buildbox',
    69 
    -                       '--remote={}'.format('https://{}:{}'.format(context.remote, context.port)),
    70 
    -                       '--server-cert={}'.format(context.server_cert),
    71 
    -                       '--client-key={}'.format(context.client_key),
    72 
    -                       '--client-cert={}'.format(context.client_cert),
    57 
    +                       '--remote={}'.format(context.remote_cas_url),
    73 58 
                            '--input-digest={}'.format(input_digest_file.name),
    74 59 
                            '--output-digest={}'.format(output_digest_file.name),
    75 60 
                            '--local={}'.format(casdir)]
    61 
    +
    62 
    +            if context.cas_client_key:
    63 
    +                command.append('--client-key={}'.format(context.cas_client_key))
    64 
    +            if context.cas_client_cert:
    65 
    +                command.append('--client-cert={}'.format(context.cas_client_cert))
    66 
    +            if context.cas_server_cert:
    67 
    +                command.append('--server-cert={}'.format(context.cas_server_cert))
    68 
    +
    76 69 
                 if 'PWD' in environment and environment['PWD']:
    77 70 
                     command.append('--chdir={}'.format(environment['PWD']))
    78 71

  • buildgrid/_app/bots/temp_directory.py
    ... ... @@ -19,70 +19,96 @@ import tempfile
    19 19
    20 20 
     from google.protobuf import any_pb2
    21 21
    22 
    -from buildgrid.utils import read_file, create_digest, write_fetch_directory, parse_to_pb2_from_fetch
    22 
    +from buildgrid.utils import write_fetch_directory, parse_to_pb2_from_fetch
    23 
    +from buildgrid.utils import output_file_maker, output_directory_maker
    23 24 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    24 25 
     from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
    25 26
    26 27
    27 28 
     def work_temp_directory(context, lease):
    28 
    -    """ Bot downloads directories and files into a temp directory,
    29 
    -    then uploads results back to CAS
    29 
    +    """Executes a lease for a build action, using host tools.
    30 30 
         """
    31 31
    32 
    -    parent = context.parent
    33 
    -    stub_bytestream = bytestream_pb2_grpc.ByteStreamStub(context.channel)
    32 
    +    instance_name = context.parent
    33 
    +    stub_bytestream = bytestream_pb2_grpc.ByteStreamStub(context.cas_channel)
    34 34
    35 35 
         action_digest = remote_execution_pb2.Digest()
    36 36 
         lease.payload.Unpack(action_digest)
    37 37
    38 
    -    action = remote_execution_pb2.Action()
    38 
    +    action = parse_to_pb2_from_fetch(remote_execution_pb2.Action(),
    39 
    +                                     stub_bytestream, action_digest, instance_name)
    39 40
    40 
    -    action = parse_to_pb2_from_fetch(action, stub_bytestream, action_digest, parent)
    41 
    +    with tempfile.TemporaryDirectory() as temp_directory:
    42 
    +        command = parse_to_pb2_from_fetch(remote_execution_pb2.Command(),
    43 
    +                                          stub_bytestream, action.command_digest, instance_name)
    41 44
    42 
    -    with tempfile.TemporaryDirectory() as temp_dir:
    45 
    +        write_fetch_directory(temp_directory, stub_bytestream,
    46 
    +                              action.input_root_digest, instance_name)
    43 47
    44 
    -        command = remote_execution_pb2.Command()
    45 
    -        command = parse_to_pb2_from_fetch(command, stub_bytestream, action.command_digest, parent)
    46 
    -
    47 
    -        arguments = "cd {} &&".format(temp_dir)
    48 
    +        execution_envionment = os.environ.copy()
    49 
    +        for variable in command.environment_variables:
    50 
    +            if variable.name not in ['PATH', 'PWD']:
    51 
    +                execution_envionment[variable.name] = variable.value
    48 52
    53 
    +        command_arguments = list()
    49 54 
             for argument in command.arguments:
    50 
    -            arguments += " {}".format(argument)
    51 
    -
    52 
    -        context.logger.info(arguments)
    53 
    -
    54 
    -        write_fetch_directory(temp_dir, stub_bytestream, action.input_root_digest, parent)
    55 
    -
    56 
    -        proc = subprocess.Popen(arguments,
    57 
    -                                shell=True,
    58 
    -                                stdin=subprocess.PIPE,
    59 
    -                                stdout=subprocess.PIPE)
    60 
    -
    61 
    -        # TODO: Should return the std_out to the user
    62 
    -        proc.communicate()
    63 
    -
    64 
    -        result = remote_execution_pb2.ActionResult()
    65 
    -        requests = []
    66 
    -        for output_file in command.output_files:
    67 
    -            path = os.path.join(temp_dir, output_file)
    68 
    -            chunk = read_file(path)
    69 
    -
    70 
    -            digest = create_digest(chunk)
    71 
    -
    72 
    -            result.output_files.extend([remote_execution_pb2.OutputFile(path=output_file,
    73 
    -                                                                        digest=digest)])
    74 
    -
    75 
    -            requests.append(remote_execution_pb2.BatchUpdateBlobsRequest.Request(
    76 
    -                digest=digest, data=chunk))
    77 
    -
    78 
    -        request = remote_execution_pb2.BatchUpdateBlobsRequest(instance_name=parent,
    79 
    -                                                               requests=requests)
    80 
    -
    81 
    -        stub_cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(context.channel)
    82 
    -        stub_cas.BatchUpdateBlobs(request)
    55 
    +            command_arguments.append(argument.strip())
    56 
    +
    57 
    +        working_directory = None
    58 
    +        if command.working_directory:
    59 
    +            working_directory = os.path.join(temp_directory,
    60 
    +                                             command.working_directory)
    61 
    +            os.makedirs(working_directory, exist_ok=True)
    62 
    +        else:
    63 
    +            working_directory = temp_directory
    64 
    +
    65 
    +        # Ensure that output files structure exists:
    66 
    +        for output_path in command.output_files:
    67 
    +            directory_path = os.path.join(working_directory,
    68 
    +                                          os.path.dirname(output_path))
    69 
    +            os.makedirs(directory_path, exist_ok=True)
    70 
    +
    71 
    +        process = subprocess.Popen(command_arguments,
    72 
    +                                   cwd=working_directory,
    73 
    +                                   universal_newlines=True,
    74 
    +                                   env=execution_envionment,
    75 
    +                                   stdin=subprocess.PIPE,
    76 
    +                                   stdout=subprocess.PIPE)
    77 
    +        # TODO: Should return the stdout and stderr to the user.
    78 
    +        process.communicate()
    79 
    +
    80 
    +        update_requests = remote_execution_pb2.BatchUpdateBlobsRequest(instance_name=instance_name)
    81 
    +        action_result = remote_execution_pb2.ActionResult()
    82 
    +
    83 
    +        for output_path in command.output_files:
    84 
    +            file_path = os.path.join(working_directory, output_path)
    85 
    +            # Missing outputs should simply be omitted in ActionResult:
    86 
    +            if not os.path.isfile(file_path):
    87 
    +                continue
    88 
    +
    89 
    +            # OutputFile.path should be relative to the working direcory:
    90 
    +            output_file, update_request = output_file_maker(file_path, working_directory)
    91 
    +
    92 
    +            action_result.output_files.extend([output_file])
    93 
    +            update_requests.requests.extend([update_request])
    94 
    +
    95 
    +        for output_path in command.output_directories:
    96 
    +            directory_path = os.path.join(working_directory, output_path)
    97 
    +            # Missing outputs should simply be omitted in ActionResult:
    98 
    +            if not os.path.isdir(directory_path):
    99 
    +                continue
    100 
    +
    101 
    +            # OutputDirectory.path should be relative to the working direcory:
    102 
    +            output_directory, update_request = output_directory_maker(directory_path, working_directory)
    103 
    +
    104 
    +            action_result.output_directories.extend([output_directory])
    105 
    +            update_requests.requests.extend(update_request)
    106 
    +
    107 
    +        stub_cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(context.cas_channel)
    108 
    +        stub_cas.BatchUpdateBlobs(update_requests)
    83 109
    84 110 
             result_any = any_pb2.Any()
    85 
    -        result_any.Pack(result)
    111 
    +        result_any.Pack(action_result)
    86 112
    87 113 
             lease.result.CopyFrom(result_any)
    88 114

  • buildgrid/_app/cli.py
    ... ... @@ -25,6 +25,10 @@ import os
    25 25 
     import logging
    26 26
    27 27 
     import click
    28 
    +import grpc
    29 
    +from xdg import XDG_CACHE_HOME, XDG_CONFIG_HOME, XDG_DATA_HOME
    30 
    +
    31 
    +from buildgrid.utils import read_file
    28 32
    29 33 
     from . import _logging
    30 34
    ... ... @@ -35,7 +39,118 @@ class Context:
    35 39
    36 40 
         def __init__(self):
    37 41 
             self.verbose = False
    38 
    -        self.home = os.getcwd()
    42 
    +
    43 
    +        self.user_home = os.getcwd()
    44 
    +
    45 
    +        self.cache_home = os.path.join(XDG_CACHE_HOME, 'buildgrid')
    46 
    +        self.config_home = os.path.join(XDG_CONFIG_HOME, 'buildgrid')
    47 
    +        self.data_home = os.path.join(XDG_DATA_HOME, 'buildgrid')
    48 
    +
    49 
    +    def load_client_credentials(self, client_key=None, client_cert=None,
    50 
    +                                server_cert=None, use_default_client_keys=False):
    51 
    +        """Looks-up and loads TLS client gRPC credentials.
    52 
    +
    53 
    +        Args:
    54 
    +            client_key(str): root certificate file path.
    55 
    +            client_cert(str): private key file path.
    56 
    +            server_cert(str): certificate chain file path.
    57 
    +            use_default_client_keys(bool, optional): whether or not to try
    58 
    +                loading client keys from default location. Defaults to False.
    59 
    +
    60 
    +        Returns:
    61 
    +            :obj:`ChannelCredentials`: The credentials for use for a
    62 
    +            TLS-encrypted gRPC client channel.
    63 
    +        """
    64 
    +        if not client_key or not os.path.exists(client_key):
    65 
    +            if use_default_client_keys:
    66 
    +                client_key = os.path.join(self.config_home, 'client.key')
    67 
    +            else:
    68 
    +                client_key = None
    69 
    +
    70 
    +        if not client_cert or not os.path.exists(client_cert):
    71 
    +            if use_default_client_keys:
    72 
    +                client_cert = os.path.join(self.config_home, 'client.crt')
    73 
    +            else:
    74 
    +                client_cert = None
    75 
    +
    76 
    +        if not server_cert or not os.path.exists(server_cert):
    77 
    +            server_cert = os.path.join(self.config_home, 'server.crt')
    78 
    +            if not os.path.exists(server_cert):
    79 
    +                return None
    80 
    +
    81 
    +        server_cert_pem = read_file(server_cert)
    82 
    +        if client_key and os.path.exists(client_key):
    83 
    +            client_key_pem = read_file(client_key)
    84 
    +        else:
    85 
    +            client_key_pem = None
    86 
    +            client_key = None
    87 
    +        if client_key_pem and client_cert and os.path.exists(client_cert):
    88 
    +            client_cert_pem = read_file(client_cert)
    89 
    +        else:
    90 
    +            client_cert_pem = None
    91 
    +            client_cert = None
    92 
    +
    93 
    +        credentials = grpc.ssl_channel_credentials(root_certificates=server_cert_pem,
    94 
    +                                                   private_key=client_key_pem,
    95 
    +                                                   certificate_chain=client_cert_pem)
    96 
    +
    97 
    +        credentials.client_key = client_key
    98 
    +        credentials.client_cert = client_cert
    99 
    +        credentials.server_cert = server_cert
    100 
    +
    101 
    +        return credentials
    102 
    +
    103 
    +    def load_server_credentials(self, server_key=None, server_cert=None,
    104 
    +                                client_certs=None, use_default_client_certs=False):
    105 
    +        """Looks-up and loads TLS server gRPC credentials.
    106 
    +
    107 
    +        Every private and public keys are expected to be PEM-encoded.
    108 
    +
    109 
    +        Args:
    110 
    +            server_key(str): private server key file path.
    111 
    +            server_cert(str): public server certificate file path.
    112 
    +            client_certs(str): public client certificates file path.
    113 
    +            use_default_client_certs(bool, optional): whether or not to try
    114 
    +                loading public client certificates from default location.
    115 
    +                Defaults to False.
    116 
    +
    117 
    +        Returns:
    118 
    +            :obj:`ServerCredentials`: The credentials for use for a
    119 
    +            TLS-encrypted gRPC server channel.
    120 
    +        """
    121 
    +        if not server_key or not os.path.exists(server_key):
    122 
    +            server_key = os.path.join(self.config_home, 'server.key')
    123 
    +            if not os.path.exists(server_key):
    124 
    +                return None
    125 
    +
    126 
    +        if not server_cert or not os.path.exists(server_cert):
    127 
    +            server_cert = os.path.join(self.config_home, 'server.crt')
    128 
    +            if not os.path.exists(server_cert):
    129 
    +                return None
    130 
    +
    131 
    +        if not client_certs or not os.path.exists(client_certs):
    132 
    +            if use_default_client_certs:
    133 
    +                client_certs = os.path.join(self.config_home, 'client.crt')
    134 
    +            else:
    135 
    +                client_certs = None
    136 
    +
    137 
    +        server_key_pem = read_file(server_key)
    138 
    +        server_cert_pem = read_file(server_cert)
    139 
    +        if client_certs and os.path.exists(client_certs):
    140 
    +            client_certs_pem = read_file(client_certs)
    141 
    +        else:
    142 
    +            client_certs_pem = None
    143 
    +            client_certs = None
    144 
    +
    145 
    +        credentials = grpc.ssl_server_credentials([(server_key_pem, server_cert_pem)],
    146 
    +                                                  root_certificates=client_certs_pem,
    147 
    +                                                  require_client_auth=bool(client_certs))
    148 
    +
    149 
    +        credentials.server_key = server_key
    150 
    +        credentials.server_cert = server_cert
    151 
    +        credentials.client_certs = client_certs
    152 
    +
    153 
    +        return credentials
    39 154
    40 155
    41 156 
     pass_context = click.make_pass_decorator(Context, ensure=True)

  • buildgrid/_app/commands/cmd_bot.py
    ... ... @@ -21,8 +21,9 @@ Create a bot interface and request work
    21 21 
     """
    22 22
    23 23 
     import logging
    24 
    -
    25 24 
     from pathlib import Path, PurePath
    25 
    +import sys
    26 
    +from urllib.parse import urlparse
    26 27
    27 28 
     import click
    28 29 
     import grpc
    ... ... @@ -35,22 +36,92 @@ from ..cli import pass_context
    35 36
    36 37
    37 38 
     @click.group(name='bot', short_help="Create and register bot clients.")
    39 
    +@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    40 
    +              help="Remote execution server's URL (port defaults to 50051 if not specified).")
    41 
    +@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    42 
    +              help="Private client key for TLS (PEM-encoded)")
    43 
    +@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    44 
    +              help="Public client certificate for TLS (PEM-encoded)")
    45 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    46 
    +              help="Public server certificate for TLS (PEM-encoded)")
    47 
    +@click.option('--remote-cas', type=click.STRING, default=None, show_default=True,
    48 
    +              help="Remote CAS server's URL (port defaults to 11001 if not specified).")
    49 
    +@click.option('--cas-client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    50 
    +              help="Private CAS client key for TLS (PEM-encoded)")
    51 
    +@click.option('--cas-client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    52 
    +              help="Public CAS client certificate for TLS (PEM-encoded)")
    53 
    +@click.option('--cas-server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    54 
    +              help="Public CAS server certificate for TLS (PEM-encoded)")
    38 55 
     @click.option('--parent', type=click.STRING, default='main', show_default=True,
    39 56 
                   help="Targeted farm resource.")
    40 
    -@click.option('--port', type=click.INT, default='50051', show_default=True,
    41 
    -              help="Remote server's port number.")
    42 
    -@click.option('--host', type=click.STRING, default='localhost', show_default=True,
    43 
    -              help="Renote server's hostname.")
    44 57 
     @pass_context
    45 
    -def cli(context, host, port, parent):
    46 
    -    channel = grpc.insecure_channel('{}:{}'.format(host, port))
    47 
    -    interface = bot_interface.BotInterface(channel)
    58 
    +def cli(context, parent, remote, client_key, client_cert, server_cert,
    59 
    +        remote_cas, cas_client_key, cas_client_cert, cas_server_cert):
    60 
    +    # Setup the remote execution server channel:
    61 
    +    url = urlparse(remote)
    48 62
    49 
    -    context.logger = logging.getLogger(__name__)
    50 
    -    context.logger.info("Starting on port {}".format(port))
    51 
    -    context.channel = channel
    63 
    +    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    64 
    +    context.remote_url = remote
    52 65 
         context.parent = parent
    53 66
    67 
    +    if url.scheme == 'http':
    68 
    +        context.channel = grpc.insecure_channel(context.remote)
    69 
    +
    70 
    +        context.client_key = None
    71 
    +        context.client_cert = None
    72 
    +        context.server_cert = None
    73 
    +    else:
    74 
    +        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    75 
    +        if not credentials:
    76 
    +            click.echo("ERROR: no TLS keys were specified and no defaults could be found.", err=True)
    77 
    +            sys.exit(-1)
    78 
    +
    79 
    +        context.channel = grpc.secure_channel(context.remote, credentials)
    80 
    +
    81 
    +        context.client_key = credentials.client_key
    82 
    +        context.client_cert = credentials.client_cert
    83 
    +        context.server_cert = credentials.server_cert
    84 
    +
    85 
    +    # Setup the remote CAS server channel, if separated:
    86 
    +    if remote_cas is not None and remote_cas != remote:
    87 
    +        cas_url = urlparse(remote_cas)
    88 
    +
    89 
    +        context.remote_cas = '{}:{}'.format(cas_url.hostname, cas_url.port or 11001)
    90 
    +        context.remote_cas_url = remote_cas
    91 
    +
    92 
    +        if cas_url.scheme == 'http':
    93 
    +            context.cas_channel = grpc.insecure_channel(context.remote_cas)
    94 
    +
    95 
    +            context.cas_client_key = None
    96 
    +            context.cas_client_cert = None
    97 
    +            context.cas_server_cert = None
    98 
    +        else:
    99 
    +            cas_credentials = context.load_client_credentials(cas_client_key, cas_client_cert, cas_server_cert)
    100 
    +            if not cas_credentials:
    101 
    +                click.echo("ERROR: no TLS keys were specified and no defaults could be found.", err=True)
    102 
    +                sys.exit(-1)
    103 
    +
    104 
    +            context.cas_channel = grpc.secure_channel(context.remote_cas, cas_credentials)
    105 
    +
    106 
    +            context.cas_client_key = cas_credentials.client_key
    107 
    +            context.cas_client_cert = cas_credentials.client_cert
    108 
    +            context.cas_server_cert = cas_credentials.server_cert
    109 
    +
    110 
    +    else:
    111 
    +        context.remote_cas = context.remote
    112 
    +        context.remote_cas_url = remote
    113 
    +
    114 
    +        context.cas_channel = context.channel
    115 
    +
    116 
    +        context.cas_client_key = context.client_key
    117 
    +        context.cas_client_cert = context.client_cert
    118 
    +        context.cas_server_cert = context.server_cert
    119 
    +
    120 
    +    context.logger = logging.getLogger(__name__)
    121 
    +    context.logger.debug("Starting for remote {}".format(context.remote))
    122 
    +
    123 
    +    interface = bot_interface.BotInterface(context.channel)
    124 
    +
    54 125 
         worker = Worker()
    55 126 
         worker.add_device(Device())
    56 127
    ... ... @@ -94,35 +165,17 @@ def run_temp_directory(context):
    94 165 
                   help="Main mount-point location.")
    95 166 
     @click.option('--local-cas', type=click.Path(readable=False), default=str(PurePath(Path.home(), 'cas')),
    96 167 
                   help="Local CAS cache directory.")
    97 
    -@click.option('--client-cert', type=click.Path(readable=False), default=str(PurePath(Path.home(), 'client.crt')),
    98 
    -              help="Public client certificate for TLS (PEM-encoded).")
    99 
    -@click.option('--client-key', type=click.Path(readable=False), default=str(PurePath(Path.home(), 'client.key')),
    100 
    -              help="Private client key for TLS (PEM-encoded).")
    101 
    -@click.option('--server-cert', type=click.Path(readable=False), default=str(PurePath(Path.home(), 'server.crt')),
    102 
    -              help="Public server certificate for TLS (PEM-encoded).")
    103 
    -@click.option('--port', type=click.INT, default=11001, show_default=True,
    104 
    -              help="Remote CAS server port.")
    105 
    -@click.option('--remote', type=click.STRING, default='localhost', show_default=True,
    106 
    -              help="Remote CAS server hostname.")
    107 168 
     @pass_context
    108 
    -def run_buildbox(context, remote, port, server_cert, client_key, client_cert, local_cas, fuse_dir):
    169 
    +def run_buildbox(context, local_cas, fuse_dir):
    109 170 
         """
    110 171 
         Uses BuildBox to run commands.
    111 172 
         """
    112 
    -
    113 
    -    context.logger.info("Creating a bot session")
    114 
    -
    115 
    -    context.remote = remote
    116 
    -    context.port = port
    117 
    -    context.server_cert = server_cert
    118 
    -    context.client_key = client_key
    119 
    -    context.client_cert = client_cert
    120 173 
         context.local_cas = local_cas
    121 174 
         context.fuse_dir = fuse_dir
    122 175
    123 176 
         try:
    124 177 
             b = bot.Bot(context.bot_session)
    125 
    -        b.session(work=buildbox.work_buildbox,
    126 
    -                  context=context)
    178 
    +        b.session(buildbox.work_buildbox,
    179 
    +                  context)
    127 180 
         except KeyboardInterrupt:
    128 181 
             pass

  • buildgrid/_app/commands/cmd_cas.py
    ... ... @@ -21,6 +21,9 @@ Request work to be executed and monitor status of jobs.
    21 21 
     """
    22 22
    23 23 
     import logging
    24 
    +import sys
    25 
    +from urllib.parse import urlparse
    26 
    +
    24 27 
     import click
    25 28 
     import grpc
    26 29
    ... ... @@ -31,20 +34,36 @@ from ..cli import pass_context
    31 34
    32 35
    33 36 
     @click.group(name='cas', short_help="Interact with the CAS server.")
    37 
    +@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    38 
    +              help="Remote execution server's URL (port defaults to 50051 if no specified).")
    39 
    +@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    40 
    +              help="Private client key for TLS (PEM-encoded)")
    41 
    +@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    42 
    +              help="Public client certificate for TLS (PEM-encoded)")
    43 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    44 
    +              help="Public server certificate for TLS (PEM-encoded)")
    34 45 
     @click.option('--instance-name', type=click.STRING, default='main', show_default=True,
    35 46 
                   help="Targeted farm instance name.")
    36 
    -@click.option('--port', type=click.INT, default='50051', show_default=True,
    37 
    -              help="Remote server's port number.")
    38 
    -@click.option('--host', type=click.STRING, default='localhost', show_default=True,
    39 
    -              help="Remote server's hostname.")
    40 47 
     @pass_context
    41 
    -def cli(context, instance_name, host, port):
    42 
    -    context.logger = logging.getLogger(__name__)
    43 
    -    context.logger.info("Starting on port {}".format(port))
    48 
    +def cli(context, remote, instance_name, client_key, client_cert, server_cert):
    49 
    +    url = urlparse(remote)
    44 50
    51 
    +    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    45 52 
         context.instance_name = instance_name
    46 
    -    context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
    47 
    -    context.port = port
    53 
    +
    54 
    +    if url.scheme == 'http':
    55 
    +        context.channel = grpc.insecure_channel(context.remote)
    56 
    +    else:
    57 
    +        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    58 
    +        if not credentials:
    59 
    +            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    60 
    +                       "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    61 
    +            sys.exit(-1)
    62 
    +
    63 
    +        context.channel = grpc.secure_channel(context.remote, credentials)
    64 
    +
    65 
    +    context.logger = logging.getLogger(__name__)
    66 
    +    context.logger.debug("Starting for remote {}".format(context.remote))
    48 67
    49 68
    50 69 
     @cli.command('upload-files', short_help="Upload files to the CAS server.")

  • buildgrid/_app/commands/cmd_execute.py
    ... ... @@ -22,8 +22,11 @@ Request work to be executed and monitor status of jobs.
    22 22
    23 23 
     import errno
    24 24 
     import logging
    25 
    -import stat
    26 25 
     import os
    26 
    +import stat
    27 
    +import sys
    28 
    +from urllib.parse import urlparse
    29 
    +
    27 30 
     import click
    28 31 
     import grpc
    29 32
    ... ... @@ -36,20 +39,36 @@ from ..cli import pass_context
    36 39
    37 40
    38 41 
     @click.group(name='execute', short_help="Execute simple operations.")
    39 
    -@click.option('--instance-name', type=click.STRING, default='main',
    40 
    -              show_default=True, help="Targeted farm instance name.")
    41 
    -@click.option('--port', type=click.INT, default='50051', show_default=True,
    42 
    -              help="Remote server's port number.")
    43 
    -@click.option('--host', type=click.STRING, default='localhost', show_default=True,
    44 
    -              help="Remote server's hostname.")
    42 
    +@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    43 
    +              help="Remote execution server's URL (port defaults to 50051 if no specified).")
    44 
    +@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    45 
    +              help="Private client key for TLS (PEM-encoded)")
    46 
    +@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    47 
    +              help="Public client certificate for TLS (PEM-encoded)")
    48 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    49 
    +              help="Public server certificate for TLS (PEM-encoded)")
    50 
    +@click.option('--instance-name', type=click.STRING, default='main', show_default=True,
    51 
    +              help="Targeted farm instance name.")
    45 52 
     @pass_context
    46 
    -def cli(context, instance_name, host, port):
    47 
    -    context.logger = logging.getLogger(__name__)
    48 
    -    context.logger.info("Starting on port {}".format(port))
    53 
    +def cli(context, remote, instance_name, client_key, client_cert, server_cert):
    54 
    +    url = urlparse(remote)
    49 55
    56 
    +    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    50 57 
         context.instance_name = instance_name
    51 
    -    context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
    52 
    -    context.port = port
    58 
    +
    59 
    +    if url.scheme == 'http':
    60 
    +        context.channel = grpc.insecure_channel(context.remote)
    61 
    +    else:
    62 
    +        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    63 
    +        if not credentials:
    64 
    +            click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    65 
    +                       "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    66 
    +            sys.exit(-1)
    67 
    +
    68 
    +        context.channel = grpc.secure_channel(context.remote, credentials)
    69 
    +
    70 
    +    context.logger = logging.getLogger(__name__)
    71 
    +    context.logger.debug("Starting for remote {}".format(context.remote))
    53 72
    54 73
    55 74 
     @cli.command('request-dummy', short_help="Send a dummy action.")

  • buildgrid/_app/commands/cmd_server.py
    ... ... @@ -22,6 +22,7 @@ Create a BuildGrid server.
    22 22
    23 23 
     import asyncio
    24 24 
     import logging
    25 
    +import sys
    25 26
    26 27 
     import click
    27 28
    ... ... @@ -41,18 +42,25 @@ _SIZE_PREFIXES = {'k': 2 ** 10, 'm': 2 ** 20, 'g': 2 ** 30, 't': 2 ** 40}
    41 42 
     @pass_context
    42 43 
     def cli(context):
    43 44 
         context.logger = logging.getLogger(__name__)
    44 
    -    context.logger.info("BuildGrid server booting up")
    45 45
    46 46
    47 47 
     @cli.command('start', short_help="Setup a new server instance.")
    48 48 
     @click.argument('instances', nargs=-1, type=click.STRING)
    49 49 
     @click.option('--port', type=click.INT, default='50051', show_default=True,
    50 50 
                   help="The port number to be listened.")
    51 
    -@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
    52 
    -              help="Maximum number of actions to keep in the ActionCache.")
    51 
    +@click.option('--server-key', type=click.Path(exists=True, dir_okay=False), default=None,
    52 
    +              help="Private server key for TLS (PEM-encoded)")
    53 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    54 
    +              help="Public server certificate for TLS (PEM-encoded)")
    55 
    +@click.option('--client-certs', type=click.Path(exists=True, dir_okay=False), default=None,
    56 
    +              help="Public client certificates for TLS (PEM-encoded, one single file)")
    57 
    +@click.option('--allow-insecure', type=click.BOOL, is_flag=True,
    58 
    +              help="Whether or not to allow unencrypted connections.")
    53 59 
     @click.option('--allow-update-action-result/--forbid-update-action-result',
    54 60 
                   'allow_uar', default=True, show_default=True,
    55 61 
                   help="Whether or not to allow clients to manually edit the action cache.")
    62 
    +@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
    63 
    +              help="Maximum number of actions to keep in the ActionCache.")
    56 64 
     @click.option('--cas', type=click.Choice(('lru', 's3', 'disk', 'with-cache')),
    57 65 
                   help="The CAS storage type to use.")
    58 66 
     @click.option('--cas-cache', type=click.Choice(('lru', 's3', 'disk')),
    ... ... @@ -68,9 +76,21 @@ def cli(context):
    68 76 
     @click.option('--cas-disk-directory', type=click.Path(file_okay=False, dir_okay=True, writable=True),
    69 77 
                   help="For --cas=disk, the folder to store CAS blobs in.")
    70 78 
     @pass_context
    71 
    -def start(context, instances, port, max_cached_actions, allow_uar, cas, **cas_args):
    72 
    -    """ Starts a BuildGrid server.
    73 
    -    """
    79 
    +def start(context, port, allow_insecure, server_key, server_cert, client_certs,
    80 
    +          instances, max_cached_actions, allow_uar, cas, **cas_args):
    81 
    +    """Setups a new server instance."""
    82 
    +    credentials = None
    83 
    +    if not allow_insecure:
    84 
    +        credentials = context.load_server_credentials(server_key, server_cert, client_certs)
    85 
    +    if not credentials and not allow_insecure:
    86 
    +        click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
    87 
    +                   "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
    88 
    +        sys.exit(-1)
    89 
    +
    90 
    +    context.credentials = credentials
    91 
    +    context.port = port
    92 
    +
    93 
    +    context.logger.info("BuildGrid server booting up")
    74 94 
         context.logger.info("Starting on port {}".format(port))
    75 95
    76 96 
         cas_storage = _make_cas_storage(context, cas, cas_args)
    ... ... @@ -85,8 +105,9 @@ def start(context, instances, port, max_cached_actions, allow_uar, cas, **cas_ar
    85 105 
         if instances is None:
    86 106 
             instances = ['main']
    87 107
    88 
    -    server = buildgrid_server.BuildGridServer(port,
    89 
    -                                              instances,
    108 
    +    server = buildgrid_server.BuildGridServer(port=context.port,
    109 
    +                                              credentials=context.credentials,
    110 
    +                                              instances=instances,
    90 111 
                                                   cas_storage=cas_storage,
    91 112 
                                                   action_cache=action_cache)
    92 113 
         loop = asyncio.get_event_loop()

  • buildgrid/server/buildgrid_server.py
    ... ... @@ -40,11 +40,16 @@ from .worker.bots_service import BotsService
    40 40
    41 41 
     class BuildGridServer:
    42 42
    43 
    -    def __init__(self, port='50051', instances=None, max_workers=10, action_cache=None, cas_storage=None):
    44 
    -        port = '[::]:{0}'.format(port)
    43 
    +    def __init__(self, port=50051, credentials=None, instances=None,
    44 
    +                 max_workers=10, action_cache=None, cas_storage=None):
    45 
    +        address = '[::]:{0}'.format(port)
    45 46
    46 47 
             self._server = grpc.server(futures.ThreadPoolExecutor(max_workers))
    47 
    -        self._server.add_insecure_port(port)
    48 
    +
    49 
    +        if credentials is not None:
    50 
    +            self._server.add_secure_port(address, credentials)
    51 
    +        else:
    52 
    +            self._server.add_insecure_port(address)
    48 53
    49 54 
             if cas_storage is not None:
    50 55 
                 cas_service = ContentAddressableStorageService(cas_storage)

  • buildgrid/server/execution/execution_service.py
    ... ... @@ -86,6 +86,11 @@ class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
    86 86 
                 yield operations_pb2.Operation()
    87 87
    88 88 
         def _get_instance(self, name):
    89 
    +        # If client does not support multiple instances, it may omit the
    90 
    +        # instance name request parameter, so better map our default:
    91 
    +        if not name and len(self._instances) == 1:
    92 
    +            name = 'main'
    93 
    +
    89 94 
             try:
    90 95 
                 return self._instances[name]
    91 96

  • buildgrid/utils.py
    ... ... @@ -13,6 +13,7 @@
    13 13 
     # limitations under the License.
    14 14
    15 15
    16 
    +from operator import attrgetter
    16 17 
     import os
    17 18
    18 19 
     from buildgrid.settings import HASH
    ... ... @@ -31,30 +32,59 @@ def gen_fetch_blob(stub, digest, instance_name=""):
    31 32 
             yield response.data
    32 33
    33 34
    34 
    -def write_fetch_directory(directory, stub, digest, instance_name=""):
    35 
    -    """ Given a directory digest, fetches files and writes them to a directory
    35 
    +def write_fetch_directory(root_directory, stub, digest, instance_name=None):
    36 
    +    """Locally replicates a directory from CAS.
    37 
    +
    38 
    +    Args:
    39 
    +        root_directory (str): local directory to populate.
    40 
    +        stub (): gRPC stub for CAS communication.
    41 
    +        digest (Digest): digest for the directory to fetch from CAS.
    42 
    +        instance_name (str, optional): farm instance name to query data from.
    36 43 
         """
    37 
    -    # TODO: Extend to symlinks and inner directories
    38 
    -    # pathlib.Path('/my/directory').mkdir(parents=True, exist_ok=True)
    44 
    +    if not os.path.isabs(root_directory):
    45 
    +        root_directory = os.path.abspath(root_directory)
    46 
    +    if not os.path.exists(root_directory):
    47 
    +        os.makedirs(root_directory, exist_ok=True)
    39 48
    40 
    -    directory_pb2 = remote_execution_pb2.Directory()
    41 
    -    directory_pb2 = parse_to_pb2_from_fetch(directory_pb2, stub, digest, instance_name)
    49 
    +    directory = parse_to_pb2_from_fetch(remote_execution_pb2.Directory(),
    50 
    +                                        stub, digest, instance_name)
    51 
    +
    52 
    +    for directory_node in directory.directories:
    53 
    +        child_path = os.path.join(root_directory, directory_node.name)
    54 
    +
    55 
    +        write_fetch_directory(child_path, stub, directory_node.digest, instance_name)
    56 
    +
    57 
    +    for file_node in directory.files:
    58 
    +        child_path = os.path.join(root_directory, file_node.name)
    59 
    +
    60 
    +        with open(child_path, 'wb') as child_file:
    61 
    +            write_fetch_blob(child_file, stub, file_node.digest, instance_name)
    62 
    +
    63 
    +    for symlink_node in directory.symlinks:
    64 
    +        child_path = os.path.join(root_directory, symlink_node.name)
    65 
    +
    66 
    +        if os.path.isabs(symlink_node.target):
    67 
    +            continue  # No out of temp-directory links for now.
    68 
    +        target_path = os.path.join(root_directory, symlink_node.target)
    69 
    +
    70 
    +        os.symlink(child_path, target_path)
    42 71
    43 
    -    for file_node in directory_pb2.files:
    44 
    -        path = os.path.join(directory, file_node.name)
    45 
    -        with open(path, 'wb') as f:
    46 
    -            write_fetch_blob(f, stub, file_node.digest, instance_name)
    47 72
    73 
    +def write_fetch_blob(target_file, stub, digest, instance_name=None):
    74 
    +    """Extracts a blob from CAS into a local file.
    48 75
    49 
    -def write_fetch_blob(out, stub, digest, instance_name=""):
    50 
    -    """ Given an output buffer, fetches blob and writes to buffer
    76 
    +    Args:
    77 
    +        target_file (str): local file to write.
    78 
    +        stub (): gRPC stub for CAS communication.
    79 
    +        digest (Digest): digest for the blob to fetch from CAS.
    80 
    +        instance_name (str, optional): farm instance name to query data from.
    51 81 
         """
    52 82
    53 83 
         for stream in gen_fetch_blob(stub, digest, instance_name):
    54 
    -        out.write(stream)
    84 
    +        target_file.write(stream)
    85 
    +    target_file.flush()
    55 86
    56 
    -    out.flush()
    57 
    -    assert digest.size_bytes == os.fstat(out.fileno()).st_size
    87 
    +    assert digest.size_bytes == os.fstat(target_file.fileno()).st_size
    58 88
    59 89
    60 90 
     def parse_to_pb2_from_fetch(pb2, stub, digest, instance_name=""):
    ... ... @@ -70,7 +100,15 @@ def parse_to_pb2_from_fetch(pb2, stub, digest, instance_name=""):
    70 100
    71 101
    72 102 
     def create_digest(bytes_to_digest):
    73 
    -    """ Creates a hash based on the hex digest and returns the digest
    103 
    +    """Computes the :obj:`Digest` of a piece of data.
    104 
    +
    105 
    +    The :obj:`Digest` of a data is a function of its hash **and** size.
    106 
    +
    107 
    +    Args:
    108 
    +        bytes_to_digest (bytes): byte data to digest.
    109 
    +
    110 
    +    Returns:
    111 
    +        :obj:`Digest`: The gRPC :obj:`Digest` for the given byte data.
    74 112 
         """
    75 113 
         return remote_execution_pb2.Digest(hash=HASH(bytes_to_digest).hexdigest(),
    76 114 
                                            size_bytes=len(bytes_to_digest))
    ... ... @@ -107,6 +145,199 @@ def file_maker(file_path, file_digest):
    107 145 
                                              is_executable=os.access(file_path, os.X_OK))
    108 146
    109 147
    110 
    -def read_file(read):
    111 
    -    with open(read, 'rb') as f:
    112 
    -        return f.read()
    148 
    +def directory_maker(directory_path):
    149 
    +    """Creates a :obj:`Directory` from a local directory.
    150 
    +
    151 
    +    Args:
    152 
    +        directory_path (str): absolute or relative path to a local directory.
    153 
    +
    154 
    +    Returns:
    155 
    +        :obj:`Directory`, list of :obj:`Directory`, list of
    156 
    +        :obj:`BatchUpdateBlobsRequest`: Tuple of a new gRPC :obj:`Directory` for
    157 
    +        the directory pointed by `directory_path`, a list of new gRPC
    158 
    +        :obj:`Directory` for every children of that directory and the
    159 
    +        corresponding list of :obj:`BatchUpdateBlobsRequest` for CAS upload.
    160 
    +
    161 
    +        The :obj:`Directory` children list may come in any order.
    162 
    +
    163 
    +        The :obj:`BatchUpdateBlobsRequest` list may come in any order. However,
    164 
    +        its last element is guaranteed to be the root :obj:`Direcotry`'s
    165 
    +        request.
    166 
    +    """
    167 
    +    if not os.path.isabs(directory_path):
    168 
    +        directory_path = os.path.abspath(directory_path)
    169 
    +
    170 
    +    child_directories = list()
    171 
    +    update_requests = list()
    172 
    +
    173 
    +    files, directories, symlinks = list(), list(), list()
    174 
    +    for directory_entry in os.scandir(directory_path):
    175 
    +        # Create a FileNode and corresponding BatchUpdateBlobsRequest:
    176 
    +        if directory_entry.is_file(follow_symlinks=False):
    177 
    +            node_blob = read_file(directory_entry.path)
    178 
    +            node_digest = create_digest(node_blob)
    179 
    +
    180 
    +            node = remote_execution_pb2.FileNode()
    181 
    +            node.name = directory_entry.name
    182 
    +            node.digest = node_digest
    183 
    +            node.is_executable = os.access(directory_entry.path, os.X_OK)
    184 
    +
    185 
    +            node_request = remote_execution_pb2.BatchUpdateBlobsRequest.Request(digest=node_digest)
    186 
    +            node_request.data = node_blob
    187 
    +
    188 
    +            update_requests.append(node_request)
    189 
    +            files.append(node)
    190 
    +
    191 
    +        # Create a DirectoryNode and corresponding BatchUpdateBlobsRequest:
    192 
    +        elif directory_entry.is_dir(follow_symlinks=False):
    193 
    +            node_directory, node_children, node_requests = directory_maker(directory_entry.path)
    194 
    +
    195 
    +            node = remote_execution_pb2.DirectoryNode()
    196 
    +            node.name = directory_entry.name
    197 
    +            node.digest = node_requests[-1].digest
    198 
    +
    199 
    +            child_directories.extend(node_children)
    200 
    +            child_directories.append(node_directory)
    201 
    +            update_requests.extend(node_requests)
    202 
    +            directories.append(node)
    203 
    +
    204 
    +        # Create a SymlinkNode if necessary;
    205 
    +        elif os.path.islink(directory_entry.path):
    206 
    +            node_target = os.readlink(directory_entry.path)
    207 
    +
    208 
    +            node = remote_execution_pb2.SymlinkNode()
    209 
    +            node.name = directory_entry.name
    210 
    +            node.target = node_target
    211 
    +
    212 
    +            symlinks.append(node)
    213 
    +
    214 
    +    directory = remote_execution_pb2.Directory()
    215 
    +    directory.files.extend(files.sort(key=attrgetter('name')))
    216 
    +    directory.directories.extend(directories.sort(key=attrgetter('name')))
    217 
    +    directory.symlinks.extend(symlinks.sort(key=attrgetter('name')))
    218 
    +
    219 
    +    directory_blob = directory.SerializeToString()
    220 
    +    directory_digest = create_digest(directory_blob)
    221 
    +
    222 
    +    update_request = remote_execution_pb2.BatchUpdateBlobsRequest.Request(digest=directory_digest)
    223 
    +    update_request.data = directory_blob
    224 
    +
    225 
    +    update_requests.append(update_request)
    226 
    +
    227 
    +    return directory, child_directories, update_requests
    228 
    +
    229 
    +
    230 
    +def read_file(file_path):
    231 
    +    """Loads raw file content in memory.
    232 
    +
    233 
    +    Returns:
    234 
    +        bytes: Raw file's content until EOF.
    235 
    +
    236 
    +    Raises:
    237 
    +        OSError: If `file_path` does not exist or is not readable.
    238 
    +    """
    239 
    +    with open(file_path, 'rb') as byte_file:
    240 
    +        return byte_file.read()
    241 
    +
    242 
    +
    243 
    +def output_file_maker(file_path, input_path):
    244 
    +    """Creates an :obj:`OutputFile` from a local file.
    245 
    +
    246 
    +    `file_path` **must** point inside or be relative to `input_path`.
    247 
    +
    248 
    +    Args:
    249 
    +        file_path (str): absolute or relative path to a local file.
    250 
    +        input_path (str): absolute or relative path to the input root directory.
    251 
    +
    252 
    +    Returns:
    253 
    +        :obj:`OutputFile`, :obj:`BatchUpdateBlobsRequest`: Tuple of a new gRPC
    254 
    +        :obj:`OutputFile` object for the file pointed by `file_path` and the
    255 
    +        corresponding :obj:`BatchUpdateBlobsRequest` for CAS upload.
    256 
    +    """
    257 
    +    if not os.path.isabs(file_path):
    258 
    +        file_path = os.path.abspath(file_path)
    259 
    +    if not os.path.isabs(input_path):
    260 
    +        input_path = os.path.abspath(input_path)
    261 
    +
    262 
    +    file_blob = read_file(file_path)
    263 
    +    file_digest = create_digest(file_blob)
    264 
    +
    265 
    +    output_file = remote_execution_pb2.OutputFile(digest=file_digest)
    266 
    +    output_file.path = os.path.relpath(file_path, start=input_path)
    267 
    +    output_file.is_executable = os.access(file_path, os.X_OK)
    268 
    +
    269 
    +    update_request = remote_execution_pb2.BatchUpdateBlobsRequest.Request(digest=file_digest)
    270 
    +    update_request.data = file_blob
    271 
    +
    272 
    +    return output_file, update_request
    273 
    +
    274 
    +
    275 
    +def output_directory_maker(directory_path, working_path):
    276 
    +    """Creates an :obj:`OutputDirectory` from a local directory.
    277 
    +
    278 
    +    `directory_path` **must** point inside or be relative to `input_path`.
    279 
    +
    280 
    +    Args:
    281 
    +        directory_path (str): absolute or relative path to a local directory.
    282 
    +        working_path (str): absolute or relative path to the working directory.
    283 
    +
    284 
    +    Returns:
    285 
    +        :obj:`OutputDirectory`, :obj:`BatchUpdateBlobsRequest`: Tuple of a new
    286 
    +        gRPC :obj:`OutputDirectory` for the directory pointed by
    287 
    +        `directory_path` and the corresponding list of
    288 
    +        :obj:`BatchUpdateBlobsRequest` for CAS upload.
    289 
    +    """
    290 
    +    if not os.path.isabs(directory_path):
    291 
    +        directory_path = os.path.abspath(directory_path)
    292 
    +    if not os.path.isabs(working_path):
    293 
    +        working_path = os.path.abspath(working_path)
    294 
    +
    295 
    +    _, update_requests = tree_maker(directory_path)
    296 
    +
    297 
    +    output_directory = remote_execution_pb2.OutputDirectory()
    298 
    +    output_directory.tree_digest = update_requests[-1].digest
    299 
    +    output_directory.path = os.path.relpath(directory_path, start=working_path)
    300 
    +
    301 
    +    output_directory_blob = output_directory.SerializeToString()
    302 
    +    output_directory_digest = create_digest(output_directory_blob)
    303 
    +
    304 
    +    update_request = remote_execution_pb2.BatchUpdateBlobsRequest.Request(digest=output_directory_digest)
    305 
    +    update_request.data = output_directory_blob
    306 
    +
    307 
    +    update_requests.append(update_request)
    308 
    +
    309 
    +    return output_directory, update_requests
    310 
    +
    311 
    +
    312 
    +def tree_maker(directory_path):
    313 
    +    """Creates a :obj:`Tree` from a local directory.
    314 
    +
    315 
    +    Args:
    316 
    +        directory_path (str): absolute or relative path to a local directory.
    317 
    +
    318 
    +    Returns:
    319 
    +        :obj:`Tree`, :obj:`BatchUpdateBlobsRequest`: Tuple of a new
    320 
    +        gRPC :obj:`Tree` for the directory pointed by `directory_path` and the
    321 
    +        corresponding list of :obj:`BatchUpdateBlobsRequest` for CAS upload.
    322 
    +
    323 
    +        The :obj:`BatchUpdateBlobsRequest` list may come in any order. However,
    324 
    +        its last element is guaranteed to be the :obj:`Tree`'s request.
    325 
    +    """
    326 
    +    if not os.path.isabs(directory_path):
    327 
    +        directory_path = os.path.abspath(directory_path)
    328 
    +
    329 
    +    directory, child_directories, update_requests = directory_maker(directory_path)
    330 
    +
    331 
    +    tree = remote_execution_pb2.Tree()
    332 
    +    tree.children.expend([child_directories])
    333 
    +    tree.root = directory
    334 
    +
    335 
    +    tree_blob = tree.SerializeToString()
    336 
    +    tree_digest = create_digest(tree_blob)
    337 
    +
    338 
    +    update_request = remote_execution_pb2.BatchUpdateBlobsRequest.Request(digest=tree_digest)
    339 
    +    update_request.data = tree_blob
    340 
    +
    341 
    +    update_requests.append(update_request)
    342 
    +
    343 
    +    return tree, update_requests

  • docs/source/configuration.rst
    1 
    +
    2 
    +.. _configuration:
    3 
    +
    4 
    +Configuration
    5 
    +=============
    6 
    +
    7 
    +The details of how to tune BuildGrid's configuration.
    8 
    +
    9 
    +
    10 
    +.. _configuration-location:
    11 
    +
    12 
    +Configuration location
    13 
    +----------------------
    14 
    +
    15 
    +Unless a configuration file is explicitly specified on the command line when
    16 
    +invoking `bgd`, BuildGrid will always attempt to load configuration resources
    17 
    +from ``$XDG_CONFIG_HOME/buildgrid``. On most Linux based systems, the location
    18 
    +will be ``~/.config/buildgrid``.
    19 
    +
    20 
    +This location is refered as ``$CONFIG_HOME`` is the rest of the document.
    21 
    +
    22 
    +
    23 
    +.. _tls-encryption:
    24 
    +
    25 
    +TLS encryption
    26 
    +--------------
    27 
    +
    28 
    +Every BuildGrid gRPC communication channel can be encrypted using SSL/TLS. By
    29 
    +default, the BuildGrid server will try to setup secure gRPC endpoints and return
    30 
    +in error if that fails. You must specify ``--allow-insecure`` explicitly if you
    31 
    +want it to use non-encrypted connections.
    32 
    +
    33 
    +The TLS protocol handshake relies on an asymmetric cryptography system that
    34 
    +requires the server and the client to own a public/private key pair. BuildGrid
    35 
    +will try to load keys from these locations by default:
    36 
    +
    37 
    +- Server private key: ``$CONFIG_HOME/server.key``
    38 
    +- Server public key/certificate: ``$CONFIG_HOME/server.crt``
    39 
    +- Client private key: ``$CONFIG_HOME/client.key``
    40 
    +- Client public key/certificate: ``$CONFIG_HOME/client.crt``
    41 
    +
    42 
    +
    43 
    +Server key pair
    44 
    +~~~~~~~~~~~~~~~
    45 
    +
    46 
    +The TLS protocol requires a key pair to be used by the server. The following
    47 
    +example generates a self-signed key ``server.key``, which requires clients to
    48 
    +have a copy of the server certificate ``server.crt``. You can of course use a
    49 
    +key pair obtained from a trusted certificate authority instead.
    50 
    +
    51 
    +.. code-block:: sh
    52 
    +
    53 
    +   openssl req -new -newkey rsa:4096 -x509 -sha256 -days 3650 -nodes -batch -subj "/CN=localhost" -out server.crt -keyout server.key
    54 
    +
    55 
    +
    56 
    +Client key pair
    57 
    +~~~~~~~~~~~~~~~
    58 
    +
    59 
    +If the server requires authentication in order to be granted special permissions
    60 
    +like uploading to CAS, a client side key pair is required. The following example
    61 
    +generates a self-signed key ``client.key``, which requires the server to have a
    62 
    +copy of the client certificate ``client.crt``.
    63 
    +
    64 
    +.. code-block:: sh
    65 
    +
    66 
    +   openssl req -new -newkey rsa:4096 -x509 -sha256 -days 3650 -nodes -batch -subj "/CN=client" -out client.crt -keyout client.key

  • docs/source/index.rst
    ... ... @@ -15,6 +15,7 @@ Remote execution service implementing Google's REAPI and RWAPI.
    15 15
    16 16 
        about.rst
    17 17 
        installation.rst
    18 
    +   configuration.rst
    18 19 
        using.rst
    19 20 
        reference.rst
    20 21 
        contributing.rst

  • docs/source/using_dummy_build.rst
    ... ... @@ -8,7 +8,7 @@ In one terminal, start a server:
    8 8
    9 9 
     .. code-block:: sh
    10 10
    11 
    -   bgd server start
    11 
    +   bgd server start --allow-insecure
    12 12
    13 13 
     In another terminal, send a request for work:
    14 14

  • docs/source/using_simple_build.rst
    ... ... @@ -27,7 +27,7 @@ Now start a BuildGrid server, passing it a directory it can write a CAS to:
    27 27
    28 28 
     .. code-block:: sh
    29 29
    30 
    -   bgd server start --cas disk --cas-cache disk --cas-disk-directory /path/to/empty/directory
    30 
    +   bgd server start --allow-insecure --cas disk --cas-cache disk --cas-disk-directory /path/to/empty/directory
    31 31
    32 32 
     Start the following bot session:
    33 33

  • setup.py
    ... ... @@ -116,6 +116,7 @@ setup(
    116 116 
             'Click',
    117 117 
             'boto3 < 1.8.0',
    118 118 
             'botocore < 1.11.0',
    119 
    +        'xdg',
    119 120 
         ],
    120 121 
         entry_points={
    121 122 
             'console_scripts': [

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]