Martin Blanchard pushed to branch mablanch/61-bazel-support at BuildGrid / buildgrid
Commits:
-
1a9ed906
by Martin Blanchard at 2018-08-29T11:21:43Z
-
cff9354d
by Martin Blanchard at 2018-08-29T11:21:45Z
-
50d2f283
by Martin Blanchard at 2018-08-29T12:27:05Z
-
9a44c825
by Martin Blanchard at 2018-08-29T12:27:07Z
-
94c7a77e
by Martin Blanchard at 2018-08-29T13:53:50Z
-
1078a53a
by Martin Blanchard at 2018-08-30T12:09:11Z
-
8036db36
by Martin Blanchard at 2018-08-30T12:09:11Z
-
0e902477
by Martin Blanchard at 2018-08-30T12:11:02Z
-
89a66aea
by Martin Blanchard at 2018-08-30T12:11:02Z
-
0dd5a638
by Martin Blanchard at 2018-08-30T12:11:49Z
-
9b769264
by Martin Blanchard at 2018-08-30T12:11:49Z
16 changed files:
- .gitlab-ci.yml
- buildgrid/_app/bots/buildbox.py
- buildgrid/_app/bots/temp_directory.py
- buildgrid/_app/cli.py
- buildgrid/_app/commands/cmd_bot.py
- buildgrid/_app/commands/cmd_cas.py
- buildgrid/_app/commands/cmd_execute.py
- buildgrid/_app/commands/cmd_server.py
- buildgrid/server/buildgrid_server.py
- buildgrid/server/execution/execution_service.py
- buildgrid/utils.py
- + docs/source/configuration.rst
- docs/source/index.rst
- docs/source/using_dummy_build.rst
- docs/source/using_simple_build.rst
- setup.py
Changes:
... | ... | @@ -28,10 +28,10 @@ before_script: |
28 | 28 |
.run-dummy-job-template: &dummy-job
|
29 | 29 |
stage: test
|
30 | 30 |
script:
|
31 |
- - ${BGD} server start &
|
|
31 |
+ - ${BGD} server start --allow-insecure &
|
|
32 | 32 |
- sleep 1 # Allow server to boot
|
33 |
- - ${BGD} bot --host=0.0.0.0 dummy &
|
|
34 |
- - ${BGD} execute --host=0.0.0.0 request-dummy --wait-for-completion
|
|
33 |
+ - ${BGD} bot dummy &
|
|
34 |
+ - ${BGD} execute request-dummy --wait-for-completion
|
|
35 | 35 |
|
36 | 36 |
tests-debian-stretch:
|
37 | 37 |
<<: *linux-tests
|
... | ... | @@ -16,13 +16,12 @@ |
16 | 16 |
import os
|
17 | 17 |
import subprocess
|
18 | 18 |
import tempfile
|
19 |
-import grpc
|
|
20 | 19 |
|
21 | 20 |
from google.protobuf import any_pb2
|
22 | 21 |
|
23 | 22 |
from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
|
24 | 23 |
from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
|
25 |
-from buildgrid.utils import read_file, parse_to_pb2_from_fetch
|
|
24 |
+from buildgrid.utils import parse_to_pb2_from_fetch
|
|
26 | 25 |
|
27 | 26 |
|
28 | 27 |
def work_buildbox(context, lease):
|
... | ... | @@ -32,18 +31,7 @@ def work_buildbox(context, lease): |
32 | 31 |
action_digest = remote_execution_pb2.Digest()
|
33 | 32 |
action_digest_any.Unpack(action_digest)
|
34 | 33 |
|
35 |
- cert_server = read_file(context.server_cert)
|
|
36 |
- cert_client = read_file(context.client_cert)
|
|
37 |
- key_client = read_file(context.client_key)
|
|
38 |
- |
|
39 |
- # create server credentials
|
|
40 |
- credentials = grpc.ssl_channel_credentials(root_certificates=cert_server,
|
|
41 |
- private_key=key_client,
|
|
42 |
- certificate_chain=cert_client)
|
|
43 |
- |
|
44 |
- channel = grpc.secure_channel('{}:{}'.format(context.remote, context.port), credentials)
|
|
45 |
- |
|
46 |
- stub = bytestream_pb2_grpc.ByteStreamStub(channel)
|
|
34 |
+ stub = bytestream_pb2_grpc.ByteStreamStub(context.cas_channel)
|
|
47 | 35 |
|
48 | 36 |
action = remote_execution_pb2.Action()
|
49 | 37 |
parse_to_pb2_from_fetch(action, stub, action_digest)
|
... | ... | @@ -66,13 +54,18 @@ def work_buildbox(context, lease): |
66 | 54 |
|
67 | 55 |
with tempfile.NamedTemporaryFile(dir=os.path.join(casdir, 'tmp')) as output_digest_file:
|
68 | 56 |
command = ['buildbox',
|
69 |
- '--remote={}'.format('https://{}:{}'.format(context.remote, context.port)),
|
|
70 |
- '--server-cert={}'.format(context.server_cert),
|
|
71 |
- '--client-key={}'.format(context.client_key),
|
|
72 |
- '--client-cert={}'.format(context.client_cert),
|
|
57 |
+ '--remote={}'.format(context.remote_cas_url),
|
|
73 | 58 |
'--input-digest={}'.format(input_digest_file.name),
|
74 | 59 |
'--output-digest={}'.format(output_digest_file.name),
|
75 | 60 |
'--local={}'.format(casdir)]
|
61 |
+ |
|
62 |
+ if context.cas_client_key:
|
|
63 |
+ command.append('--client-key={}'.format(context.cas_client_key))
|
|
64 |
+ if context.cas_client_cert:
|
|
65 |
+ command.append('--client-cert={}'.format(context.cas_client_cert))
|
|
66 |
+ if context.cas_server_cert:
|
|
67 |
+ command.append('--server-cert={}'.format(context.cas_server_cert))
|
|
68 |
+ |
|
76 | 69 |
if 'PWD' in environment and environment['PWD']:
|
77 | 70 |
command.append('--chdir={}'.format(environment['PWD']))
|
78 | 71 |
|
... | ... | @@ -19,70 +19,96 @@ import tempfile |
19 | 19 |
|
20 | 20 |
from google.protobuf import any_pb2
|
21 | 21 |
|
22 |
-from buildgrid.utils import read_file, create_digest, write_fetch_directory, parse_to_pb2_from_fetch
|
|
22 |
+from buildgrid.utils import write_fetch_directory, parse_to_pb2_from_fetch
|
|
23 |
+from buildgrid.utils import output_file_maker, output_directory_maker
|
|
23 | 24 |
from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
|
24 | 25 |
from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
|
25 | 26 |
|
26 | 27 |
|
27 | 28 |
def work_temp_directory(context, lease):
|
28 |
- """ Bot downloads directories and files into a temp directory,
|
|
29 |
- then uploads results back to CAS
|
|
29 |
+ """Executes a lease for a build action, using host tools.
|
|
30 | 30 |
"""
|
31 | 31 |
|
32 |
- parent = context.parent
|
|
33 |
- stub_bytestream = bytestream_pb2_grpc.ByteStreamStub(context.channel)
|
|
32 |
+ instance_name = context.parent
|
|
33 |
+ stub_bytestream = bytestream_pb2_grpc.ByteStreamStub(context.cas_channel)
|
|
34 | 34 |
|
35 | 35 |
action_digest = remote_execution_pb2.Digest()
|
36 | 36 |
lease.payload.Unpack(action_digest)
|
37 | 37 |
|
38 |
- action = remote_execution_pb2.Action()
|
|
38 |
+ action = parse_to_pb2_from_fetch(remote_execution_pb2.Action(),
|
|
39 |
+ stub_bytestream, action_digest, instance_name)
|
|
39 | 40 |
|
40 |
- action = parse_to_pb2_from_fetch(action, stub_bytestream, action_digest, parent)
|
|
41 |
+ with tempfile.TemporaryDirectory() as temp_directory:
|
|
42 |
+ command = parse_to_pb2_from_fetch(remote_execution_pb2.Command(),
|
|
43 |
+ stub_bytestream, action.command_digest, instance_name)
|
|
41 | 44 |
|
42 |
- with tempfile.TemporaryDirectory() as temp_dir:
|
|
45 |
+ write_fetch_directory(temp_directory, stub_bytestream,
|
|
46 |
+ action.input_root_digest, instance_name)
|
|
43 | 47 |
|
44 |
- command = remote_execution_pb2.Command()
|
|
45 |
- command = parse_to_pb2_from_fetch(command, stub_bytestream, action.command_digest, parent)
|
|
46 |
- |
|
47 |
- arguments = "cd {} &&".format(temp_dir)
|
|
48 |
+ execution_envionment = os.environ.copy()
|
|
49 |
+ for variable in command.environment_variables:
|
|
50 |
+ if variable.name not in ['PATH', 'PWD']:
|
|
51 |
+ execution_envionment[variable.name] = variable.value
|
|
48 | 52 |
|
53 |
+ command_arguments = list()
|
|
49 | 54 |
for argument in command.arguments:
|
50 |
- arguments += " {}".format(argument)
|
|
51 |
- |
|
52 |
- context.logger.info(arguments)
|
|
53 |
- |
|
54 |
- write_fetch_directory(temp_dir, stub_bytestream, action.input_root_digest, parent)
|
|
55 |
- |
|
56 |
- proc = subprocess.Popen(arguments,
|
|
57 |
- shell=True,
|
|
58 |
- stdin=subprocess.PIPE,
|
|
59 |
- stdout=subprocess.PIPE)
|
|
60 |
- |
|
61 |
- # TODO: Should return the std_out to the user
|
|
62 |
- proc.communicate()
|
|
63 |
- |
|
64 |
- result = remote_execution_pb2.ActionResult()
|
|
65 |
- requests = []
|
|
66 |
- for output_file in command.output_files:
|
|
67 |
- path = os.path.join(temp_dir, output_file)
|
|
68 |
- chunk = read_file(path)
|
|
69 |
- |
|
70 |
- digest = create_digest(chunk)
|
|
71 |
- |
|
72 |
- result.output_files.extend([remote_execution_pb2.OutputFile(path=output_file,
|
|
73 |
- digest=digest)])
|
|
74 |
- |
|
75 |
- requests.append(remote_execution_pb2.BatchUpdateBlobsRequest.Request(
|
|
76 |
- digest=digest, data=chunk))
|
|
77 |
- |
|
78 |
- request = remote_execution_pb2.BatchUpdateBlobsRequest(instance_name=parent,
|
|
79 |
- requests=requests)
|
|
80 |
- |
|
81 |
- stub_cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(context.channel)
|
|
82 |
- stub_cas.BatchUpdateBlobs(request)
|
|
55 |
+ command_arguments.append(argument.strip())
|
|
56 |
+ |
|
57 |
+ working_directory = None
|
|
58 |
+ if command.working_directory:
|
|
59 |
+ working_directory = os.path.join(temp_directory,
|
|
60 |
+ command.working_directory)
|
|
61 |
+ os.makedirs(working_directory, exist_ok=True)
|
|
62 |
+ else:
|
|
63 |
+ working_directory = temp_directory
|
|
64 |
+ |
|
65 |
+ # Ensure that output files structure exists:
|
|
66 |
+ for output_path in command.output_files:
|
|
67 |
+ directory_path = os.path.join(working_directory,
|
|
68 |
+ os.path.dirname(output_path))
|
|
69 |
+ os.makedirs(directory_path, exist_ok=True)
|
|
70 |
+ |
|
71 |
+ process = subprocess.Popen(command_arguments,
|
|
72 |
+ cwd=working_directory,
|
|
73 |
+ universal_newlines=True,
|
|
74 |
+ env=execution_envionment,
|
|
75 |
+ stdin=subprocess.PIPE,
|
|
76 |
+ stdout=subprocess.PIPE)
|
|
77 |
+ # TODO: Should return the stdout and stderr to the user.
|
|
78 |
+ process.communicate()
|
|
79 |
+ |
|
80 |
+ update_requests = remote_execution_pb2.BatchUpdateBlobsRequest(instance_name=instance_name)
|
|
81 |
+ action_result = remote_execution_pb2.ActionResult()
|
|
82 |
+ |
|
83 |
+ for output_path in command.output_files:
|
|
84 |
+ file_path = os.path.join(working_directory, output_path)
|
|
85 |
+ # Missing outputs should simply be omitted in ActionResult:
|
|
86 |
+ if not os.path.isfile(file_path):
|
|
87 |
+ continue
|
|
88 |
+ |
|
89 |
+ # OutputFile.path should be relative to the working direcory:
|
|
90 |
+ output_file, update_request = output_file_maker(file_path, working_directory)
|
|
91 |
+ |
|
92 |
+ action_result.output_files.extend([output_file])
|
|
93 |
+ update_requests.requests.extend([update_request])
|
|
94 |
+ |
|
95 |
+ for output_path in command.output_directories:
|
|
96 |
+ directory_path = os.path.join(working_directory, output_path)
|
|
97 |
+ # Missing outputs should simply be omitted in ActionResult:
|
|
98 |
+ if not os.path.isdir(directory_path):
|
|
99 |
+ continue
|
|
100 |
+ |
|
101 |
+ # OutputDirectory.path should be relative to the working direcory:
|
|
102 |
+ output_directory, update_request = output_directory_maker(directory_path, working_directory)
|
|
103 |
+ |
|
104 |
+ action_result.output_directories.extend([output_directory])
|
|
105 |
+ update_requests.requests.extend(update_request)
|
|
106 |
+ |
|
107 |
+ stub_cas = remote_execution_pb2_grpc.ContentAddressableStorageStub(context.cas_channel)
|
|
108 |
+ stub_cas.BatchUpdateBlobs(update_requests)
|
|
83 | 109 |
|
84 | 110 |
result_any = any_pb2.Any()
|
85 |
- result_any.Pack(result)
|
|
111 |
+ result_any.Pack(action_result)
|
|
86 | 112 |
|
87 | 113 |
lease.result.CopyFrom(result_any)
|
88 | 114 |
|
... | ... | @@ -25,6 +25,10 @@ import os |
25 | 25 |
import logging
|
26 | 26 |
|
27 | 27 |
import click
|
28 |
+import grpc
|
|
29 |
+from xdg import XDG_CACHE_HOME, XDG_CONFIG_HOME, XDG_DATA_HOME
|
|
30 |
+ |
|
31 |
+from buildgrid.utils import read_file
|
|
28 | 32 |
|
29 | 33 |
from . import _logging
|
30 | 34 |
|
... | ... | @@ -35,7 +39,118 @@ class Context: |
35 | 39 |
|
36 | 40 |
def __init__(self):
|
37 | 41 |
self.verbose = False
|
38 |
- self.home = os.getcwd()
|
|
42 |
+ |
|
43 |
+ self.user_home = os.getcwd()
|
|
44 |
+ |
|
45 |
+ self.cache_home = os.path.join(XDG_CACHE_HOME, 'buildgrid')
|
|
46 |
+ self.config_home = os.path.join(XDG_CONFIG_HOME, 'buildgrid')
|
|
47 |
+ self.data_home = os.path.join(XDG_DATA_HOME, 'buildgrid')
|
|
48 |
+ |
|
49 |
+ def load_client_credentials(self, client_key=None, client_cert=None,
|
|
50 |
+ server_cert=None, use_default_client_keys=False):
|
|
51 |
+ """Looks-up and loads TLS client gRPC credentials.
|
|
52 |
+ |
|
53 |
+ Args:
|
|
54 |
+ client_key(str): root certificate file path.
|
|
55 |
+ client_cert(str): private key file path.
|
|
56 |
+ server_cert(str): certificate chain file path.
|
|
57 |
+ use_default_client_keys(bool, optional): whether or not to try
|
|
58 |
+ loading client keys from default location. Defaults to False.
|
|
59 |
+ |
|
60 |
+ Returns:
|
|
61 |
+ :obj:`ChannelCredentials`: The credentials for use for a
|
|
62 |
+ TLS-encrypted gRPC client channel.
|
|
63 |
+ """
|
|
64 |
+ if not client_key or not os.path.exists(client_key):
|
|
65 |
+ if use_default_client_keys:
|
|
66 |
+ client_key = os.path.join(self.config_home, 'client.key')
|
|
67 |
+ else:
|
|
68 |
+ client_key = None
|
|
69 |
+ |
|
70 |
+ if not client_cert or not os.path.exists(client_cert):
|
|
71 |
+ if use_default_client_keys:
|
|
72 |
+ client_cert = os.path.join(self.config_home, 'client.crt')
|
|
73 |
+ else:
|
|
74 |
+ client_cert = None
|
|
75 |
+ |
|
76 |
+ if not server_cert or not os.path.exists(server_cert):
|
|
77 |
+ server_cert = os.path.join(self.config_home, 'server.crt')
|
|
78 |
+ if not os.path.exists(server_cert):
|
|
79 |
+ return None
|
|
80 |
+ |
|
81 |
+ server_cert_pem = read_file(server_cert)
|
|
82 |
+ if client_key and os.path.exists(client_key):
|
|
83 |
+ client_key_pem = read_file(client_key)
|
|
84 |
+ else:
|
|
85 |
+ client_key_pem = None
|
|
86 |
+ client_key = None
|
|
87 |
+ if client_key_pem and client_cert and os.path.exists(client_cert):
|
|
88 |
+ client_cert_pem = read_file(client_cert)
|
|
89 |
+ else:
|
|
90 |
+ client_cert_pem = None
|
|
91 |
+ client_cert = None
|
|
92 |
+ |
|
93 |
+ credentials = grpc.ssl_channel_credentials(root_certificates=server_cert_pem,
|
|
94 |
+ private_key=client_key_pem,
|
|
95 |
+ certificate_chain=client_cert_pem)
|
|
96 |
+ |
|
97 |
+ credentials.client_key = client_key
|
|
98 |
+ credentials.client_cert = client_cert
|
|
99 |
+ credentials.server_cert = server_cert
|
|
100 |
+ |
|
101 |
+ return credentials
|
|
102 |
+ |
|
103 |
+ def load_server_credentials(self, server_key=None, server_cert=None,
|
|
104 |
+ client_certs=None, use_default_client_certs=False):
|
|
105 |
+ """Looks-up and loads TLS server gRPC credentials.
|
|
106 |
+ |
|
107 |
+ Every private and public keys are expected to be PEM-encoded.
|
|
108 |
+ |
|
109 |
+ Args:
|
|
110 |
+ server_key(str): private server key file path.
|
|
111 |
+ server_cert(str): public server certificate file path.
|
|
112 |
+ client_certs(str): public client certificates file path.
|
|
113 |
+ use_default_client_certs(bool, optional): whether or not to try
|
|
114 |
+ loading public client certificates from default location.
|
|
115 |
+ Defaults to False.
|
|
116 |
+ |
|
117 |
+ Returns:
|
|
118 |
+ :obj:`ServerCredentials`: The credentials for use for a
|
|
119 |
+ TLS-encrypted gRPC server channel.
|
|
120 |
+ """
|
|
121 |
+ if not server_key or not os.path.exists(server_key):
|
|
122 |
+ server_key = os.path.join(self.config_home, 'server.key')
|
|
123 |
+ if not os.path.exists(server_key):
|
|
124 |
+ return None
|
|
125 |
+ |
|
126 |
+ if not server_cert or not os.path.exists(server_cert):
|
|
127 |
+ server_cert = os.path.join(self.config_home, 'server.crt')
|
|
128 |
+ if not os.path.exists(server_cert):
|
|
129 |
+ return None
|
|
130 |
+ |
|
131 |
+ if not client_certs or not os.path.exists(client_certs):
|
|
132 |
+ if use_default_client_certs:
|
|
133 |
+ client_certs = os.path.join(self.config_home, 'client.crt')
|
|
134 |
+ else:
|
|
135 |
+ client_certs = None
|
|
136 |
+ |
|
137 |
+ server_key_pem = read_file(server_key)
|
|
138 |
+ server_cert_pem = read_file(server_cert)
|
|
139 |
+ if client_certs and os.path.exists(client_certs):
|
|
140 |
+ client_certs_pem = read_file(client_certs)
|
|
141 |
+ else:
|
|
142 |
+ client_certs_pem = None
|
|
143 |
+ client_certs = None
|
|
144 |
+ |
|
145 |
+ credentials = grpc.ssl_server_credentials([(server_key_pem, server_cert_pem)],
|
|
146 |
+ root_certificates=client_certs_pem,
|
|
147 |
+ require_client_auth=bool(client_certs))
|
|
148 |
+ |
|
149 |
+ credentials.server_key = server_key
|
|
150 |
+ credentials.server_cert = server_cert
|
|
151 |
+ credentials.client_certs = client_certs
|
|
152 |
+ |
|
153 |
+ return credentials
|
|
39 | 154 |
|
40 | 155 |
|
41 | 156 |
pass_context = click.make_pass_decorator(Context, ensure=True)
|
... | ... | @@ -21,8 +21,9 @@ Create a bot interface and request work |
21 | 21 |
"""
|
22 | 22 |
|
23 | 23 |
import logging
|
24 |
- |
|
25 | 24 |
from pathlib import Path, PurePath
|
25 |
+import sys
|
|
26 |
+from urllib.parse import urlparse
|
|
26 | 27 |
|
27 | 28 |
import click
|
28 | 29 |
import grpc
|
... | ... | @@ -35,22 +36,92 @@ from ..cli import pass_context |
35 | 36 |
|
36 | 37 |
|
37 | 38 |
@click.group(name='bot', short_help="Create and register bot clients.")
|
39 |
+@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
|
|
40 |
+ help="Remote execution server's URL (port defaults to 50051 if not specified).")
|
|
41 |
+@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
42 |
+ help="Private client key for TLS (PEM-encoded)")
|
|
43 |
+@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
44 |
+ help="Public client certificate for TLS (PEM-encoded)")
|
|
45 |
+@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
46 |
+ help="Public server certificate for TLS (PEM-encoded)")
|
|
47 |
+@click.option('--remote-cas', type=click.STRING, default=None, show_default=True,
|
|
48 |
+ help="Remote CAS server's URL (port defaults to 11001 if not specified).")
|
|
49 |
+@click.option('--cas-client-key', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
50 |
+ help="Private CAS client key for TLS (PEM-encoded)")
|
|
51 |
+@click.option('--cas-client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
52 |
+ help="Public CAS client certificate for TLS (PEM-encoded)")
|
|
53 |
+@click.option('--cas-server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
54 |
+ help="Public CAS server certificate for TLS (PEM-encoded)")
|
|
38 | 55 |
@click.option('--parent', type=click.STRING, default='main', show_default=True,
|
39 | 56 |
help="Targeted farm resource.")
|
40 |
-@click.option('--port', type=click.INT, default='50051', show_default=True,
|
|
41 |
- help="Remote server's port number.")
|
|
42 |
-@click.option('--host', type=click.STRING, default='localhost', show_default=True,
|
|
43 |
- help="Renote server's hostname.")
|
|
44 | 57 |
@pass_context
|
45 |
-def cli(context, host, port, parent):
|
|
46 |
- channel = grpc.insecure_channel('{}:{}'.format(host, port))
|
|
47 |
- interface = bot_interface.BotInterface(channel)
|
|
58 |
+def cli(context, parent, remote, client_key, client_cert, server_cert,
|
|
59 |
+ remote_cas, cas_client_key, cas_client_cert, cas_server_cert):
|
|
60 |
+ # Setup the remote execution server channel:
|
|
61 |
+ url = urlparse(remote)
|
|
48 | 62 |
|
49 |
- context.logger = logging.getLogger(__name__)
|
|
50 |
- context.logger.info("Starting on port {}".format(port))
|
|
51 |
- context.channel = channel
|
|
63 |
+ context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
|
|
64 |
+ context.remote_url = remote
|
|
52 | 65 |
context.parent = parent
|
53 | 66 |
|
67 |
+ if url.scheme == 'http':
|
|
68 |
+ context.channel = grpc.insecure_channel(context.remote)
|
|
69 |
+ |
|
70 |
+ context.client_key = None
|
|
71 |
+ context.client_cert = None
|
|
72 |
+ context.server_cert = None
|
|
73 |
+ else:
|
|
74 |
+ credentials = context.load_client_credentials(client_key, client_cert, server_cert)
|
|
75 |
+ if not credentials:
|
|
76 |
+ click.echo("ERROR: no TLS keys were specified and no defaults could be found.", err=True)
|
|
77 |
+ sys.exit(-1)
|
|
78 |
+ |
|
79 |
+ context.channel = grpc.secure_channel(context.remote, credentials)
|
|
80 |
+ |
|
81 |
+ context.client_key = credentials.client_key
|
|
82 |
+ context.client_cert = credentials.client_cert
|
|
83 |
+ context.server_cert = credentials.server_cert
|
|
84 |
+ |
|
85 |
+ # Setup the remote CAS server channel, if separated:
|
|
86 |
+ if remote_cas is not None and remote_cas != remote:
|
|
87 |
+ cas_url = urlparse(remote_cas)
|
|
88 |
+ |
|
89 |
+ context.remote_cas = '{}:{}'.format(cas_url.hostname, cas_url.port or 11001)
|
|
90 |
+ context.remote_cas_url = remote_cas
|
|
91 |
+ |
|
92 |
+ if cas_url.scheme == 'http':
|
|
93 |
+ context.cas_channel = grpc.insecure_channel(context.remote_cas)
|
|
94 |
+ |
|
95 |
+ context.cas_client_key = None
|
|
96 |
+ context.cas_client_cert = None
|
|
97 |
+ context.cas_server_cert = None
|
|
98 |
+ else:
|
|
99 |
+ cas_credentials = context.load_client_credentials(cas_client_key, cas_client_cert, cas_server_cert)
|
|
100 |
+ if not cas_credentials:
|
|
101 |
+ click.echo("ERROR: no TLS keys were specified and no defaults could be found.", err=True)
|
|
102 |
+ sys.exit(-1)
|
|
103 |
+ |
|
104 |
+ context.cas_channel = grpc.secure_channel(context.remote_cas, cas_credentials)
|
|
105 |
+ |
|
106 |
+ context.cas_client_key = cas_credentials.client_key
|
|
107 |
+ context.cas_client_cert = cas_credentials.client_cert
|
|
108 |
+ context.cas_server_cert = cas_credentials.server_cert
|
|
109 |
+ |
|
110 |
+ else:
|
|
111 |
+ context.remote_cas = context.remote
|
|
112 |
+ context.remote_cas_url = remote
|
|
113 |
+ |
|
114 |
+ context.cas_channel = context.channel
|
|
115 |
+ |
|
116 |
+ context.cas_client_key = context.client_key
|
|
117 |
+ context.cas_client_cert = context.client_cert
|
|
118 |
+ context.cas_server_cert = context.server_cert
|
|
119 |
+ |
|
120 |
+ context.logger = logging.getLogger(__name__)
|
|
121 |
+ context.logger.debug("Starting for remote {}".format(context.remote))
|
|
122 |
+ |
|
123 |
+ interface = bot_interface.BotInterface(context.channel)
|
|
124 |
+ |
|
54 | 125 |
worker = Worker()
|
55 | 126 |
worker.add_device(Device())
|
56 | 127 |
|
... | ... | @@ -94,35 +165,17 @@ def run_temp_directory(context): |
94 | 165 |
help="Main mount-point location.")
|
95 | 166 |
@click.option('--local-cas', type=click.Path(readable=False), default=str(PurePath(Path.home(), 'cas')),
|
96 | 167 |
help="Local CAS cache directory.")
|
97 |
-@click.option('--client-cert', type=click.Path(readable=False), default=str(PurePath(Path.home(), 'client.crt')),
|
|
98 |
- help="Public client certificate for TLS (PEM-encoded).")
|
|
99 |
-@click.option('--client-key', type=click.Path(readable=False), default=str(PurePath(Path.home(), 'client.key')),
|
|
100 |
- help="Private client key for TLS (PEM-encoded).")
|
|
101 |
-@click.option('--server-cert', type=click.Path(readable=False), default=str(PurePath(Path.home(), 'server.crt')),
|
|
102 |
- help="Public server certificate for TLS (PEM-encoded).")
|
|
103 |
-@click.option('--port', type=click.INT, default=11001, show_default=True,
|
|
104 |
- help="Remote CAS server port.")
|
|
105 |
-@click.option('--remote', type=click.STRING, default='localhost', show_default=True,
|
|
106 |
- help="Remote CAS server hostname.")
|
|
107 | 168 |
@pass_context
|
108 |
-def run_buildbox(context, remote, port, server_cert, client_key, client_cert, local_cas, fuse_dir):
|
|
169 |
+def run_buildbox(context, local_cas, fuse_dir):
|
|
109 | 170 |
"""
|
110 | 171 |
Uses BuildBox to run commands.
|
111 | 172 |
"""
|
112 |
- |
|
113 |
- context.logger.info("Creating a bot session")
|
|
114 |
- |
|
115 |
- context.remote = remote
|
|
116 |
- context.port = port
|
|
117 |
- context.server_cert = server_cert
|
|
118 |
- context.client_key = client_key
|
|
119 |
- context.client_cert = client_cert
|
|
120 | 173 |
context.local_cas = local_cas
|
121 | 174 |
context.fuse_dir = fuse_dir
|
122 | 175 |
|
123 | 176 |
try:
|
124 | 177 |
b = bot.Bot(context.bot_session)
|
125 |
- b.session(work=buildbox.work_buildbox,
|
|
126 |
- context=context)
|
|
178 |
+ b.session(buildbox.work_buildbox,
|
|
179 |
+ context)
|
|
127 | 180 |
except KeyboardInterrupt:
|
128 | 181 |
pass
|
... | ... | @@ -21,6 +21,9 @@ Request work to be executed and monitor status of jobs. |
21 | 21 |
"""
|
22 | 22 |
|
23 | 23 |
import logging
|
24 |
+import sys
|
|
25 |
+from urllib.parse import urlparse
|
|
26 |
+ |
|
24 | 27 |
import click
|
25 | 28 |
import grpc
|
26 | 29 |
|
... | ... | @@ -31,20 +34,36 @@ from ..cli import pass_context |
31 | 34 |
|
32 | 35 |
|
33 | 36 |
@click.group(name='cas', short_help="Interact with the CAS server.")
|
37 |
+@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
|
|
38 |
+ help="Remote execution server's URL (port defaults to 50051 if no specified).")
|
|
39 |
+@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
40 |
+ help="Private client key for TLS (PEM-encoded)")
|
|
41 |
+@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
42 |
+ help="Public client certificate for TLS (PEM-encoded)")
|
|
43 |
+@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
44 |
+ help="Public server certificate for TLS (PEM-encoded)")
|
|
34 | 45 |
@click.option('--instance-name', type=click.STRING, default='main', show_default=True,
|
35 | 46 |
help="Targeted farm instance name.")
|
36 |
-@click.option('--port', type=click.INT, default='50051', show_default=True,
|
|
37 |
- help="Remote server's port number.")
|
|
38 |
-@click.option('--host', type=click.STRING, default='localhost', show_default=True,
|
|
39 |
- help="Remote server's hostname.")
|
|
40 | 47 |
@pass_context
|
41 |
-def cli(context, instance_name, host, port):
|
|
42 |
- context.logger = logging.getLogger(__name__)
|
|
43 |
- context.logger.info("Starting on port {}".format(port))
|
|
48 |
+def cli(context, remote, instance_name, client_key, client_cert, server_cert):
|
|
49 |
+ url = urlparse(remote)
|
|
44 | 50 |
|
51 |
+ context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
|
|
45 | 52 |
context.instance_name = instance_name
|
46 |
- context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
|
|
47 |
- context.port = port
|
|
53 |
+ |
|
54 |
+ if url.scheme == 'http':
|
|
55 |
+ context.channel = grpc.insecure_channel(context.remote)
|
|
56 |
+ else:
|
|
57 |
+ credentials = context.load_client_credentials(client_key, client_cert, server_cert)
|
|
58 |
+ if not credentials:
|
|
59 |
+ click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
|
|
60 |
+ "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
|
|
61 |
+ sys.exit(-1)
|
|
62 |
+ |
|
63 |
+ context.channel = grpc.secure_channel(context.remote, credentials)
|
|
64 |
+ |
|
65 |
+ context.logger = logging.getLogger(__name__)
|
|
66 |
+ context.logger.debug("Starting for remote {}".format(context.remote))
|
|
48 | 67 |
|
49 | 68 |
|
50 | 69 |
@cli.command('upload-files', short_help="Upload files to the CAS server.")
|
... | ... | @@ -22,8 +22,11 @@ Request work to be executed and monitor status of jobs. |
22 | 22 |
|
23 | 23 |
import errno
|
24 | 24 |
import logging
|
25 |
-import stat
|
|
26 | 25 |
import os
|
26 |
+import stat
|
|
27 |
+import sys
|
|
28 |
+from urllib.parse import urlparse
|
|
29 |
+ |
|
27 | 30 |
import click
|
28 | 31 |
import grpc
|
29 | 32 |
|
... | ... | @@ -36,20 +39,36 @@ from ..cli import pass_context |
36 | 39 |
|
37 | 40 |
|
38 | 41 |
@click.group(name='execute', short_help="Execute simple operations.")
|
39 |
-@click.option('--instance-name', type=click.STRING, default='main',
|
|
40 |
- show_default=True, help="Targeted farm instance name.")
|
|
41 |
-@click.option('--port', type=click.INT, default='50051', show_default=True,
|
|
42 |
- help="Remote server's port number.")
|
|
43 |
-@click.option('--host', type=click.STRING, default='localhost', show_default=True,
|
|
44 |
- help="Remote server's hostname.")
|
|
42 |
+@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
|
|
43 |
+ help="Remote execution server's URL (port defaults to 50051 if no specified).")
|
|
44 |
+@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
45 |
+ help="Private client key for TLS (PEM-encoded)")
|
|
46 |
+@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
47 |
+ help="Public client certificate for TLS (PEM-encoded)")
|
|
48 |
+@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
49 |
+ help="Public server certificate for TLS (PEM-encoded)")
|
|
50 |
+@click.option('--instance-name', type=click.STRING, default='main', show_default=True,
|
|
51 |
+ help="Targeted farm instance name.")
|
|
45 | 52 |
@pass_context
|
46 |
-def cli(context, instance_name, host, port):
|
|
47 |
- context.logger = logging.getLogger(__name__)
|
|
48 |
- context.logger.info("Starting on port {}".format(port))
|
|
53 |
+def cli(context, remote, instance_name, client_key, client_cert, server_cert):
|
|
54 |
+ url = urlparse(remote)
|
|
49 | 55 |
|
56 |
+ context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
|
|
50 | 57 |
context.instance_name = instance_name
|
51 |
- context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
|
|
52 |
- context.port = port
|
|
58 |
+ |
|
59 |
+ if url.scheme == 'http':
|
|
60 |
+ context.channel = grpc.insecure_channel(context.remote)
|
|
61 |
+ else:
|
|
62 |
+ credentials = context.load_client_credentials(client_key, client_cert, server_cert)
|
|
63 |
+ if not credentials:
|
|
64 |
+ click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
|
|
65 |
+ "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
|
|
66 |
+ sys.exit(-1)
|
|
67 |
+ |
|
68 |
+ context.channel = grpc.secure_channel(context.remote, credentials)
|
|
69 |
+ |
|
70 |
+ context.logger = logging.getLogger(__name__)
|
|
71 |
+ context.logger.debug("Starting for remote {}".format(context.remote))
|
|
53 | 72 |
|
54 | 73 |
|
55 | 74 |
@cli.command('request-dummy', short_help="Send a dummy action.")
|
... | ... | @@ -22,6 +22,7 @@ Create a BuildGrid server. |
22 | 22 |
|
23 | 23 |
import asyncio
|
24 | 24 |
import logging
|
25 |
+import sys
|
|
25 | 26 |
|
26 | 27 |
import click
|
27 | 28 |
|
... | ... | @@ -41,18 +42,25 @@ _SIZE_PREFIXES = {'k': 2 ** 10, 'm': 2 ** 20, 'g': 2 ** 30, 't': 2 ** 40} |
41 | 42 |
@pass_context
|
42 | 43 |
def cli(context):
|
43 | 44 |
context.logger = logging.getLogger(__name__)
|
44 |
- context.logger.info("BuildGrid server booting up")
|
|
45 | 45 |
|
46 | 46 |
|
47 | 47 |
@cli.command('start', short_help="Setup a new server instance.")
|
48 | 48 |
@click.argument('instances', nargs=-1, type=click.STRING)
|
49 | 49 |
@click.option('--port', type=click.INT, default='50051', show_default=True,
|
50 | 50 |
help="The port number to be listened.")
|
51 |
-@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
|
|
52 |
- help="Maximum number of actions to keep in the ActionCache.")
|
|
51 |
+@click.option('--server-key', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
52 |
+ help="Private server key for TLS (PEM-encoded)")
|
|
53 |
+@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
54 |
+ help="Public server certificate for TLS (PEM-encoded)")
|
|
55 |
+@click.option('--client-certs', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
56 |
+ help="Public client certificates for TLS (PEM-encoded, one single file)")
|
|
57 |
+@click.option('--allow-insecure', type=click.BOOL, is_flag=True,
|
|
58 |
+ help="Whether or not to allow unencrypted connections.")
|
|
53 | 59 |
@click.option('--allow-update-action-result/--forbid-update-action-result',
|
54 | 60 |
'allow_uar', default=True, show_default=True,
|
55 | 61 |
help="Whether or not to allow clients to manually edit the action cache.")
|
62 |
+@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
|
|
63 |
+ help="Maximum number of actions to keep in the ActionCache.")
|
|
56 | 64 |
@click.option('--cas', type=click.Choice(('lru', 's3', 'disk', 'with-cache')),
|
57 | 65 |
help="The CAS storage type to use.")
|
58 | 66 |
@click.option('--cas-cache', type=click.Choice(('lru', 's3', 'disk')),
|
... | ... | @@ -68,9 +76,21 @@ def cli(context): |
68 | 76 |
@click.option('--cas-disk-directory', type=click.Path(file_okay=False, dir_okay=True, writable=True),
|
69 | 77 |
help="For --cas=disk, the folder to store CAS blobs in.")
|
70 | 78 |
@pass_context
|
71 |
-def start(context, instances, port, max_cached_actions, allow_uar, cas, **cas_args):
|
|
72 |
- """ Starts a BuildGrid server.
|
|
73 |
- """
|
|
79 |
+def start(context, port, allow_insecure, server_key, server_cert, client_certs,
|
|
80 |
+ instances, max_cached_actions, allow_uar, cas, **cas_args):
|
|
81 |
+ """Setups a new server instance."""
|
|
82 |
+ credentials = None
|
|
83 |
+ if not allow_insecure:
|
|
84 |
+ credentials = context.load_server_credentials(server_key, server_cert, client_certs)
|
|
85 |
+ if not credentials and not allow_insecure:
|
|
86 |
+ click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
|
|
87 |
+ "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
|
|
88 |
+ sys.exit(-1)
|
|
89 |
+ |
|
90 |
+ context.credentials = credentials
|
|
91 |
+ context.port = port
|
|
92 |
+ |
|
93 |
+ context.logger.info("BuildGrid server booting up")
|
|
74 | 94 |
context.logger.info("Starting on port {}".format(port))
|
75 | 95 |
|
76 | 96 |
cas_storage = _make_cas_storage(context, cas, cas_args)
|
... | ... | @@ -85,8 +105,9 @@ def start(context, instances, port, max_cached_actions, allow_uar, cas, **cas_ar |
85 | 105 |
if instances is None:
|
86 | 106 |
instances = ['main']
|
87 | 107 |
|
88 |
- server = buildgrid_server.BuildGridServer(port,
|
|
89 |
- instances,
|
|
108 |
+ server = buildgrid_server.BuildGridServer(port=context.port,
|
|
109 |
+ credentials=context.credentials,
|
|
110 |
+ instances=instances,
|
|
90 | 111 |
cas_storage=cas_storage,
|
91 | 112 |
action_cache=action_cache)
|
92 | 113 |
loop = asyncio.get_event_loop()
|
... | ... | @@ -40,11 +40,16 @@ from .worker.bots_service import BotsService |
40 | 40 |
|
41 | 41 |
class BuildGridServer:
|
42 | 42 |
|
43 |
- def __init__(self, port='50051', instances=None, max_workers=10, action_cache=None, cas_storage=None):
|
|
44 |
- port = '[::]:{0}'.format(port)
|
|
43 |
+ def __init__(self, port=50051, credentials=None, instances=None,
|
|
44 |
+ max_workers=10, action_cache=None, cas_storage=None):
|
|
45 |
+ address = '[::]:{0}'.format(port)
|
|
45 | 46 |
|
46 | 47 |
self._server = grpc.server(futures.ThreadPoolExecutor(max_workers))
|
47 |
- self._server.add_insecure_port(port)
|
|
48 |
+ |
|
49 |
+ if credentials is not None:
|
|
50 |
+ self._server.add_secure_port(address, credentials)
|
|
51 |
+ else:
|
|
52 |
+ self._server.add_insecure_port(address)
|
|
48 | 53 |
|
49 | 54 |
if cas_storage is not None:
|
50 | 55 |
cas_service = ContentAddressableStorageService(cas_storage)
|
... | ... | @@ -86,6 +86,11 @@ class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer): |
86 | 86 |
yield operations_pb2.Operation()
|
87 | 87 |
|
88 | 88 |
def _get_instance(self, name):
|
89 |
+ # If client does not support multiple instances, it may omit the
|
|
90 |
+ # instance name request parameter, so better map our default:
|
|
91 |
+ if not name and len(self._instances) == 1:
|
|
92 |
+ name = 'main'
|
|
93 |
+ |
|
89 | 94 |
try:
|
90 | 95 |
return self._instances[name]
|
91 | 96 |
|
... | ... | @@ -13,6 +13,7 @@ |
13 | 13 |
# limitations under the License.
|
14 | 14 |
|
15 | 15 |
|
16 |
+from operator import attrgetter
|
|
16 | 17 |
import os
|
17 | 18 |
|
18 | 19 |
from buildgrid.settings import HASH
|
... | ... | @@ -31,30 +32,59 @@ def gen_fetch_blob(stub, digest, instance_name=""): |
31 | 32 |
yield response.data
|
32 | 33 |
|
33 | 34 |
|
34 |
-def write_fetch_directory(directory, stub, digest, instance_name=""):
|
|
35 |
- """ Given a directory digest, fetches files and writes them to a directory
|
|
35 |
+def write_fetch_directory(root_directory, stub, digest, instance_name=None):
|
|
36 |
+ """Locally replicates a directory from CAS.
|
|
37 |
+ |
|
38 |
+ Args:
|
|
39 |
+ root_directory (str): local directory to populate.
|
|
40 |
+ stub (): gRPC stub for CAS communication.
|
|
41 |
+ digest (Digest): digest for the directory to fetch from CAS.
|
|
42 |
+ instance_name (str, optional): farm instance name to query data from.
|
|
36 | 43 |
"""
|
37 |
- # TODO: Extend to symlinks and inner directories
|
|
38 |
- # pathlib.Path('/my/directory').mkdir(parents=True, exist_ok=True)
|
|
44 |
+ if not os.path.isabs(root_directory):
|
|
45 |
+ root_directory = os.path.abspath(root_directory)
|
|
46 |
+ if not os.path.exists(root_directory):
|
|
47 |
+ os.makedirs(root_directory, exist_ok=True)
|
|
39 | 48 |
|
40 |
- directory_pb2 = remote_execution_pb2.Directory()
|
|
41 |
- directory_pb2 = parse_to_pb2_from_fetch(directory_pb2, stub, digest, instance_name)
|
|
49 |
+ directory = parse_to_pb2_from_fetch(remote_execution_pb2.Directory(),
|
|
50 |
+ stub, digest, instance_name)
|
|
51 |
+ |
|
52 |
+ for directory_node in directory.directories:
|
|
53 |
+ child_path = os.path.join(root_directory, directory_node.name)
|
|
54 |
+ |
|
55 |
+ write_fetch_directory(child_path, stub, directory_node.digest, instance_name)
|
|
56 |
+ |
|
57 |
+ for file_node in directory.files:
|
|
58 |
+ child_path = os.path.join(root_directory, file_node.name)
|
|
59 |
+ |
|
60 |
+ with open(child_path, 'wb') as child_file:
|
|
61 |
+ write_fetch_blob(child_file, stub, file_node.digest, instance_name)
|
|
62 |
+ |
|
63 |
+ for symlink_node in directory.symlinks:
|
|
64 |
+ child_path = os.path.join(root_directory, symlink_node.name)
|
|
65 |
+ |
|
66 |
+ if os.path.isabs(symlink_node.target):
|
|
67 |
+ continue # No out of temp-directory links for now.
|
|
68 |
+ target_path = os.path.join(root_directory, symlink_node.target)
|
|
69 |
+ |
|
70 |
+ os.symlink(child_path, target_path)
|
|
42 | 71 |
|
43 |
- for file_node in directory_pb2.files:
|
|
44 |
- path = os.path.join(directory, file_node.name)
|
|
45 |
- with open(path, 'wb') as f:
|
|
46 |
- write_fetch_blob(f, stub, file_node.digest, instance_name)
|
|
47 | 72 |
|
73 |
+def write_fetch_blob(target_file, stub, digest, instance_name=None):
|
|
74 |
+ """Extracts a blob from CAS into a local file.
|
|
48 | 75 |
|
49 |
-def write_fetch_blob(out, stub, digest, instance_name=""):
|
|
50 |
- """ Given an output buffer, fetches blob and writes to buffer
|
|
76 |
+ Args:
|
|
77 |
+ target_file (str): local file to write.
|
|
78 |
+ stub (): gRPC stub for CAS communication.
|
|
79 |
+ digest (Digest): digest for the blob to fetch from CAS.
|
|
80 |
+ instance_name (str, optional): farm instance name to query data from.
|
|
51 | 81 |
"""
|
52 | 82 |
|
53 | 83 |
for stream in gen_fetch_blob(stub, digest, instance_name):
|
54 |
- out.write(stream)
|
|
84 |
+ target_file.write(stream)
|
|
85 |
+ target_file.flush()
|
|
55 | 86 |
|
56 |
- out.flush()
|
|
57 |
- assert digest.size_bytes == os.fstat(out.fileno()).st_size
|
|
87 |
+ assert digest.size_bytes == os.fstat(target_file.fileno()).st_size
|
|
58 | 88 |
|
59 | 89 |
|
60 | 90 |
def parse_to_pb2_from_fetch(pb2, stub, digest, instance_name=""):
|
... | ... | @@ -70,7 +100,15 @@ def parse_to_pb2_from_fetch(pb2, stub, digest, instance_name=""): |
70 | 100 |
|
71 | 101 |
|
72 | 102 |
def create_digest(bytes_to_digest):
|
73 |
- """ Creates a hash based on the hex digest and returns the digest
|
|
103 |
+ """Computes the :obj:`Digest` of a piece of data.
|
|
104 |
+ |
|
105 |
+ The :obj:`Digest` of a data is a function of its hash **and** size.
|
|
106 |
+ |
|
107 |
+ Args:
|
|
108 |
+ bytes_to_digest (bytes): byte data to digest.
|
|
109 |
+ |
|
110 |
+ Returns:
|
|
111 |
+ :obj:`Digest`: The gRPC :obj:`Digest` for the given byte data.
|
|
74 | 112 |
"""
|
75 | 113 |
return remote_execution_pb2.Digest(hash=HASH(bytes_to_digest).hexdigest(),
|
76 | 114 |
size_bytes=len(bytes_to_digest))
|
... | ... | @@ -107,6 +145,199 @@ def file_maker(file_path, file_digest): |
107 | 145 |
is_executable=os.access(file_path, os.X_OK))
|
108 | 146 |
|
109 | 147 |
|
110 |
-def read_file(read):
|
|
111 |
- with open(read, 'rb') as f:
|
|
112 |
- return f.read()
|
|
148 |
+def directory_maker(directory_path):
|
|
149 |
+ """Creates a :obj:`Directory` from a local directory.
|
|
150 |
+ |
|
151 |
+ Args:
|
|
152 |
+ directory_path (str): absolute or relative path to a local directory.
|
|
153 |
+ |
|
154 |
+ Returns:
|
|
155 |
+ :obj:`Directory`, list of :obj:`Directory`, list of
|
|
156 |
+ :obj:`BatchUpdateBlobsRequest`: Tuple of a new gRPC :obj:`Directory` for
|
|
157 |
+ the directory pointed by `directory_path`, a list of new gRPC
|
|
158 |
+ :obj:`Directory` for every children of that directory and the
|
|
159 |
+ corresponding list of :obj:`BatchUpdateBlobsRequest` for CAS upload.
|
|
160 |
+ |
|
161 |
+ The :obj:`Directory` children list may come in any order.
|
|
162 |
+ |
|
163 |
+ The :obj:`BatchUpdateBlobsRequest` list may come in any order. However,
|
|
164 |
+ its last element is guaranteed to be the root :obj:`Direcotry`'s
|
|
165 |
+ request.
|
|
166 |
+ """
|
|
167 |
+ if not os.path.isabs(directory_path):
|
|
168 |
+ directory_path = os.path.abspath(directory_path)
|
|
169 |
+ |
|
170 |
+ child_directories = list()
|
|
171 |
+ update_requests = list()
|
|
172 |
+ |
|
173 |
+ files, directories, symlinks = list(), list(), list()
|
|
174 |
+ for directory_entry in os.scandir(directory_path):
|
|
175 |
+ # Create a FileNode and corresponding BatchUpdateBlobsRequest:
|
|
176 |
+ if directory_entry.is_file(follow_symlinks=False):
|
|
177 |
+ node_blob = read_file(directory_entry.path)
|
|
178 |
+ node_digest = create_digest(node_blob)
|
|
179 |
+ |
|
180 |
+ node = remote_execution_pb2.FileNode()
|
|
181 |
+ node.name = directory_entry.name
|
|
182 |
+ node.digest = node_digest
|
|
183 |
+ node.is_executable = os.access(directory_entry.path, os.X_OK)
|
|
184 |
+ |
|
185 |
+ node_request = remote_execution_pb2.BatchUpdateBlobsRequest.Request(digest=node_digest)
|
|
186 |
+ node_request.data = node_blob
|
|
187 |
+ |
|
188 |
+ update_requests.append(node_request)
|
|
189 |
+ files.append(node)
|
|
190 |
+ |
|
191 |
+ # Create a DirectoryNode and corresponding BatchUpdateBlobsRequest:
|
|
192 |
+ elif directory_entry.is_dir(follow_symlinks=False):
|
|
193 |
+ node_directory, node_children, node_requests = directory_maker(directory_entry.path)
|
|
194 |
+ |
|
195 |
+ node = remote_execution_pb2.DirectoryNode()
|
|
196 |
+ node.name = directory_entry.name
|
|
197 |
+ node.digest = node_requests[-1].digest
|
|
198 |
+ |
|
199 |
+ child_directories.extend(node_children)
|
|
200 |
+ child_directories.append(node_directory)
|
|
201 |
+ update_requests.extend(node_requests)
|
|
202 |
+ directories.append(node)
|
|
203 |
+ |
|
204 |
+ # Create a SymlinkNode if necessary;
|
|
205 |
+ elif os.path.islink(directory_entry.path):
|
|
206 |
+ node_target = os.readlink(directory_entry.path)
|
|
207 |
+ |
|
208 |
+ node = remote_execution_pb2.SymlinkNode()
|
|
209 |
+ node.name = directory_entry.name
|
|
210 |
+ node.target = node_target
|
|
211 |
+ |
|
212 |
+ symlinks.append(node)
|
|
213 |
+ |
|
214 |
+ directory = remote_execution_pb2.Directory()
|
|
215 |
+ directory.files.extend(files.sort(key=attrgetter('name')))
|
|
216 |
+ directory.directories.extend(directories.sort(key=attrgetter('name')))
|
|
217 |
+ directory.symlinks.extend(symlinks.sort(key=attrgetter('name')))
|
|
218 |
+ |
|
219 |
+ directory_blob = directory.SerializeToString()
|
|
220 |
+ directory_digest = create_digest(directory_blob)
|
|
221 |
+ |
|
222 |
+ update_request = remote_execution_pb2.BatchUpdateBlobsRequest.Request(digest=directory_digest)
|
|
223 |
+ update_request.data = directory_blob
|
|
224 |
+ |
|
225 |
+ update_requests.append(update_request)
|
|
226 |
+ |
|
227 |
+ return directory, child_directories, update_requests
|
|
228 |
+ |
|
229 |
+ |
|
230 |
+def read_file(file_path):
|
|
231 |
+ """Loads raw file content in memory.
|
|
232 |
+ |
|
233 |
+ Returns:
|
|
234 |
+ bytes: Raw file's content until EOF.
|
|
235 |
+ |
|
236 |
+ Raises:
|
|
237 |
+ OSError: If `file_path` does not exist or is not readable.
|
|
238 |
+ """
|
|
239 |
+ with open(file_path, 'rb') as byte_file:
|
|
240 |
+ return byte_file.read()
|
|
241 |
+ |
|
242 |
+ |
|
243 |
+def output_file_maker(file_path, input_path):
|
|
244 |
+ """Creates an :obj:`OutputFile` from a local file.
|
|
245 |
+ |
|
246 |
+ `file_path` **must** point inside or be relative to `input_path`.
|
|
247 |
+ |
|
248 |
+ Args:
|
|
249 |
+ file_path (str): absolute or relative path to a local file.
|
|
250 |
+ input_path (str): absolute or relative path to the input root directory.
|
|
251 |
+ |
|
252 |
+ Returns:
|
|
253 |
+ :obj:`OutputFile`, :obj:`BatchUpdateBlobsRequest`: Tuple of a new gRPC
|
|
254 |
+ :obj:`OutputFile` object for the file pointed by `file_path` and the
|
|
255 |
+ corresponding :obj:`BatchUpdateBlobsRequest` for CAS upload.
|
|
256 |
+ """
|
|
257 |
+ if not os.path.isabs(file_path):
|
|
258 |
+ file_path = os.path.abspath(file_path)
|
|
259 |
+ if not os.path.isabs(input_path):
|
|
260 |
+ input_path = os.path.abspath(input_path)
|
|
261 |
+ |
|
262 |
+ file_blob = read_file(file_path)
|
|
263 |
+ file_digest = create_digest(file_blob)
|
|
264 |
+ |
|
265 |
+ output_file = remote_execution_pb2.OutputFile(digest=file_digest)
|
|
266 |
+ output_file.path = os.path.relpath(file_path, start=input_path)
|
|
267 |
+ output_file.is_executable = os.access(file_path, os.X_OK)
|
|
268 |
+ |
|
269 |
+ update_request = remote_execution_pb2.BatchUpdateBlobsRequest.Request(digest=file_digest)
|
|
270 |
+ update_request.data = file_blob
|
|
271 |
+ |
|
272 |
+ return output_file, update_request
|
|
273 |
+ |
|
274 |
+ |
|
275 |
+def output_directory_maker(directory_path, working_path):
|
|
276 |
+ """Creates an :obj:`OutputDirectory` from a local directory.
|
|
277 |
+ |
|
278 |
+ `directory_path` **must** point inside or be relative to `input_path`.
|
|
279 |
+ |
|
280 |
+ Args:
|
|
281 |
+ directory_path (str): absolute or relative path to a local directory.
|
|
282 |
+ working_path (str): absolute or relative path to the working directory.
|
|
283 |
+ |
|
284 |
+ Returns:
|
|
285 |
+ :obj:`OutputDirectory`, :obj:`BatchUpdateBlobsRequest`: Tuple of a new
|
|
286 |
+ gRPC :obj:`OutputDirectory` for the directory pointed by
|
|
287 |
+ `directory_path` and the corresponding list of
|
|
288 |
+ :obj:`BatchUpdateBlobsRequest` for CAS upload.
|
|
289 |
+ """
|
|
290 |
+ if not os.path.isabs(directory_path):
|
|
291 |
+ directory_path = os.path.abspath(directory_path)
|
|
292 |
+ if not os.path.isabs(working_path):
|
|
293 |
+ working_path = os.path.abspath(working_path)
|
|
294 |
+ |
|
295 |
+ _, update_requests = tree_maker(directory_path)
|
|
296 |
+ |
|
297 |
+ output_directory = remote_execution_pb2.OutputDirectory()
|
|
298 |
+ output_directory.tree_digest = update_requests[-1].digest
|
|
299 |
+ output_directory.path = os.path.relpath(directory_path, start=working_path)
|
|
300 |
+ |
|
301 |
+ output_directory_blob = output_directory.SerializeToString()
|
|
302 |
+ output_directory_digest = create_digest(output_directory_blob)
|
|
303 |
+ |
|
304 |
+ update_request = remote_execution_pb2.BatchUpdateBlobsRequest.Request(digest=output_directory_digest)
|
|
305 |
+ update_request.data = output_directory_blob
|
|
306 |
+ |
|
307 |
+ update_requests.append(update_request)
|
|
308 |
+ |
|
309 |
+ return output_directory, update_requests
|
|
310 |
+ |
|
311 |
+ |
|
312 |
+def tree_maker(directory_path):
|
|
313 |
+ """Creates a :obj:`Tree` from a local directory.
|
|
314 |
+ |
|
315 |
+ Args:
|
|
316 |
+ directory_path (str): absolute or relative path to a local directory.
|
|
317 |
+ |
|
318 |
+ Returns:
|
|
319 |
+ :obj:`Tree`, :obj:`BatchUpdateBlobsRequest`: Tuple of a new
|
|
320 |
+ gRPC :obj:`Tree` for the directory pointed by `directory_path` and the
|
|
321 |
+ corresponding list of :obj:`BatchUpdateBlobsRequest` for CAS upload.
|
|
322 |
+ |
|
323 |
+ The :obj:`BatchUpdateBlobsRequest` list may come in any order. However,
|
|
324 |
+ its last element is guaranteed to be the :obj:`Tree`'s request.
|
|
325 |
+ """
|
|
326 |
+ if not os.path.isabs(directory_path):
|
|
327 |
+ directory_path = os.path.abspath(directory_path)
|
|
328 |
+ |
|
329 |
+ directory, child_directories, update_requests = directory_maker(directory_path)
|
|
330 |
+ |
|
331 |
+ tree = remote_execution_pb2.Tree()
|
|
332 |
+ tree.children.expend([child_directories])
|
|
333 |
+ tree.root = directory
|
|
334 |
+ |
|
335 |
+ tree_blob = tree.SerializeToString()
|
|
336 |
+ tree_digest = create_digest(tree_blob)
|
|
337 |
+ |
|
338 |
+ update_request = remote_execution_pb2.BatchUpdateBlobsRequest.Request(digest=tree_digest)
|
|
339 |
+ update_request.data = tree_blob
|
|
340 |
+ |
|
341 |
+ update_requests.append(update_request)
|
|
342 |
+ |
|
343 |
+ return tree, update_requests
|
1 |
+ |
|
2 |
+.. _configuration:
|
|
3 |
+ |
|
4 |
+Configuration
|
|
5 |
+=============
|
|
6 |
+ |
|
7 |
+The details of how to tune BuildGrid's configuration.
|
|
8 |
+ |
|
9 |
+ |
|
10 |
+.. _configuration-location:
|
|
11 |
+ |
|
12 |
+Configuration location
|
|
13 |
+----------------------
|
|
14 |
+ |
|
15 |
+Unless a configuration file is explicitly specified on the command line when
|
|
16 |
+invoking `bgd`, BuildGrid will always attempt to load configuration resources
|
|
17 |
+from ``$XDG_CONFIG_HOME/buildgrid``. On most Linux based systems, the location
|
|
18 |
+will be ``~/.config/buildgrid``.
|
|
19 |
+ |
|
20 |
+This location is refered as ``$CONFIG_HOME`` is the rest of the document.
|
|
21 |
+ |
|
22 |
+ |
|
23 |
+.. _tls-encryption:
|
|
24 |
+ |
|
25 |
+TLS encryption
|
|
26 |
+--------------
|
|
27 |
+ |
|
28 |
+Every BuildGrid gRPC communication channel can be encrypted using SSL/TLS. By
|
|
29 |
+default, the BuildGrid server will try to setup secure gRPC endpoints and return
|
|
30 |
+in error if that fails. You must specify ``--allow-insecure`` explicitly if you
|
|
31 |
+want it to use non-encrypted connections.
|
|
32 |
+ |
|
33 |
+The TLS protocol handshake relies on an asymmetric cryptography system that
|
|
34 |
+requires the server and the client to own a public/private key pair. BuildGrid
|
|
35 |
+will try to load keys from these locations by default:
|
|
36 |
+ |
|
37 |
+- Server private key: ``$CONFIG_HOME/server.key``
|
|
38 |
+- Server public key/certificate: ``$CONFIG_HOME/server.crt``
|
|
39 |
+- Client private key: ``$CONFIG_HOME/client.key``
|
|
40 |
+- Client public key/certificate: ``$CONFIG_HOME/client.crt``
|
|
41 |
+ |
|
42 |
+ |
|
43 |
+Server key pair
|
|
44 |
+~~~~~~~~~~~~~~~
|
|
45 |
+ |
|
46 |
+The TLS protocol requires a key pair to be used by the server. The following
|
|
47 |
+example generates a self-signed key ``server.key``, which requires clients to
|
|
48 |
+have a copy of the server certificate ``server.crt``. You can of course use a
|
|
49 |
+key pair obtained from a trusted certificate authority instead.
|
|
50 |
+ |
|
51 |
+.. code-block:: sh
|
|
52 |
+ |
|
53 |
+ openssl req -new -newkey rsa:4096 -x509 -sha256 -days 3650 -nodes -batch -subj "/CN=localhost" -out server.crt -keyout server.key
|
|
54 |
+ |
|
55 |
+ |
|
56 |
+Client key pair
|
|
57 |
+~~~~~~~~~~~~~~~
|
|
58 |
+ |
|
59 |
+If the server requires authentication in order to be granted special permissions
|
|
60 |
+like uploading to CAS, a client side key pair is required. The following example
|
|
61 |
+generates a self-signed key ``client.key``, which requires the server to have a
|
|
62 |
+copy of the client certificate ``client.crt``.
|
|
63 |
+ |
|
64 |
+.. code-block:: sh
|
|
65 |
+ |
|
66 |
+ openssl req -new -newkey rsa:4096 -x509 -sha256 -days 3650 -nodes -batch -subj "/CN=client" -out client.crt -keyout client.key
|
... | ... | @@ -15,6 +15,7 @@ Remote execution service implementing Google's REAPI and RWAPI. |
15 | 15 |
|
16 | 16 |
about.rst
|
17 | 17 |
installation.rst
|
18 |
+ configuration.rst
|
|
18 | 19 |
using.rst
|
19 | 20 |
reference.rst
|
20 | 21 |
contributing.rst
|
... | ... | @@ -8,7 +8,7 @@ In one terminal, start a server: |
8 | 8 |
|
9 | 9 |
.. code-block:: sh
|
10 | 10 |
|
11 |
- bgd server start
|
|
11 |
+ bgd server start --allow-insecure
|
|
12 | 12 |
|
13 | 13 |
In another terminal, send a request for work:
|
14 | 14 |
|
... | ... | @@ -27,7 +27,7 @@ Now start a BuildGrid server, passing it a directory it can write a CAS to: |
27 | 27 |
|
28 | 28 |
.. code-block:: sh
|
29 | 29 |
|
30 |
- bgd server start --cas disk --cas-cache disk --cas-disk-directory /path/to/empty/directory
|
|
30 |
+ bgd server start --allow-insecure --cas disk --cas-cache disk --cas-disk-directory /path/to/empty/directory
|
|
31 | 31 |
|
32 | 32 |
Start the following bot session:
|
33 | 33 |
|
... | ... | @@ -116,6 +116,7 @@ setup( |
116 | 116 |
'Click',
|
117 | 117 |
'boto3 < 1.8.0',
|
118 | 118 |
'botocore < 1.11.0',
|
119 |
+ 'xdg',
|
|
119 | 120 |
],
|
120 | 121 |
entry_points={
|
121 | 122 |
'console_scripts': [
|