[Notes] [Git][BuildGrid/buildgrid][finn/80-remote-parser] 6 commits: utils.py: Add a simple file writer helper

[finn <gitlab mg gitlab com>]

Title: GitLab

finn pushed to branch finn/80-remote-parser at BuildGrid / buildgrid

Commits:

• d7ef832c
  by Martin Blanchard at 2018-09-11T16:51:58Z

  utils.py: Add a simple file writer helper
  
  https://gitlab.com/BuildGrid/buildgrid/issues/71
  

• 2d44c601
  by Martin Blanchard at 2018-09-11T16:51:58Z

  buildbox.py: Better action execution support
  
  https://gitlab.com/BuildGrid/buildgrid/issues/71
  

• a1dd6783
  by Martin Blanchard at 2018-09-11T16:51:58Z

  buildbox.py: Manually forge a Tree message
  
  https://gitlab.com/BuildGrid/buildgrid/issues/71
  

• aa688609
  by finnball at 2018-09-12T09:13:18Z

  Added remote parser.
  
  Constructs a channel using the `Context` class and passes it into
  the `RemoteStorage` class
  

• 8d947213
  by finnball at 2018-09-12T09:13:18Z

  Added new config files and a missing `__init__.py`
  
  Also moved location of a variable in `default.yml`
  

• 1dc1bb60
  by finnball at 2018-09-12T09:13:18Z

  Stop command from crashing if credentials aren't specified.
  

8 changed files:

• buildgrid/_app/bots/buildbox.py
• buildgrid/_app/commands/cmd_server.py
• + buildgrid/_app/settings/__init__.py
• + buildgrid/_app/settings/cas.yml
• buildgrid/_app/settings/default.yml
• buildgrid/_app/settings/parser.py
• + buildgrid/_app/settings/remote-storage.yml
• buildgrid/utils.py

Changes:

buildgrid/_app/bots/buildbox.py

---

...	...	@@ -19,83 +19,126 @@ import tempfile
19	19
20	20	from google.protobuf import any_pb2
21	21
	22	+from buildgrid.client.cas import upload
22	23	from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
23	24	from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
24		-from buildgrid.utils import parse_to_pb2_from_fetch
	25	+from buildgrid.utils import read_file, write_file, parse_to_pb2_from_fetch
25	26
26	27
27	28	def work_buildbox(context, lease):
	29	+ """Executes a lease for a build action, using buildbox.
	30	+ """
	31	+
	32	+ stub_bytestream = bytestream_pb2_grpc.ByteStreamStub(context.cas_channel)
	33	+ local_cas_directory = context.local_cas
28	34	logger = context.logger
29	35
30		- action_digest_any = lease.payload
31	36	action_digest = remote_execution_pb2.Digest()
32		- action_digest_any.Unpack(action_digest)
	37	+ lease.payload.Unpack(action_digest)
	38	+
	39	+ action = parse_to_pb2_from_fetch(remote_execution_pb2.Action(),
	40	+ stub_bytestream, action_digest)
33	41
34		- stub = bytestream_pb2_grpc.ByteStreamStub(context.cas_channel)
	42	+ command = parse_to_pb2_from_fetch(remote_execution_pb2.Command(),
	43	+ stub_bytestream, action.command_digest)
35	44
36		- action = remote_execution_pb2.Action()
37		- parse_to_pb2_from_fetch(action, stub, action_digest)
	45	+ environment = dict()
	46	+ for variable in command.environment_variables:
	47	+ if variable.name not in ['PWD']:
	48	+ environment[variable.name] = variable.value
38	49
39		- casdir = context.local_cas
40		- remote_command = remote_execution_pb2.Command()
41		- parse_to_pb2_from_fetch(remote_command, stub, action.command_digest)
	50	+ if command.working_directory:
	51	+ working_directory = command.working_directory
	52	+ else:
	53	+ working_directory = '/'
42	54
43		- environment = dict((x.name, x.value) for x in remote_command.environment_variables)
44	55	logger.debug("command hash: {}".format(action.command_digest.hash))
45	56	logger.debug("vdir hash: {}".format(action.input_root_digest.hash))
46		- logger.debug("\n{}".format(' '.join(remote_command.arguments)))
47		-
48		- # Input hash must be written to disk for buildbox.
49		- os.makedirs(os.path.join(casdir, 'tmp'), exist_ok=True)
50		- with tempfile.NamedTemporaryFile(dir=os.path.join(casdir, 'tmp')) as input_digest_file:
51		- with open(input_digest_file.name, 'wb') as f:
52		- f.write(action.input_root_digest.SerializeToString())
53		- f.flush()
54		-
55		- with tempfile.NamedTemporaryFile(dir=os.path.join(casdir, 'tmp')) as output_digest_file:
56		- command = ['buildbox',
57		- '--remote={}'.format(context.remote_cas_url),
58		- '--input-digest={}'.format(input_digest_file.name),
59		- '--output-digest={}'.format(output_digest_file.name),
60		- '--local={}'.format(casdir)]
	57	+ logger.debug("\n{}".format(' '.join(command.arguments)))
	58	+
	59	+ os.makedirs(os.path.join(local_cas_directory, 'tmp'), exist_ok=True)
	60	+ os.makedirs(context.fuse_dir, exist_ok=True)
	61	+
	62	+ with tempfile.NamedTemporaryFile(dir=os.path.join(local_cas_directory, 'tmp')) as input_digest_file:
	63	+ # Input hash must be written to disk for BuildBox
	64	+ write_file(input_digest_file.name, action.input_root_digest.SerializeToString())
	65	+
	66	+ with tempfile.NamedTemporaryFile(dir=os.path.join(local_cas_directory, 'tmp')) as output_digest_file:
	67	+ command_line = ['buildbox',
	68	+ '--remote={}'.format(context.remote_cas_url),
	69	+ '--input-digest={}'.format(input_digest_file.name),
	70	+ '--output-digest={}'.format(output_digest_file.name),
	71	+ '--chdir={}'.format(working_directory),
	72	+ '--local={}'.format(local_cas_directory)]
61	73
62	74	if context.cas_client_key:
63		- command.append('--client-key={}'.format(context.cas_client_key))
	75	+ command_line.append('--client-key={}'.format(context.cas_client_key))
64	76	if context.cas_client_cert:
65		- command.append('--client-cert={}'.format(context.cas_client_cert))
	77	+ command_line.append('--client-cert={}'.format(context.cas_client_cert))
66	78	if context.cas_server_cert:
67		- command.append('--server-cert={}'.format(context.cas_server_cert))
68		-
69		- if 'PWD' in environment and environment['PWD']:
70		- command.append('--chdir={}'.format(environment['PWD']))
	79	+ command_line.append('--server-cert={}'.format(context.cas_server_cert))
71	80
72		- command.append(context.fuse_dir)
73		- command.extend(remote_command.arguments)
	81	+ command_line.append(context.fuse_dir)
	82	+ command_line.extend(command.arguments)
74	83
75		- logger.debug(' '.join(command))
	84	+ logger.debug(' '.join(command_line))
76	85	logger.debug("Input root digest:\n{}".format(action.input_root_digest))
77	86	logger.info("Launching process")
78	87
79		- proc = subprocess.Popen(command,
80		- stdin=subprocess.PIPE,
81		- stdout=subprocess.PIPE)
82		- proc.communicate()
	88	+ command_line = subprocess.Popen(command_line,
	89	+ stdin=subprocess.PIPE,
	90	+ stdout=subprocess.PIPE)
	91	+ # TODO: Should return the stdout and stderr to the user.
	92	+ command_line.communicate()
83	93
84		- output_root_digest = remote_execution_pb2.Digest()
85		- with open(output_digest_file.name, 'rb') as f:
86		- output_root_digest.ParseFromString(f.read())
87		- logger.debug("Output root digest: {}".format(output_root_digest))
	94	+ output_digest = remote_execution_pb2.Digest()
	95	+ output_digest.ParseFromString(read_file(output_digest_file.name))
88	96
89		- if len(output_root_digest.hash) < 64:
	97	+ logger.debug("Output root digest: {}".format(output_digest))
	98	+
	99	+ if len(output_digest.hash) < 64:
90	100	logger.warning("Buildbox command failed - no output root digest present.")
91		- output_file = remote_execution_pb2.OutputDirectory(tree_digest=output_root_digest)
92	101
93		- action_result = remote_execution_pb2.ActionResult()
94		- action_result.output_directories.extend([output_file])
	102	+ # TODO: Have BuildBox helping us creating the Tree instance here
	103	+ # See https://gitlab.com/BuildStream/buildbox/issues/7 for details
	104	+ output_tree = _cas_tree_maker(stub_bytestream, output_digest)
	105	+
	106	+ with upload(context.cas_channel) as cas:
	107	+ output_tree_digest = cas.send_message(output_tree)
95	108
96		- action_result_any = any_pb2.Any()
97		- action_result_any.Pack(action_result)
	109	+ output_directory = remote_execution_pb2.OutputDirectory()
	110	+ output_directory.tree_digest.CopyFrom(output_tree_digest)
	111	+ output_directory.path = os.path.relpath(working_directory, start='/')
98	112
99		- lease.result.CopyFrom(action_result_any)
	113	+ action_result = remote_execution_pb2.ActionResult()
	114	+ action_result.output_directories.extend([output_directory])
	115	+
	116	+ action_result_any = any_pb2.Any()
	117	+ action_result_any.Pack(action_result)
	118	+
	119	+ lease.result.CopyFrom(action_result_any)
100	120
101	121	return lease
	122	+
	123	+
	124	+def _cas_tree_maker(stub_bytestream, directory_digest):
	125	+ # Generates and stores a Tree for a given Directory. This is very inefficient
	126	+ # and only temporary. See https://gitlab.com/BuildStream/buildbox/issues/7.
	127	+ output_tree = remote_execution_pb2.Tree()
	128	+
	129	+ def list_directories(parent_directory):
	130	+ directory_list = list()
	131	+ for directory_node in parent_directory.directories:
	132	+ directory = parse_to_pb2_from_fetch(remote_execution_pb2.Directory(),
	133	+ stub_bytestream, directory_node.digest)
	134	+ directory_list.extend(list_directories(directory))
	135	+ directory_list.append(directory)
	136	+
	137	+ return directory_list
	138	+
	139	+ root_directory = parse_to_pb2_from_fetch(remote_execution_pb2.Directory(),
	140	+ stub_bytestream, directory_digest)
	141	+ output_tree.children.extend(list_directories(root_directory))
	142	+ output_tree.root.CopyFrom(root_directory)
	143	+
	144	+ return output_tree

buildgrid/_app/commands/cmd_server.py

---

...	...	@@ -53,10 +53,16 @@ def start(context, config):
53	53	insecure_mode = server_settings['insecure-mode']
54	54
55	55	credentials = None
	56	+ credentials_settings = server_settings.get('credentials')
56	57	if not insecure_mode:
57		- server_key = server_settings['tls-server-key']
58		- server_cert = server_settings['tls-server-cert']
59		- client_certs = server_settings['tls-client-certs']
	58	+ if not credentials_settings:
	59	+ click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
	60	+ "Set `insecure-mode: false` in order to deactivate TLS encryption.\n", err=True)
	61	+ sys.exit(-1)
	62	+
	63	+ server_key = credentials_settings['tls-server-key']
	64	+ server_cert = credentials_settings['tls-server-cert']
	65	+ client_certs = credentials_settings['tls-client-certs']
60	66	credentials = context.load_server_credentials(server_key, server_cert, client_certs)
61	67
62	68	if not credentials:

buildgrid/_app/settings/__init__.py

---

buildgrid/_app/settings/cas.yml

---

	1	+server:
	2	+ port: 50052
	3	+ insecure-mode: true
	4	+ tls-server-key: null
	5	+ tls-server-cert: null
	6	+ tls-client-certs: null
	7	+
	8	+description: |
	9	+ Just a CAS.
	10	+
	11	+instances:
	12	+ - name: main
	13	+ description: |
	14	+ The main server
	15	+
	16	+ storages:
	17	+ - !disk-storage &main-storage
	18	+ path: ~/cas/
	19	+
	20	+ services:
	21	+ - !cas
	22	+ storage: *main-storage

buildgrid/_app/settings/default.yml

---

1	1	server:
2	2	port: 50051
	3	+ insecure-mode: true
3	4	tls-server-key: null
4	5	tls-server-cert: null
5	6	tls-client-certs: null
6		- insecure-mode: true
7	7
8	8	description: |
9	9	A single default instance

buildgrid/_app/settings/parser.py

---

...	...	@@ -14,7 +14,11 @@
14	14
15	15
16	16	import os
	17	+import sys
	18	+from urllib.parse import urlparse
17	19
	20	+import click
	21	+import grpc
18	22	import yaml
19	23
20	24	from buildgrid.server.controller import ExecutionController
...	...	@@ -22,9 +26,12 @@ from buildgrid.server.actioncache.storage import ActionCache
22	26	from buildgrid.server.cas.instance import ByteStreamInstance, ContentAddressableStorageInstance
23	27	from buildgrid.server.cas.storage.disk import DiskStorage
24	28	from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
	29	+from buildgrid.server.cas.storage.remote import RemoteStorage
25	30	from buildgrid.server.cas.storage.s3 import S3Storage
26	31	from buildgrid.server.cas.storage.with_cache import WithCacheStorage
27	32
	33	+from ..cli import Context
	34	+
28	35
29	36	class YamlFactory(yaml.YAMLObject):
30	37	@classmethod
...	...	@@ -58,6 +65,45 @@ class S3(YamlFactory):
58	65	return S3Storage(bucket, endpoint_url=endpoint)
59	66
60	67
	68	+class Remote(YamlFactory):
	69	+
	70	+ yaml_tag = u'!remote-storage'
	71	+
	72	+ def __new__(cls, url, credentials=None):
	73	+ # TODO: Context could be passed into the parser.
	74	+ context = Context()
	75	+
	76	+ url = urlparse(url)
	77	+ remote = '{}:{}'.format(url.hostname, url.port or 50051)
	78	+
	79	+ channel = None
	80	+ if url.scheme == 'http':
	81	+ channel = grpc.insecure_channel(remote)
	82	+
	83	+ else:
	84	+ if not credentials:
	85	+ click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
	86	+ "Set remote url scheme to `http` in order to deactivate" +
	87	+ "TLS encryption.\n", err=True)
	88	+ sys.exit(-1)
	89	+
	90	+ client_key = credentials['tls-client-key']
	91	+ client_cert = credentials['tls-client-cert']
	92	+ server_cert = credentials['tls-server-cert']
	93	+ credentials = context.load_client_credentials(client_key,
	94	+ client_cert,
	95	+ server_cert)
	96	+ if not credentials:
	97	+ click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
	98	+ "Set remote url scheme to `http` in order to deactivate" +
	99	+ "TLS encryption.\n", err=True)
	100	+ sys.exit(-1)
	101	+
	102	+ channel = grpc.secure_channel(remote, credentials)
	103	+
	104	+ return RemoteStorage(channel)
	105	+
	106	+
61	107	class WithCache(YamlFactory):
62	108
63	109	yaml_tag = u'!with-cache-storage'
...	...	@@ -118,6 +164,7 @@ def get_parser():
118	164	yaml.SafeLoader.add_constructor(Disk.yaml_tag, Disk.from_yaml)
119	165	yaml.SafeLoader.add_constructor(LRU.yaml_tag, LRU.from_yaml)
120	166	yaml.SafeLoader.add_constructor(S3.yaml_tag, S3.from_yaml)
	167	+ yaml.SafeLoader.add_constructor(Remote.yaml_tag, Remote.from_yaml)
121	168	yaml.SafeLoader.add_constructor(WithCache.yaml_tag, WithCache.from_yaml)
122	169	yaml.SafeLoader.add_constructor(CAS.yaml_tag, CAS.from_yaml)
123	170	yaml.SafeLoader.add_constructor(ByteStream.yaml_tag, ByteStream.from_yaml)

buildgrid/_app/settings/remote-storage.yml

---

	1	+server:
	2	+ port: 50051
	3	+ insecure-mode: false
	4	+ tls-server-key: null
	5	+ tls-server-cert: null
	6	+ tls-client-certs: null
	7	+
	8	+
	9	+description: |
	10	+ A single default instance with remote storage.
	11	+
	12	+instances:
	13	+ - name: main
	14	+ description: |
	15	+ The main server
	16	+
	17	+ storages:
	18	+ - !remote-storage &main-storage
	19	+ url: "https://localhost:50052"
	20	+ credentials:
	21	+ tls-client-key: null
	22	+ tls-client-cert: null
	23	+ tls-server-cert: null
	24	+
	25	+ services:
	26	+ - !action-cache &main-action
	27	+ storage: *main-storage
	28	+ max_cached_refs: 256
	29	+ allow_updates: true
	30	+
	31	+ - !execution
	32	+ storage: *main-storage
	33	+ action_cache: *main-action
	34	+
	35	+ - !cas
	36	+ storage: *main-storage
	37	+
	38	+ - !bytestream
	39	+ storage: *main-storage

buildgrid/utils.py

---

...	...	@@ -304,6 +304,21 @@ def read_file(file_path):
304	304	return byte_file.read()
305	305
306	306
	307	+def write_file(file_path, content):
	308	+ """Dumps raw memory content to a file.
	309	+
	310	+ Args:
	311	+ file_path (str): path to the target file.
	312	+ content (bytes): raw file's content.
	313	+
	314	+ Raises:
	315	+ OSError: If `file_path` does not exist or is not writable.
	316	+ """
	317	+ with open(file_path, 'wb') as byte_file:
	318	+ byte_file.write(content)
	319	+ byte_file.flush()
	320	+
	321	+
307	322	def output_file_maker(file_path, input_path, cas=None):
308	323	"""Creates an :obj:`OutputFile` from a local file and possibly upload it.
309	324

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.