finn pushed to branch finn/separate-services at BuildGrid / buildgrid
Commits:
-
5cef4f30
by finn at 2018-09-06T15:51:31Z
15 changed files:
- buildgrid/_app/cli.py
- buildgrid/_app/commands/cmd_server.py
- + buildgrid/_app/server.py
- + buildgrid/_app/settings/default.yml
- + buildgrid/_app/settings/parser.py
- buildgrid/server/actioncache/service.py
- buildgrid/server/bots/service.py
- − buildgrid/server/buildgrid_server.py
- buildgrid/server/cas/service.py
- buildgrid/server/instance.py → buildgrid/server/controller.py
- buildgrid/server/execution/service.py
- buildgrid/server/operations/service.py
- buildgrid/server/referencestorage/service.py
- buildgrid/server/scheduler.py
- setup.py
Changes:
... | ... | @@ -174,7 +174,7 @@ class BuildGridCLI(click.MultiCommand): |
174 | 174 |
mod = __import__(name='buildgrid._app.commands.cmd_{}'.format(name),
|
175 | 175 |
fromlist=['cli'])
|
176 | 176 |
except ImportError:
|
177 |
- return None
|
|
177 |
+ raise
|
|
178 | 178 |
return mod.cli
|
179 | 179 |
|
180 | 180 |
|
... | ... | @@ -26,16 +26,14 @@ import sys |
26 | 26 |
|
27 | 27 |
import click
|
28 | 28 |
|
29 |
-from buildgrid.server import buildgrid_server
|
|
30 |
-from buildgrid.server.cas.storage.disk import DiskStorage
|
|
31 |
-from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
|
|
32 |
-from buildgrid.server.cas.storage.s3 import S3Storage
|
|
33 |
-from buildgrid.server.cas.storage.with_cache import WithCacheStorage
|
|
29 |
+from buildgrid.server.controller import ExecutionController
|
|
34 | 30 |
from buildgrid.server.actioncache.storage import ActionCache
|
31 |
+from buildgrid.server.cas.instance import ByteStreamInstance, ContentAddressableStorageInstance
|
|
32 |
+from buildgrid.server.referencestorage.storage import ReferenceCache
|
|
35 | 33 |
|
36 | 34 |
from ..cli import pass_context
|
37 |
- |
|
38 |
-_SIZE_PREFIXES = {'k': 2 ** 10, 'm': 2 ** 20, 'g': 2 ** 30, 't': 2 ** 40}
|
|
35 |
+from ..settings import parser
|
|
36 |
+from ..server import BuildGridServer
|
|
39 | 37 |
|
40 | 38 |
|
41 | 39 |
@click.group(name='server', short_help="Start a local server instance.")
|
... | ... | @@ -45,71 +43,31 @@ def cli(context): |
45 | 43 |
|
46 | 44 |
|
47 | 45 |
@cli.command('start', short_help="Setup a new server instance.")
|
48 |
-@click.argument('instances', nargs=-1, type=click.STRING)
|
|
49 |
-@click.option('--port', type=click.INT, default='50051', show_default=True,
|
|
50 |
- help="The port number to be listened.")
|
|
51 |
-@click.option('--server-key', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
52 |
- help="Private server key for TLS (PEM-encoded)")
|
|
53 |
-@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
54 |
- help="Public server certificate for TLS (PEM-encoded)")
|
|
55 |
-@click.option('--client-certs', type=click.Path(exists=True, dir_okay=False), default=None,
|
|
56 |
- help="Public client certificates for TLS (PEM-encoded, one single file)")
|
|
57 |
-@click.option('--allow-insecure', type=click.BOOL, is_flag=True,
|
|
58 |
- help="Whether or not to allow unencrypted connections.")
|
|
59 |
-@click.option('--allow-update-action-result/--forbid-update-action-result',
|
|
60 |
- 'allow_uar', default=True, show_default=True,
|
|
61 |
- help="Whether or not to allow clients to manually edit the action cache.")
|
|
62 |
-@click.option('--max-cached-actions', type=click.INT, default=50, show_default=True,
|
|
63 |
- help="Maximum number of actions to keep in the ActionCache.")
|
|
64 |
-@click.option('--cas', type=click.Choice(('lru', 's3', 'disk', 'with-cache')),
|
|
65 |
- help="The CAS storage type to use.")
|
|
66 |
-@click.option('--cas-cache', type=click.Choice(('lru', 's3', 'disk')),
|
|
67 |
- help="For --cas=with-cache, the CAS storage to use as the cache.")
|
|
68 |
-@click.option('--cas-fallback', type=click.Choice(('lru', 's3', 'disk')),
|
|
69 |
- help="For --cas=with-cache, the CAS storage to use as the fallback.")
|
|
70 |
-@click.option('--cas-lru-size', type=click.STRING,
|
|
71 |
- help="For --cas=lru, the LRU cache's memory limit.")
|
|
72 |
-@click.option('--cas-s3-bucket', type=click.STRING,
|
|
73 |
- help="For --cas=s3, the bucket name.")
|
|
74 |
-@click.option('--cas-s3-endpoint', type=click.STRING,
|
|
75 |
- help="For --cas=s3, the endpoint URI.")
|
|
76 |
-@click.option('--cas-disk-directory', type=click.Path(file_okay=False, dir_okay=True, writable=True),
|
|
77 |
- help="For --cas=disk, the folder to store CAS blobs in.")
|
|
46 |
+@click.argument('yml', type=click.Path(file_okay=True, dir_okay=False, writable=False))
|
|
78 | 47 |
@pass_context
|
79 |
-def start(context, port, allow_insecure, server_key, server_cert, client_certs,
|
|
80 |
- instances, max_cached_actions, allow_uar, cas, **cas_args):
|
|
81 |
- """Setups a new server instance."""
|
|
82 |
- credentials = None
|
|
83 |
- if not allow_insecure:
|
|
84 |
- credentials = context.load_server_credentials(server_key, server_cert, client_certs)
|
|
85 |
- if not credentials and not allow_insecure:
|
|
86 |
- click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
|
|
87 |
- "Use --allow-insecure in order to deactivate TLS encryption.\n", err=True)
|
|
88 |
- sys.exit(-1)
|
|
89 |
- |
|
90 |
- context.credentials = credentials
|
|
91 |
- context.port = port
|
|
92 |
- |
|
93 |
- context.logger.info("BuildGrid server booting up")
|
|
94 |
- context.logger.info("Starting on port {}".format(port))
|
|
95 |
- |
|
96 |
- cas_storage = _make_cas_storage(context, cas, cas_args)
|
|
97 |
- |
|
98 |
- if cas_storage is None:
|
|
99 |
- context.logger.info("Running without CAS - action cache will be unavailable")
|
|
100 |
- action_cache = None
|
|
101 |
- |
|
102 |
- else:
|
|
103 |
- action_cache = ActionCache(cas_storage, max_cached_actions, allow_uar)
|
|
104 |
- |
|
105 |
- if instances is None:
|
|
106 |
- instances = ['main']
|
|
107 |
- |
|
108 |
- server = buildgrid_server.BuildGridServer(port=context.port,
|
|
109 |
- credentials=context.credentials,
|
|
110 |
- instances=instances,
|
|
111 |
- cas_storage=cas_storage,
|
|
112 |
- action_cache=action_cache)
|
|
48 |
+def start(context, yml):
|
|
49 |
+ with open(yml) as f:
|
|
50 |
+ settings = parser.get_parser().safe_load(f)
|
|
51 |
+ |
|
52 |
+ server_settings = settings['server']
|
|
53 |
+ |
|
54 |
+ instances = settings['instances']
|
|
55 |
+ |
|
56 |
+ execution_controllers = _instance_maker(instances, ExecutionController)
|
|
57 |
+ reference_caches = _instance_maker(instances, ReferenceCache)
|
|
58 |
+ action_caches = _instance_maker(instances, ActionCache)
|
|
59 |
+ cas = _instance_maker(instances, ContentAddressableStorageInstance)
|
|
60 |
+ bytestreams = _instance_maker(instances, ByteStreamInstance)
|
|
61 |
+ |
|
62 |
+ port = server_settings['port']
|
|
63 |
+ server = BuildGridServer(port=port,
|
|
64 |
+ execution_controller=execution_controllers,
|
|
65 |
+ reference_storage_instances=reference_caches,
|
|
66 |
+ action_cache_instances=action_caches,
|
|
67 |
+ cas_instances=cas,
|
|
68 |
+ bytestream_instances=bytestreams)
|
|
69 |
+ |
|
70 |
+ context.logger.info("Starting server on port {}".format(port))
|
|
113 | 71 |
loop = asyncio.get_event_loop()
|
114 | 72 |
try:
|
115 | 73 |
server.start()
|
... | ... | @@ -119,57 +77,20 @@ def start(context, port, allow_insecure, server_key, server_cert, client_certs, |
119 | 77 |
pass
|
120 | 78 |
|
121 | 79 |
finally:
|
80 |
+ context.logger.info("Stopping server")
|
|
122 | 81 |
server.stop()
|
123 | 82 |
loop.close()
|
124 | 83 |
|
125 | 84 |
|
126 |
-def _make_cas_storage(context, cas_type, cas_args):
|
|
127 |
- """Returns the storage provider corresponding to the given `cas_type`,
|
|
128 |
- or None if the provider cannot be created.
|
|
129 |
- """
|
|
130 |
- if cas_type == "lru":
|
|
131 |
- if cas_args["cas_lru_size"] is None:
|
|
132 |
- context.logger.error("--cas-lru-size is required for LRU CAS")
|
|
133 |
- return None
|
|
134 |
- try:
|
|
135 |
- size = _parse_size(cas_args["cas_lru_size"])
|
|
136 |
- except ValueError:
|
|
137 |
- context.logger.error('Invalid LRU size "{0}"'.format(cas_args["cas_lru_size"]))
|
|
138 |
- return None
|
|
139 |
- return LRUMemoryCache(size)
|
|
140 |
- elif cas_type == "s3":
|
|
141 |
- if cas_args["cas_s3_bucket"] is None:
|
|
142 |
- context.logger.error("--cas-s3-bucket is required for S3 CAS")
|
|
143 |
- return None
|
|
144 |
- if cas_args["cas_s3_endpoint"] is not None:
|
|
145 |
- return S3Storage(cas_args["cas_s3_bucket"],
|
|
146 |
- endpoint_url=cas_args["cas_s3_endpoint"])
|
|
147 |
- return S3Storage(cas_args["cas_s3_bucket"])
|
|
148 |
- elif cas_type == "disk":
|
|
149 |
- if cas_args["cas_disk_directory"] is None:
|
|
150 |
- context.logger.error("--cas-disk-directory is required for disk CAS")
|
|
151 |
- return None
|
|
152 |
- return DiskStorage(cas_args["cas_disk_directory"])
|
|
153 |
- elif cas_type == "with-cache":
|
|
154 |
- cache = _make_cas_storage(context, cas_args["cas_cache"], cas_args)
|
|
155 |
- fallback = _make_cas_storage(context, cas_args["cas_fallback"], cas_args)
|
|
156 |
- if cache is None:
|
|
157 |
- context.logger.error("Missing cache provider for --cas=with-cache")
|
|
158 |
- return None
|
|
159 |
- elif fallback is None:
|
|
160 |
- context.logger.error("Missing fallback provider for --cas=with-cache")
|
|
161 |
- return None
|
|
162 |
- return WithCacheStorage(cache, fallback)
|
|
163 |
- elif cas_type is None:
|
|
164 |
- return None
|
|
165 |
- return None
|
|
166 |
- |
|
167 |
- |
|
168 |
-def _parse_size(size):
|
|
169 |
- """Convert a string containing a size in bytes (e.g. '2GB') to a number."""
|
|
170 |
- size = size.lower()
|
|
171 |
- if size[-1] == 'b':
|
|
172 |
- size = size[:-1]
|
|
173 |
- if size[-1] in _SIZE_PREFIXES:
|
|
174 |
- return int(size[:-1]) * _SIZE_PREFIXES[size[-1]]
|
|
175 |
- return int(size)
|
|
85 |
+# Turn away now if you want to keep your eyes
|
|
86 |
+def _instance_maker(instances, service_type):
|
|
87 |
+ # TODO get this mapped in parser
|
|
88 |
+ made = {}
|
|
89 |
+ |
|
90 |
+ for instance in instances:
|
|
91 |
+ services = instance['services']
|
|
92 |
+ instance_name = instance['name']
|
|
93 |
+ for service in services:
|
|
94 |
+ if type(service) == service_type:
|
|
95 |
+ made[instance_name] = service
|
|
96 |
+ return made
|
1 |
+# Copyright (C) 2018 Bloomberg LP
|
|
2 |
+#
|
|
3 |
+# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4 |
+# you may not use this file except in compliance with the License.
|
|
5 |
+# You may obtain a copy of the License at
|
|
6 |
+#
|
|
7 |
+# <http://www.apache.org/licenses/LICENSE-2.0>
|
|
8 |
+#
|
|
9 |
+# Unless required by applicable law or agreed to in writing, software
|
|
10 |
+# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11 |
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12 |
+# See the License for the specific language governing permissions and
|
|
13 |
+# limitations under the License.
|
|
14 |
+ |
|
15 |
+ |
|
16 |
+"""
|
|
17 |
+BuildGridServer
|
|
18 |
+==============
|
|
19 |
+ |
|
20 |
+Creates the user a local BuildGrid server.
|
|
21 |
+"""
|
|
22 |
+ |
|
23 |
+import logging
|
|
24 |
+from concurrent import futures
|
|
25 |
+ |
|
26 |
+import grpc
|
|
27 |
+ |
|
28 |
+from buildgrid.server.cas.service import ByteStreamService, ContentAddressableStorageService
|
|
29 |
+from buildgrid.server.actioncache.service import ActionCacheService
|
|
30 |
+from buildgrid.server.execution.service import ExecutionService
|
|
31 |
+from buildgrid.server.operations.service import OperationsService
|
|
32 |
+from buildgrid.server.bots.service import BotsService
|
|
33 |
+ |
|
34 |
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc
|
|
35 |
+ |
|
36 |
+ |
|
37 |
+class BuildGridServer:
|
|
38 |
+ |
|
39 |
+ def __init__(self, port=50051, max_workers=10, credentials=None,
|
|
40 |
+ execution_controller=None, reference_storage_instances = None,
|
|
41 |
+ action_cache_instances=None, cas_instances=None, bytestream_instances = None):
|
|
42 |
+ |
|
43 |
+ self.logger = logging.getLogger(__name__)
|
|
44 |
+ address = '[::]:{0}'.format(port)
|
|
45 |
+ |
|
46 |
+ server = grpc.server(futures.ThreadPoolExecutor(max_workers))
|
|
47 |
+ |
|
48 |
+ if credentials is not None:
|
|
49 |
+ self.logger.info("Secure connection")
|
|
50 |
+ server.add_secure_port(address, credentials)
|
|
51 |
+ |
|
52 |
+ else:
|
|
53 |
+ self.logger.info("Insecure connection")
|
|
54 |
+ server.add_insecure_port(address)
|
|
55 |
+ |
|
56 |
+ if execution_controller:
|
|
57 |
+ self.logger.info("Adding execution controllers {}".format(
|
|
58 |
+ execution_controller.keys()))
|
|
59 |
+ ExecutionService(server, execution_controller)
|
|
60 |
+ BotsService(server, execution_controller)
|
|
61 |
+ OperationsService(server, execution_controller)
|
|
62 |
+ |
|
63 |
+ if reference_storage_instances:
|
|
64 |
+ ReferenceStorageService(server, reference_storage_instances)
|
|
65 |
+ |
|
66 |
+ if action_cache_instances:
|
|
67 |
+ ActionCacheService(server, action_cache_instances)
|
|
68 |
+ |
|
69 |
+ if cas_instances:
|
|
70 |
+ ContentAddressableStorageService(server, cas_instances)
|
|
71 |
+ |
|
72 |
+ if bytestream_instances:
|
|
73 |
+ ByteStreamService(server, bytestream_instances)
|
|
74 |
+ |
|
75 |
+ self._server = server
|
|
76 |
+ |
|
77 |
+ def start(self):
|
|
78 |
+ self._server.start()
|
|
79 |
+ |
|
80 |
+ def stop(self):
|
|
81 |
+ self._server.stop(0)
|
1 |
+server:
|
|
2 |
+ port: 50051
|
|
3 |
+ tls-server-key: null
|
|
4 |
+ tls-server-cert: null
|
|
5 |
+ tls-client-certs: null
|
|
6 |
+ insecure-mode: true
|
|
7 |
+ |
|
8 |
+description: |
|
|
9 |
+ A single default instance
|
|
10 |
+ |
|
11 |
+instances:
|
|
12 |
+ - name: main
|
|
13 |
+ description: |
|
|
14 |
+ The main server
|
|
15 |
+ |
|
16 |
+ storages:
|
|
17 |
+ - !disk-storage &main-storage
|
|
18 |
+ path: ~/
|
|
19 |
+ |
|
20 |
+ - !lru-storage &main-lru-storage
|
|
21 |
+ size: 10mb
|
|
22 |
+ |
|
23 |
+ services:
|
|
24 |
+ - !action-cache &main-action
|
|
25 |
+ storage: *main-storage
|
|
26 |
+ max_cached_refs: 256
|
|
27 |
+ allow_updates: true
|
|
28 |
+ |
|
29 |
+ - !execution
|
|
30 |
+ storage: *main-storage
|
|
31 |
+ action_cache: *main-action
|
|
32 |
+ |
|
33 |
+ - !cas
|
|
34 |
+ storage: *main-storage
|
|
35 |
+ |
|
36 |
+ - !bytestream
|
|
37 |
+ storage: *main-storage
|
1 |
+# Copyright (C) 2018 Bloomberg LP
|
|
2 |
+#
|
|
3 |
+# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4 |
+# you may not use this file except in compliance with the License.
|
|
5 |
+# You may obtain a copy of the License at
|
|
6 |
+#
|
|
7 |
+# <http://www.apache.org/licenses/LICENSE-2.0>
|
|
8 |
+#
|
|
9 |
+# Unless required by applicable law or agreed to in writing, software
|
|
10 |
+# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11 |
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12 |
+# See the License for the specific language governing permissions and
|
|
13 |
+# limitations under the License.
|
|
14 |
+ |
|
15 |
+ |
|
16 |
+import sys
|
|
17 |
+ |
|
18 |
+import yaml
|
|
19 |
+ |
|
20 |
+from buildgrid.server.controller import ExecutionController
|
|
21 |
+from buildgrid.server.scheduler import Scheduler
|
|
22 |
+from buildgrid.server.actioncache.storage import ActionCache
|
|
23 |
+from buildgrid.server.cas.instance import ByteStreamInstance, ContentAddressableStorageInstance
|
|
24 |
+from buildgrid.server.cas.storage.disk import DiskStorage
|
|
25 |
+from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
|
|
26 |
+ |
|
27 |
+ |
|
28 |
+class YamlFactory(yaml.YAMLObject):
|
|
29 |
+ @classmethod
|
|
30 |
+ def from_yaml(cls, loader, node):
|
|
31 |
+ values = loader.construct_mapping(node, deep=True)
|
|
32 |
+ return cls(**values)
|
|
33 |
+ |
|
34 |
+ |
|
35 |
+class Disk(YamlFactory):
|
|
36 |
+ |
|
37 |
+ yaml_tag = u'!disk-storage'
|
|
38 |
+ |
|
39 |
+ def __new__(cls, path):
|
|
40 |
+ return DiskStorage(path)
|
|
41 |
+ |
|
42 |
+ |
|
43 |
+class LRU(YamlFactory):
|
|
44 |
+ |
|
45 |
+ yaml_tag = u'!lru-storage'
|
|
46 |
+ |
|
47 |
+ def __new__(cls, size):
|
|
48 |
+ return LRUMemoryCache(_parse_size(size))
|
|
49 |
+ |
|
50 |
+ |
|
51 |
+class Execution(YamlFactory):
|
|
52 |
+ |
|
53 |
+ yaml_tag = u'!execution'
|
|
54 |
+ |
|
55 |
+ def __new__(cls, storage, action_cache=None):
|
|
56 |
+ return ExecutionController(action_cache, storage)
|
|
57 |
+ |
|
58 |
+ |
|
59 |
+class Action(YamlFactory):
|
|
60 |
+ |
|
61 |
+ yaml_tag = u'!action-cache'
|
|
62 |
+ |
|
63 |
+ def __new__(cls, storage, max_cached_refs=0, allow_updates=True):
|
|
64 |
+ return ActionCache(storage, max_cached_refs, allow_updates)
|
|
65 |
+ |
|
66 |
+ |
|
67 |
+class CAS(YamlFactory):
|
|
68 |
+ |
|
69 |
+ yaml_tag = u'!cas'
|
|
70 |
+ |
|
71 |
+ def __new__(cls, storage):
|
|
72 |
+ return ContentAddressableStorageInstance(storage)
|
|
73 |
+ |
|
74 |
+ |
|
75 |
+class ByteStream(YamlFactory):
|
|
76 |
+ |
|
77 |
+ yaml_tag = u'!bytestream'
|
|
78 |
+ |
|
79 |
+ def __new__(cls, storage):
|
|
80 |
+ return ByteStreamInstance(storage)
|
|
81 |
+ |
|
82 |
+ |
|
83 |
+def _parse_size(size):
|
|
84 |
+ """Convert a string containing a size in bytes (e.g. '2GB') to a number."""
|
|
85 |
+ _size_prefixes = {'k': 2 ** 10, 'm': 2 ** 20, 'g': 2 ** 30, 't': 2 ** 40}
|
|
86 |
+ size = size.lower()
|
|
87 |
+ |
|
88 |
+ if size[-1] == 'b':
|
|
89 |
+ size = size[:-1]
|
|
90 |
+ if size[-1] in _size_prefixes:
|
|
91 |
+ return int(size[:-1]) * _size_prefixes[size[-1]]
|
|
92 |
+ return int(size)
|
|
93 |
+ |
|
94 |
+def get_parser():
|
|
95 |
+ |
|
96 |
+ yaml.SafeLoader.add_constructor(Execution.yaml_tag, Execution.from_yaml)
|
|
97 |
+ yaml.SafeLoader.add_constructor(Execution.yaml_tag, Execution.from_yaml)
|
|
98 |
+ yaml.SafeLoader.add_constructor(Action.yaml_tag, Action.from_yaml)
|
|
99 |
+ yaml.SafeLoader.add_constructor(Disk.yaml_tag, Disk.from_yaml)
|
|
100 |
+ yaml.SafeLoader.add_constructor(LRU.yaml_tag, LRU.from_yaml)
|
|
101 |
+ yaml.SafeLoader.add_constructor(CAS.yaml_tag, CAS.from_yaml)
|
|
102 |
+ yaml.SafeLoader.add_constructor(ByteStream.yaml_tag, ByteStream.from_yaml)
|
|
103 |
+ |
|
104 |
+ return yaml
|
... | ... | @@ -32,13 +32,22 @@ from .._exceptions import NotFoundError |
32 | 32 |
|
33 | 33 |
class ActionCacheService(remote_execution_pb2_grpc.ActionCacheServicer):
|
34 | 34 |
|
35 |
- def __init__(self, action_cache):
|
|
36 |
- self._action_cache = action_cache
|
|
35 |
+ def __init__(self, server, instances):
|
|
36 |
+ self._instances = instances
|
|
37 |
+ |
|
37 | 38 |
self.logger = logging.getLogger(__name__)
|
38 | 39 |
|
40 |
+ remote_execution_pb2_grpc.add_ActionCacheServicer_to_server(self, server)
|
|
41 |
+ |
|
39 | 42 |
def GetActionResult(self, request, context):
|
40 | 43 |
try:
|
41 |
- return self._action_cache.get_action_result(request.action_digest)
|
|
44 |
+ instance = self._get_instance(request.instance_name)
|
|
45 |
+ return instance.get_action_result(request.action_digest)
|
|
46 |
+ |
|
47 |
+ except InvalidArgumentError as e:
|
|
48 |
+ self.logger.error(e)
|
|
49 |
+ context.set_details(str(e))
|
|
50 |
+ context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
|
|
42 | 51 |
|
43 | 52 |
except NotFoundError as e:
|
44 | 53 |
self.logger.error(e)
|
... | ... | @@ -48,11 +57,24 @@ class ActionCacheService(remote_execution_pb2_grpc.ActionCacheServicer): |
48 | 57 |
|
49 | 58 |
def UpdateActionResult(self, request, context):
|
50 | 59 |
try:
|
51 |
- self._action_cache.update_action_result(request.action_digest, request.action_result)
|
|
60 |
+ instance = self._get_instance(request.instance_name)
|
|
61 |
+ instance.update_action_result(request.action_digest, request.action_result)
|
|
52 | 62 |
return request.action_result
|
53 | 63 |
|
64 |
+ except InvalidArgumentError as e:
|
|
65 |
+ self.logger.error(e)
|
|
66 |
+ context.set_details(str(e))
|
|
67 |
+ context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
|
|
68 |
+ |
|
54 | 69 |
except NotImplementedError as e:
|
55 | 70 |
self.logger.error(e)
|
56 | 71 |
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
57 | 72 |
|
58 | 73 |
return remote_execution_pb2.ActionResult()
|
74 |
+ |
|
75 |
+ def _get_instance(self, instance_name):
|
|
76 |
+ try:
|
|
77 |
+ return self._instances[instance_name]
|
|
78 |
+ |
|
79 |
+ except KeyError:
|
|
80 |
+ raise InvalidArgumentError("Invalid instance name: {}".format(instance_name))
|
... | ... | @@ -33,10 +33,12 @@ from .._exceptions import InvalidArgumentError, OutofSyncError |
33 | 33 |
|
34 | 34 |
class BotsService(bots_pb2_grpc.BotsServicer):
|
35 | 35 |
|
36 |
- def __init__(self, instances):
|
|
36 |
+ def __init__(self, server, instances):
|
|
37 | 37 |
self._instances = instances
|
38 | 38 |
self.logger = logging.getLogger(__name__)
|
39 | 39 |
|
40 |
+ bots_pb2_grpc.add_BotsServicer_to_server(self, server)
|
|
41 |
+ |
|
40 | 42 |
def CreateBotSession(self, request, context):
|
41 | 43 |
try:
|
42 | 44 |
parent = request.parent
|
1 |
-# Copyright (C) 2018 Bloomberg LP
|
|
2 |
-#
|
|
3 |
-# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4 |
-# you may not use this file except in compliance with the License.
|
|
5 |
-# You may obtain a copy of the License at
|
|
6 |
-#
|
|
7 |
-# <http://www.apache.org/licenses/LICENSE-2.0>
|
|
8 |
-#
|
|
9 |
-# Unless required by applicable law or agreed to in writing, software
|
|
10 |
-# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11 |
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12 |
-# See the License for the specific language governing permissions and
|
|
13 |
-# limitations under the License.
|
|
14 |
- |
|
15 |
- |
|
16 |
-"""
|
|
17 |
-BuildGridServer
|
|
18 |
-==============
|
|
19 |
- |
|
20 |
-Creates the user a local server BuildGrid server.
|
|
21 |
-"""
|
|
22 |
- |
|
23 |
-from concurrent import futures
|
|
24 |
- |
|
25 |
-import grpc
|
|
26 |
- |
|
27 |
-from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
|
|
28 |
-from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc
|
|
29 |
-from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2_grpc
|
|
30 |
-from buildgrid._protos.google.longrunning import operations_pb2_grpc
|
|
31 |
- |
|
32 |
-from .instance import BuildGridInstance
|
|
33 |
-from .cas.service import ByteStreamService, ContentAddressableStorageService
|
|
34 |
-from .actioncache.service import ActionCacheService
|
|
35 |
-from .execution.service import ExecutionService
|
|
36 |
-from .operations.service import OperationsService
|
|
37 |
-from .bots.service import BotsService
|
|
38 |
- |
|
39 |
- |
|
40 |
-class BuildGridServer:
|
|
41 |
- |
|
42 |
- def __init__(self, port=50051, credentials=None, instances=None,
|
|
43 |
- max_workers=10, action_cache=None, cas_storage=None):
|
|
44 |
- address = '[::]:{0}'.format(port)
|
|
45 |
- |
|
46 |
- self._server = grpc.server(futures.ThreadPoolExecutor(max_workers))
|
|
47 |
- |
|
48 |
- if credentials is not None:
|
|
49 |
- self._server.add_secure_port(address, credentials)
|
|
50 |
- else:
|
|
51 |
- self._server.add_insecure_port(address)
|
|
52 |
- |
|
53 |
- if cas_storage is not None:
|
|
54 |
- cas_service = ContentAddressableStorageService(cas_storage)
|
|
55 |
- remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server(cas_service,
|
|
56 |
- self._server)
|
|
57 |
- bytestream_pb2_grpc.add_ByteStreamServicer_to_server(ByteStreamService(cas_storage),
|
|
58 |
- self._server)
|
|
59 |
- if action_cache is not None:
|
|
60 |
- action_cache_service = ActionCacheService(action_cache)
|
|
61 |
- remote_execution_pb2_grpc.add_ActionCacheServicer_to_server(action_cache_service,
|
|
62 |
- self._server)
|
|
63 |
- |
|
64 |
- buildgrid_instances = {}
|
|
65 |
- if not instances:
|
|
66 |
- buildgrid_instances["main"] = BuildGridInstance(action_cache, cas_storage)
|
|
67 |
- else:
|
|
68 |
- for name in instances:
|
|
69 |
- buildgrid_instances[name] = BuildGridInstance(action_cache, cas_storage)
|
|
70 |
- |
|
71 |
- bots_pb2_grpc.add_BotsServicer_to_server(BotsService(buildgrid_instances),
|
|
72 |
- self._server)
|
|
73 |
- remote_execution_pb2_grpc.add_ExecutionServicer_to_server(ExecutionService(buildgrid_instances),
|
|
74 |
- self._server)
|
|
75 |
- operations_pb2_grpc.add_OperationsServicer_to_server(OperationsService(buildgrid_instances),
|
|
76 |
- self._server)
|
|
77 |
- |
|
78 |
- def start(self):
|
|
79 |
- self._server.start()
|
|
80 |
- |
|
81 |
- def stop(self):
|
|
82 |
- self._server.stop(0)
|
... | ... | @@ -27,18 +27,20 @@ import logging |
27 | 27 |
import grpc
|
28 | 28 |
|
29 | 29 |
from buildgrid._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
|
30 |
-from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2
|
|
31 |
-from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc as re_pb2_grpc
|
|
30 |
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
|
|
31 |
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc
|
|
32 | 32 |
|
33 | 33 |
from .._exceptions import InvalidArgumentError, NotFoundError, OutOfRangeError
|
34 | 34 |
|
35 | 35 |
|
36 |
-class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServicer):
|
|
36 |
+class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressableStorageServicer):
|
|
37 | 37 |
|
38 |
- def __init__(self, instances):
|
|
38 |
+ def __init__(self, server, instances):
|
|
39 | 39 |
self.logger = logging.getLogger(__name__)
|
40 | 40 |
self._instances = instances
|
41 | 41 |
|
42 |
+ remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server(self, server)
|
|
43 |
+ |
|
42 | 44 |
def FindMissingBlobs(self, request, context):
|
43 | 45 |
try:
|
44 | 46 |
instance = self._get_instance(request.instance_name)
|
... | ... | @@ -49,7 +51,7 @@ class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServ |
49 | 51 |
context.set_details(str(e))
|
50 | 52 |
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
|
51 | 53 |
|
52 |
- return re_pb2.FindMissingBlobsResponse()
|
|
54 |
+ return remote_execution_pb2.FindMissingBlobsResponse()
|
|
53 | 55 |
|
54 | 56 |
def BatchUpdateBlobs(self, request, context):
|
55 | 57 |
try:
|
... | ... | @@ -61,7 +63,7 @@ class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServ |
61 | 63 |
context.set_details(str(e))
|
62 | 64 |
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
|
63 | 65 |
|
64 |
- return re_pb2.BatchReadBlobsResponse()
|
|
66 |
+ return remote_execution_pb2.BatchReadBlobsResponse()
|
|
65 | 67 |
|
66 | 68 |
def _get_instance(self, instance_name):
|
67 | 69 |
try:
|
... | ... | @@ -73,10 +75,12 @@ class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServ |
73 | 75 |
|
74 | 76 |
class ByteStreamService(bytestream_pb2_grpc.ByteStreamServicer):
|
75 | 77 |
|
76 |
- def __init__(self, instances):
|
|
78 |
+ def __init__(self, server, instances):
|
|
77 | 79 |
self.logger = logging.getLogger(__name__)
|
78 | 80 |
self._instances = instances
|
79 | 81 |
|
82 |
+ bytestream_pb2_grpc.add_ByteStreamServicer_to_server(self, server)
|
|
83 |
+ |
|
80 | 84 |
def Read(self, request, context):
|
81 | 85 |
try:
|
82 | 86 |
path = request.resource_name.split("/")
|
... | ... | @@ -14,31 +14,36 @@ |
14 | 14 |
|
15 | 15 |
|
16 | 16 |
"""
|
17 |
-BuildGrid Instance
|
|
17 |
+Execution Controller
|
|
18 | 18 |
==================
|
19 | 19 |
|
20 |
-An instance of the BuildGrid server.
|
|
20 |
+An instance of the Execution controller.
|
|
21 | 21 |
|
22 |
-Contains scheduler, execution instance and an interface to the bots.
|
|
22 |
+All this stuff you need to make the execution service work.
|
|
23 |
+ |
|
24 |
+Contains scheduler, execution instance, an interface to the bots
|
|
25 |
+and an operations instance.
|
|
23 | 26 |
"""
|
24 | 27 |
|
25 | 28 |
|
26 | 29 |
import logging
|
27 | 30 |
|
28 |
-from .execution.instance import ExecutionInstance
|
|
29 | 31 |
from .scheduler import Scheduler
|
30 | 32 |
from .bots.instance import BotsInterface
|
33 |
+from .execution.instance import ExecutionInstance
|
|
34 |
+from .operations.instance import OperationsInstance
|
|
31 | 35 |
|
32 | 36 |
|
33 |
-class BuildGridInstance(ExecutionInstance, BotsInterface):
|
|
37 |
+class ExecutionController(ExecutionInstance, BotsInterface, OperationsInstance):
|
|
34 | 38 |
|
35 |
- def __init__(self, action_cache=None, cas_storage=None):
|
|
39 |
+ def __init__(self, action_cache=None, storage=None):
|
|
36 | 40 |
scheduler = Scheduler(action_cache)
|
37 | 41 |
|
38 | 42 |
self.logger = logging.getLogger(__name__)
|
39 | 43 |
|
40 |
- ExecutionInstance.__init__(self, scheduler, cas_storage)
|
|
44 |
+ ExecutionInstance.__init__(self, scheduler, storage)
|
|
41 | 45 |
BotsInterface.__init__(self, scheduler)
|
46 |
+ OperationsInstance.__init__(self, scheduler)
|
|
42 | 47 |
|
43 | 48 |
def stream_operation_updates(self, message_queue, operation_name):
|
44 | 49 |
operation = message_queue.get()
|
... | ... | @@ -35,10 +35,12 @@ from .._exceptions import InvalidArgumentError |
35 | 35 |
|
36 | 36 |
class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
|
37 | 37 |
|
38 |
- def __init__(self, instances):
|
|
38 |
+ def __init__(self, server, instances):
|
|
39 | 39 |
self.logger = logging.getLogger(__name__)
|
40 | 40 |
self._instances = instances
|
41 | 41 |
|
42 |
+ remote_execution_pb2_grpc.add_ExecutionServicer_to_server(self, server)
|
|
43 |
+ |
|
42 | 44 |
def Execute(self, request, context):
|
43 | 45 |
try:
|
44 | 46 |
message_queue = queue.Queue()
|
... | ... | @@ -29,13 +29,14 @@ from buildgrid._protos.google.longrunning import operations_pb2_grpc, operations |
29 | 29 |
|
30 | 30 |
from .._exceptions import InvalidArgumentError
|
31 | 31 |
|
32 |
- |
|
33 | 32 |
class OperationsService(operations_pb2_grpc.OperationsServicer):
|
34 | 33 |
|
35 |
- def __init__(self, instances):
|
|
34 |
+ def __init__(self, server, instances):
|
|
36 | 35 |
self._instances = instances
|
37 | 36 |
self.logger = logging.getLogger(__name__)
|
38 | 37 |
|
38 |
+ operations_pb2_grpc.add_OperationsServicer_to_server(self, server)
|
|
39 |
+ |
|
39 | 40 |
def GetOperation(self, request, context):
|
40 | 41 |
try:
|
41 | 42 |
name = request.name
|
... | ... | @@ -25,29 +25,64 @@ from .._exceptions import NotFoundError |
25 | 25 |
|
26 | 26 |
class ReferenceStorageService(buildstream_pb2_grpc.ReferenceStorageServicer):
|
27 | 27 |
|
28 |
- def __init__(self, reference_cache):
|
|
28 |
+ def __init__(self, server, instances):
|
|
29 | 29 |
self._reference_cache = reference_cache
|
30 | 30 |
self.logger = logging.getLogger(__name__)
|
31 | 31 |
|
32 |
+ self._instances = instances
|
|
33 |
+ |
|
34 |
+ buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server(self, server)
|
|
35 |
+ |
|
32 | 36 |
def GetReference(self, request, context):
|
33 | 37 |
try:
|
34 |
- response = buildstream_pb2.GetReferenceResponse()
|
|
35 |
- response.digest.CopyFrom(self._reference_cache.get_digest_reference(request.key))
|
|
36 |
- return response
|
|
38 |
+ instance = self._get_instance(request.instance_name)
|
|
39 |
+ digest = instance.get_digest_reference(request.key)
|
|
40 |
+ return buildstream_pb2.GetReferenceResponse(digest)
|
|
41 |
+ |
|
42 |
+ except InvalidArgumentError as e:
|
|
43 |
+ self.logger.error(e)
|
|
44 |
+ context.set_details(str(e))
|
|
45 |
+ context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
|
|
37 | 46 |
|
38 | 47 |
except NotFoundError:
|
39 | 48 |
context.set_code(grpc.StatusCode.NOT_FOUND)
|
40 | 49 |
|
50 |
+ return buildstream_pb2.GetReferenceResponse()
|
|
51 |
+ |
|
41 | 52 |
def UpdateReference(self, request, context):
|
42 | 53 |
try:
|
54 |
+ instance = self._get_instance(request.instance_name)
|
|
55 |
+ digest = request.digest
|
|
56 |
+ |
|
43 | 57 |
for key in request.keys:
|
44 |
- self._reference_cache.update_reference(key, request.digest)
|
|
58 |
+ instance.update_reference(key, digest)
|
|
45 | 59 |
|
46 |
- return buildstream_pb2.UpdateReferenceResponse()
|
|
60 |
+ except InvalidArgumentError as e:
|
|
61 |
+ self.logger.error(e)
|
|
62 |
+ context.set_details(str(e))
|
|
63 |
+ context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
|
|
47 | 64 |
|
48 | 65 |
except NotImplementedError:
|
49 | 66 |
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
|
50 | 67 |
|
68 |
+ return buildstream_pb2.UpdateReferenceResponse()
|
|
69 |
+ |
|
51 | 70 |
def Status(self, request, context):
|
52 |
- allow_updates = self._reference_cache.allow_updates
|
|
53 |
- return buildstream_pb2.StatusResponse(allow_updates=allow_updates)
|
|
71 |
+ try:
|
|
72 |
+ instance = self._get_instance(request.instance_name)
|
|
73 |
+ allow_updates = instance.allow_updates
|
|
74 |
+ return buildstream_pb2.StatusResponse(allow_updates=allow_updates)
|
|
75 |
+ |
|
76 |
+ except InvalidArgumentError as e:
|
|
77 |
+ self.logger.error(e)
|
|
78 |
+ context.set_details(str(e))
|
|
79 |
+ context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
|
|
80 |
+ |
|
81 |
+ return buildstream_pb2.StatusResponse()
|
|
82 |
+ |
|
83 |
+ def _get_instance(self, instance_name):
|
|
84 |
+ try:
|
|
85 |
+ return self._instances[instance_name]
|
|
86 |
+ |
|
87 |
+ except KeyError:
|
|
88 |
+ raise InvalidArgumentError("Invalid instance name: {}".format(instance_name))
|
... | ... | @@ -87,10 +87,10 @@ class Scheduler: |
87 | 87 |
def job_complete(self, name, result):
|
88 | 88 |
job = self.jobs[name]
|
89 | 89 |
job.result = result
|
90 |
- job.update_execute_stage(ExecuteStage.COMPLETED)
|
|
91 | 90 |
self.jobs[name] = job
|
92 | 91 |
if not job.do_not_cache and self._action_cache is not None:
|
93 | 92 |
self._action_cache.update_action_result(job.action_digest, result)
|
93 |
+ job.update_execute_stage(ExecuteStage.COMPLETED)
|
|
94 | 94 |
|
95 | 95 |
def get_operations(self):
|
96 | 96 |
response = operations_pb2.ListOperationsResponse()
|
... | ... | @@ -114,6 +114,7 @@ setup( |
114 | 114 |
'protobuf',
|
115 | 115 |
'grpcio',
|
116 | 116 |
'Click',
|
117 |
+ 'pyaml',
|
|
117 | 118 |
'boto3 < 1.8.0',
|
118 | 119 |
'botocore < 1.11.0',
|
119 | 120 |
'xdg',
|