finn pushed to branch finn/87-multi-channel at BuildGrid / buildgrid
Commits:
-
7647db3c
by finnball at 2018-09-18T15:23:04Z
-
b4f074fd
by finnball at 2018-09-18T15:23:12Z
-
00da595d
by finnball at 2018-09-18T15:23:12Z
-
e9b92ccc
by finnball at 2018-09-18T15:23:12Z
-
076dbe8b
by finnball at 2018-09-18T15:23:12Z
24 changed files:
- buildgrid/_app/commands/cmd_server.py
- buildgrid/_app/settings/cas.yml
- buildgrid/_app/settings/default.yml
- buildgrid/_app/settings/remote-storage.yml
- buildgrid/server/actioncache/service.py
- buildgrid/server/actioncache/storage.py
- buildgrid/server/bots/instance.py
- buildgrid/server/bots/service.py
- buildgrid/server/cas/instance.py
- buildgrid/server/cas/service.py
- buildgrid/server/controller.py
- buildgrid/server/execution/instance.py
- buildgrid/server/execution/service.py
- buildgrid/server/operations/instance.py
- buildgrid/server/operations/service.py
- buildgrid/server/referencestorage/service.py
- buildgrid/server/referencestorage/storage.py
- tests/cas/test_services.py
- tests/cas/test_storage.py
- tests/integration/action_cache_service.py
- tests/integration/bots_service.py
- tests/integration/execution_service.py
- tests/integration/operations_service.py
- tests/integration/reference_storage_service.py
Changes:
... | ... | @@ -26,14 +26,10 @@ import sys |
26 | 26 |
|
27 | 27 |
import click
|
28 | 28 |
|
29 |
-from buildgrid.server.controller import ExecutionController
|
|
30 |
-from buildgrid.server.actioncache.storage import ActionCache
|
|
31 |
-from buildgrid.server.cas.instance import ByteStreamInstance, ContentAddressableStorageInstance
|
|
32 |
-from buildgrid.server.referencestorage.storage import ReferenceCache
|
|
29 |
+from buildgrid.server.instance import BuildGridServer
|
|
33 | 30 |
|
34 | 31 |
from ..cli import pass_context
|
35 | 32 |
from ..settings import parser
|
36 |
-from ..server import BuildGridServer
|
|
37 | 33 |
|
38 | 34 |
|
39 | 35 |
@click.group(name='server', short_help="Start a local server instance.")
|
... | ... | @@ -51,57 +47,23 @@ def start(context, config): |
51 | 47 |
|
52 | 48 |
try:
|
53 | 49 |
server_settings = settings['server']
|
54 |
- insecure_mode = server_settings['insecure-mode']
|
|
55 |
- |
|
56 |
- credentials = None
|
|
57 |
- if not insecure_mode:
|
|
58 |
- credential_settings = server_settings['credentials']
|
|
59 |
- server_key = credential_settings['tls-server-key']
|
|
60 |
- server_cert = credential_settings['tls-server-cert']
|
|
61 |
- client_certs = credential_settings['tls-client-certs']
|
|
62 |
- credentials = context.load_server_credentials(server_key, server_cert, client_certs)
|
|
63 |
- |
|
64 |
- if not credentials:
|
|
65 |
- click.echo("ERROR: no TLS keys were specified and no defaults could be found.\n" +
|
|
66 |
- "Set `insecure-mode: false` in order to deactivate TLS encryption.\n", err=True)
|
|
67 |
- sys.exit(-1)
|
|
68 |
- |
|
69 |
- port = server_settings['port']
|
|
70 |
- instances = settings['instances']
|
|
71 |
- |
|
72 |
- execution_controllers = _instance_maker(instances, ExecutionController)
|
|
73 | 50 |
|
74 |
- execution_instances = {}
|
|
75 |
- bots_interfaces = {}
|
|
76 |
- operations_instances = {}
|
|
51 |
+ server = BuildGridServer()
|
|
52 |
+ for channel in server_settings:
|
|
53 |
+ server.add_port(channel.address, channel.credentials)
|
|
77 | 54 |
|
78 |
- # TODO: map properly in parser
|
|
79 |
- # Issue 82
|
|
80 |
- for k, v in execution_controllers.items():
|
|
81 |
- execution_instances[k] = v.execution_instance
|
|
82 |
- bots_interfaces[k] = v.bots_interface
|
|
83 |
- operations_instances[k] = v.operations_instance
|
|
55 |
+ instances = settings['instances']
|
|
56 |
+ for instance in instances:
|
|
57 |
+ instance_name = instance['name']
|
|
58 |
+ services = instance['services']
|
|
84 | 59 |
|
85 |
- reference_caches = _instance_maker(instances, ReferenceCache)
|
|
86 |
- action_caches = _instance_maker(instances, ActionCache)
|
|
87 |
- cas = _instance_maker(instances, ContentAddressableStorageInstance)
|
|
88 |
- bytestreams = _instance_maker(instances, ByteStreamInstance)
|
|
60 |
+ for service in services:
|
|
61 |
+ service.add_self_to_service(instance_name, server)
|
|
89 | 62 |
|
90 | 63 |
except KeyError as e:
|
91 | 64 |
click.echo("ERROR: Could not parse config: {}.\n".format(str(e)), err=True)
|
92 | 65 |
sys.exit(-1)
|
93 | 66 |
|
94 |
- server = BuildGridServer(port=port,
|
|
95 |
- credentials=credentials,
|
|
96 |
- execution_instances=execution_instances,
|
|
97 |
- bots_interfaces=bots_interfaces,
|
|
98 |
- operations_instances=operations_instances,
|
|
99 |
- reference_storage_instances=reference_caches,
|
|
100 |
- action_cache_instances=action_caches,
|
|
101 |
- cas_instances=cas,
|
|
102 |
- bytestream_instances=bytestreams)
|
|
103 |
- |
|
104 |
- context.logger.info("Starting server on port {}".format(port))
|
|
105 | 67 |
loop = asyncio.get_event_loop()
|
106 | 68 |
try:
|
107 | 69 |
server.start()
|
... | ... | @@ -114,17 +76,3 @@ def start(context, config): |
114 | 76 |
context.logger.info("Stopping server")
|
115 | 77 |
server.stop()
|
116 | 78 |
loop.close()
|
117 |
- |
|
118 |
- |
|
119 |
-# Turn away now if you want to keep your eyes
|
|
120 |
-def _instance_maker(instances, service_type):
|
|
121 |
- # TODO get this mapped in parser
|
|
122 |
- made = {}
|
|
123 |
- |
|
124 |
- for instance in instances:
|
|
125 |
- services = instance['services']
|
|
126 |
- instance_name = instance['name']
|
|
127 |
- for service in services:
|
|
128 |
- if isinstance(service, service_type):
|
|
129 |
- made[instance_name] = service
|
|
130 |
- return made
|
1 | 1 |
server:
|
2 |
- port: 50052
|
|
3 |
- insecure-mode: true
|
|
4 |
- credentials:
|
|
5 |
- tls-server-key: null
|
|
6 |
- tls-server-cert: null
|
|
7 |
- tls-client-certs: null
|
|
2 |
+ - !channel
|
|
3 |
+ port: 50051
|
|
4 |
+ insecure_mode: true
|
|
5 |
+ credentials:
|
|
6 |
+ tls-server-key: null
|
|
7 |
+ tls-server-cert: null
|
|
8 |
+ tls-client-certs: null
|
|
8 | 9 |
|
9 | 10 |
description: |
|
10 | 11 |
Just a CAS with some reference storage.
|
1 | 1 |
server:
|
2 |
- port: 50051
|
|
3 |
- insecure-mode: true
|
|
4 |
- credentials:
|
|
5 |
- tls-server-key: null
|
|
6 |
- tls-server-cert: null
|
|
7 |
- tls-client-certs: null
|
|
2 |
+ - !channel
|
|
3 |
+ port: 50051
|
|
4 |
+ insecure_mode: true
|
|
5 |
+ credentials:
|
|
6 |
+ tls-server-key: null
|
|
7 |
+ tls-server-cert: null
|
|
8 |
+ tls-client-certs: null
|
|
8 | 9 |
|
9 | 10 |
description: |
|
10 | 11 |
A single default instance
|
1 | 1 |
server:
|
2 |
- port: 50051
|
|
3 |
- insecure-mode: true
|
|
4 |
- credentials:
|
|
5 |
- tls-server-key: null
|
|
6 |
- tls-server-cert: null
|
|
7 |
- tls-client-certs: null
|
|
8 |
- |
|
2 |
+ - !channel
|
|
3 |
+ port: 50051
|
|
4 |
+ insecure-mode: true
|
|
5 |
+ credentials:
|
|
6 |
+ tls-server-key: null
|
|
7 |
+ tls-server-cert: null
|
|
8 |
+ tls-client-certs: null
|
|
9 | 9 |
|
10 | 10 |
description: |
|
11 | 11 |
A single default instance with remote storage.
|
... | ... | @@ -32,13 +32,16 @@ from .._exceptions import InvalidArgumentError, NotFoundError |
32 | 32 |
|
33 | 33 |
class ActionCacheService(remote_execution_pb2_grpc.ActionCacheServicer):
|
34 | 34 |
|
35 |
- def __init__(self, server, instances):
|
|
36 |
- self._instances = instances
|
|
37 |
- |
|
35 |
+ def __init__(self, server):
|
|
38 | 36 |
self.logger = logging.getLogger(__name__)
|
39 | 37 |
|
38 |
+ self._instances = dict()
|
|
39 |
+ |
|
40 | 40 |
remote_execution_pb2_grpc.add_ActionCacheServicer_to_server(self, server)
|
41 | 41 |
|
42 |
+ def add_instance(self, name, instance):
|
|
43 |
+ self._instances[name] = instance
|
|
44 |
+ |
|
42 | 45 |
def GetActionResult(self, request, context):
|
43 | 46 |
try:
|
44 | 47 |
instance = self._get_instance(request.instance_name)
|
... | ... | @@ -26,6 +26,9 @@ from ..referencestorage.storage import ReferenceCache |
26 | 26 |
|
27 | 27 |
class ActionCache(ReferenceCache):
|
28 | 28 |
|
29 |
+ def add_self_to_service(self, instance_name, server):
|
|
30 |
+ server.add_action_cache_instance(self, instance_name)
|
|
31 |
+ |
|
29 | 32 |
def get_action_result(self, action_digest):
|
30 | 33 |
key = self._get_key(action_digest)
|
31 | 34 |
return self.get_action_reference(key)
|
... | ... | @@ -36,6 +36,9 @@ class BotsInterface: |
36 | 36 |
self._bot_sessions = {}
|
37 | 37 |
self._scheduler = scheduler
|
38 | 38 |
|
39 |
+ def add_self_to_service(self, instance_name, server):
|
|
40 |
+ server.add_bots_interface(self, instance_name)
|
|
41 |
+ |
|
39 | 42 |
def create_bot_session(self, parent, bot_session):
|
40 | 43 |
""" Creates a new bot session. Server should assign a unique
|
41 | 44 |
name to the session. If a bot with the same bot id tries to
|
... | ... | @@ -33,12 +33,16 @@ from .._exceptions import InvalidArgumentError, OutofSyncError |
33 | 33 |
|
34 | 34 |
class BotsService(bots_pb2_grpc.BotsServicer):
|
35 | 35 |
|
36 |
- def __init__(self, server, instances):
|
|
37 |
- self._instances = instances
|
|
36 |
+ def __init__(self, server):
|
|
38 | 37 |
self.logger = logging.getLogger(__name__)
|
39 | 38 |
|
39 |
+ self._instances = dict()
|
|
40 |
+ |
|
40 | 41 |
bots_pb2_grpc.add_BotsServicer_to_server(self, server)
|
41 | 42 |
|
43 |
+ def add_instance(self, name, instance):
|
|
44 |
+ self._instances[name] = instance
|
|
45 |
+ |
|
42 | 46 |
def CreateBotSession(self, request, context):
|
43 | 47 |
try:
|
44 | 48 |
parent = request.parent
|
... | ... | @@ -31,6 +31,9 @@ class ContentAddressableStorageInstance: |
31 | 31 |
def __init__(self, storage):
|
32 | 32 |
self._storage = storage
|
33 | 33 |
|
34 |
+ def add_self_to_service(self, instance_name, server):
|
|
35 |
+ server.add_cas_instance(self, instance_name)
|
|
36 |
+ |
|
34 | 37 |
def find_missing_blobs(self, blob_digests):
|
35 | 38 |
storage = self._storage
|
36 | 39 |
return re_pb2.FindMissingBlobsResponse(
|
... | ... | @@ -60,6 +63,9 @@ class ByteStreamInstance: |
60 | 63 |
def __init__(self, storage):
|
61 | 64 |
self._storage = storage
|
62 | 65 |
|
66 |
+ def add_self_to_service(self, instance_name, server):
|
|
67 |
+ server.add_bytestream_instance(self, instance_name)
|
|
68 |
+ |
|
63 | 69 |
def read(self, path, read_offset, read_limit):
|
64 | 70 |
storage = self._storage
|
65 | 71 |
|
... | ... | @@ -35,12 +35,16 @@ from .._exceptions import InvalidArgumentError, NotFoundError, OutOfRangeError |
35 | 35 |
|
36 | 36 |
class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressableStorageServicer):
|
37 | 37 |
|
38 |
- def __init__(self, server, instances):
|
|
38 |
+ def __init__(self, server):
|
|
39 | 39 |
self.logger = logging.getLogger(__name__)
|
40 |
- self._instances = instances
|
|
40 |
+ |
|
41 |
+ self._instances = dict()
|
|
41 | 42 |
|
42 | 43 |
remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server(self, server)
|
43 | 44 |
|
45 |
+ def add_instance(self, name, instance):
|
|
46 |
+ self._instances[name] = instance
|
|
47 |
+ |
|
44 | 48 |
def FindMissingBlobs(self, request, context):
|
45 | 49 |
try:
|
46 | 50 |
instance = self._get_instance(request.instance_name)
|
... | ... | @@ -75,12 +79,16 @@ class ContentAddressableStorageService(remote_execution_pb2_grpc.ContentAddressa |
75 | 79 |
|
76 | 80 |
class ByteStreamService(bytestream_pb2_grpc.ByteStreamServicer):
|
77 | 81 |
|
78 |
- def __init__(self, server, instances):
|
|
82 |
+ def __init__(self, server):
|
|
79 | 83 |
self.logger = logging.getLogger(__name__)
|
80 |
- self._instances = instances
|
|
84 |
+ |
|
85 |
+ self._instances = dict()
|
|
81 | 86 |
|
82 | 87 |
bytestream_pb2_grpc.add_ByteStreamServicer_to_server(self, server)
|
83 | 88 |
|
89 |
+ def add_instance(self, name, instance):
|
|
90 |
+ self._instances[name] = instance
|
|
91 |
+ |
|
84 | 92 |
def Read(self, request, context):
|
85 | 93 |
try:
|
86 | 94 |
path = request.resource_name.split("/")
|
... | ... | @@ -45,6 +45,11 @@ class ExecutionController: |
45 | 45 |
self._bots_interface = BotsInterface(scheduler)
|
46 | 46 |
self._operations_instance = OperationsInstance(scheduler)
|
47 | 47 |
|
48 |
+ def add_self_to_service(self, instance_name, server):
|
|
49 |
+ server.add_execution_instance(self._execution_instance, instance_name)
|
|
50 |
+ server.add_bots_interface(self._bots_interface, instance_name)
|
|
51 |
+ server.add_operations_instance(self._operations_instance, instance_name)
|
|
52 |
+ |
|
48 | 53 |
def stream_operation_updates(self, message_queue, operation_name):
|
49 | 54 |
operation = message_queue.get()
|
50 | 55 |
while not operation.done:
|
... | ... | @@ -34,6 +34,9 @@ class ExecutionInstance: |
34 | 34 |
self._storage = storage
|
35 | 35 |
self._scheduler = scheduler
|
36 | 36 |
|
37 |
+ def add_self_to_service(self, instance_name, server):
|
|
38 |
+ server.add_execution_instance(self, instance_name)
|
|
39 |
+ |
|
37 | 40 |
def execute(self, action_digest, skip_cache_lookup, message_queue=None):
|
38 | 41 |
""" Sends a job for execution.
|
39 | 42 |
Queues an action and creates an Operation instance to be associated with
|
... | ... | @@ -35,12 +35,14 @@ from .._exceptions import InvalidArgumentError |
35 | 35 |
|
36 | 36 |
class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
|
37 | 37 |
|
38 |
- def __init__(self, server, instances):
|
|
38 |
+ def __init__(self, server):
|
|
39 | 39 |
self.logger = logging.getLogger(__name__)
|
40 |
- self._instances = instances
|
|
41 |
- |
|
40 |
+ self._instances = dict()
|
|
42 | 41 |
remote_execution_pb2_grpc.add_ExecutionServicer_to_server(self, server)
|
43 | 42 |
|
43 |
+ def add_instance(self, name, instance):
|
|
44 |
+ self._instances[name] = instance
|
|
45 |
+ |
|
44 | 46 |
def Execute(self, request, context):
|
45 | 47 |
try:
|
46 | 48 |
message_queue = queue.Queue()
|
... | ... | @@ -30,6 +30,9 @@ class OperationsInstance: |
30 | 30 |
self.logger = logging.getLogger(__name__)
|
31 | 31 |
self._scheduler = scheduler
|
32 | 32 |
|
33 |
+ def add_self_to_service(self, instance_name, server):
|
|
34 |
+ server.add_operations_instance(self, instance_name)
|
|
35 |
+ |
|
33 | 36 |
def get_operation(self, name):
|
34 | 37 |
operation = self._scheduler.jobs.get(name)
|
35 | 38 |
|
... | ... | @@ -32,12 +32,16 @@ from .._exceptions import InvalidArgumentError |
32 | 32 |
|
33 | 33 |
class OperationsService(operations_pb2_grpc.OperationsServicer):
|
34 | 34 |
|
35 |
- def __init__(self, server, instances):
|
|
36 |
- self._instances = instances
|
|
35 |
+ def __init__(self, server):
|
|
37 | 36 |
self.logger = logging.getLogger(__name__)
|
38 | 37 |
|
38 |
+ self._instances = dict()
|
|
39 |
+ |
|
39 | 40 |
operations_pb2_grpc.add_OperationsServicer_to_server(self, server)
|
40 | 41 |
|
42 |
+ def add_instance(self, name, instance):
|
|
43 |
+ self._instances[name] = instance
|
|
44 |
+ |
|
41 | 45 |
def GetOperation(self, request, context):
|
42 | 46 |
try:
|
43 | 47 |
name = request.name
|
... | ... | @@ -25,13 +25,16 @@ from .._exceptions import InvalidArgumentError, NotFoundError |
25 | 25 |
|
26 | 26 |
class ReferenceStorageService(buildstream_pb2_grpc.ReferenceStorageServicer):
|
27 | 27 |
|
28 |
- def __init__(self, server, instances):
|
|
28 |
+ def __init__(self, server):
|
|
29 | 29 |
self.logger = logging.getLogger(__name__)
|
30 | 30 |
|
31 |
- self._instances = instances
|
|
31 |
+ self._instances = dict()
|
|
32 | 32 |
|
33 | 33 |
buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server(self, server)
|
34 | 34 |
|
35 |
+ def add_instance(self, name, instance):
|
|
36 |
+ self._instances[name] = instance
|
|
37 |
+ |
|
35 | 38 |
def GetReference(self, request, context):
|
36 | 39 |
try:
|
37 | 40 |
instance = self._get_instance(request.instance_name)
|
... | ... | @@ -44,6 +44,9 @@ class ReferenceCache: |
44 | 44 |
self._max_cached_refs = max_cached_refs
|
45 | 45 |
self._digest_map = collections.OrderedDict()
|
46 | 46 |
|
47 |
+ def add_self_to_service(self, instance_name, server):
|
|
48 |
+ server.add_reference_storage_instance(self, instance_name)
|
|
49 |
+ |
|
47 | 50 |
@property
|
48 | 51 |
def allow_updates(self):
|
49 | 52 |
return self._allow_updates
|
... | ... | @@ -80,7 +80,8 @@ def test_bytestream_read(mocked, data_to_read, instance): |
80 | 80 |
storage = SimpleStorage([b"abc", b"defg", data_to_read])
|
81 | 81 |
|
82 | 82 |
bs_instance = ByteStreamInstance(storage)
|
83 |
- servicer = ByteStreamService(server, {instance: bs_instance})
|
|
83 |
+ servicer = ByteStreamService(server)
|
|
84 |
+ servicer.add_instance(instance, bs_instance)
|
|
84 | 85 |
|
85 | 86 |
request = bytestream_pb2.ReadRequest()
|
86 | 87 |
if instance != "":
|
... | ... | @@ -99,8 +100,10 @@ def test_bytestream_read_many(mocked, instance): |
99 | 100 |
data_to_read = b"testing" * 10000
|
100 | 101 |
|
101 | 102 |
storage = SimpleStorage([b"abc", b"defg", data_to_read])
|
103 |
+ |
|
102 | 104 |
bs_instance = ByteStreamInstance(storage)
|
103 |
- servicer = ByteStreamService(server, {instance: bs_instance})
|
|
105 |
+ servicer = ByteStreamService(server)
|
|
106 |
+ servicer.add_instance(instance, bs_instance)
|
|
104 | 107 |
|
105 | 108 |
request = bytestream_pb2.ReadRequest()
|
106 | 109 |
if instance != "":
|
... | ... | @@ -118,8 +121,10 @@ def test_bytestream_read_many(mocked, instance): |
118 | 121 |
@mock.patch.object(service, 'bytestream_pb2_grpc', autospec=True)
|
119 | 122 |
def test_bytestream_write(mocked, instance, extra_data):
|
120 | 123 |
storage = SimpleStorage()
|
124 |
+ |
|
121 | 125 |
bs_instance = ByteStreamInstance(storage)
|
122 |
- servicer = ByteStreamService(server, {instance: bs_instance})
|
|
126 |
+ servicer = ByteStreamService(server)
|
|
127 |
+ servicer.add_instance(instance, bs_instance)
|
|
123 | 128 |
|
124 | 129 |
resource_name = ""
|
125 | 130 |
if instance != "":
|
... | ... | @@ -142,8 +147,10 @@ def test_bytestream_write(mocked, instance, extra_data): |
142 | 147 |
@mock.patch.object(service, 'bytestream_pb2_grpc', autospec=True)
|
143 | 148 |
def test_bytestream_write_rejects_wrong_hash(mocked):
|
144 | 149 |
storage = SimpleStorage()
|
150 |
+ |
|
145 | 151 |
bs_instance = ByteStreamInstance(storage)
|
146 |
- servicer = ByteStreamService(server, {"": bs_instance})
|
|
152 |
+ servicer = ByteStreamService(server)
|
|
153 |
+ servicer.add_instance("", bs_instance)
|
|
147 | 154 |
|
148 | 155 |
data = b'some data'
|
149 | 156 |
wrong_hash = HASH(b'incorrect').hexdigest()
|
... | ... | @@ -163,7 +170,9 @@ def test_bytestream_write_rejects_wrong_hash(mocked): |
163 | 170 |
def test_cas_find_missing_blobs(mocked, instance):
|
164 | 171 |
storage = SimpleStorage([b'abc', b'def'])
|
165 | 172 |
cas_instance = ContentAddressableStorageInstance(storage)
|
166 |
- servicer = ContentAddressableStorageService(server, {instance: cas_instance})
|
|
173 |
+ servicer = ContentAddressableStorageService(server)
|
|
174 |
+ servicer.add_instance(instance, cas_instance)
|
|
175 |
+ |
|
167 | 176 |
digests = [
|
168 | 177 |
re_pb2.Digest(hash=HASH(b'def').hexdigest(), size_bytes=3),
|
169 | 178 |
re_pb2.Digest(hash=HASH(b'ghij').hexdigest(), size_bytes=4)
|
... | ... | @@ -178,8 +187,10 @@ def test_cas_find_missing_blobs(mocked, instance): |
178 | 187 |
@mock.patch.object(service, 'remote_execution_pb2_grpc', autospec=True)
|
179 | 188 |
def test_cas_batch_update_blobs(mocked, instance):
|
180 | 189 |
storage = SimpleStorage()
|
190 |
+ |
|
181 | 191 |
cas_instance = ContentAddressableStorageInstance(storage)
|
182 |
- servicer = ContentAddressableStorageService(server, {instance: cas_instance})
|
|
192 |
+ servicer = ContentAddressableStorageService(server)
|
|
193 |
+ servicer.add_instance(instance, cas_instance)
|
|
183 | 194 |
|
184 | 195 |
update_requests = [
|
185 | 196 |
re_pb2.BatchUpdateBlobsRequest.Request(
|
... | ... | @@ -69,9 +69,13 @@ class MockStubServer: |
69 | 69 |
instances = {"": MockCASStorage(), "dna": MockCASStorage()}
|
70 | 70 |
self._requests = []
|
71 | 71 |
with mock.patch.object(service, 'bytestream_pb2_grpc'):
|
72 |
- self._bs_service = service.ByteStreamService(server, instances)
|
|
72 |
+ self._bs_service = service.ByteStreamService(server)
|
|
73 |
+ for k, v in instances.items():
|
|
74 |
+ self._bs_service.add_instance(k, v)
|
|
73 | 75 |
with mock.patch.object(service, 'remote_execution_pb2_grpc'):
|
74 |
- self._cas_service = service.ContentAddressableStorageService(server, instances)
|
|
76 |
+ self._cas_service = service.ContentAddressableStorageService(server)
|
|
77 |
+ for k, v in instances.items():
|
|
78 |
+ self._cas_service.add_instance(k, v)
|
|
75 | 79 |
|
76 | 80 |
def Read(self, request):
|
77 | 81 |
yield from self._bs_service.Read(request, context)
|
... | ... | @@ -127,7 +131,7 @@ def any_storage(request): |
127 | 131 |
with mock.patch.object(remote, 'bytestream_pb2_grpc'):
|
128 | 132 |
with mock.patch.object(remote, 'remote_execution_pb2_grpc'):
|
129 | 133 |
mock_server = MockStubServer()
|
130 |
- storage = remote.RemoteStorage(instance, "")
|
|
134 |
+ storage = remote.RemoteStorage(None, "")
|
|
131 | 135 |
storage._stub_bs = mock_server
|
132 | 136 |
storage._stub_cas = mock_server
|
133 | 137 |
yield storage
|
... | ... | @@ -52,9 +52,11 @@ def cache_instances(cas): |
52 | 52 |
|
53 | 53 |
def test_simple_action_result(cache_instances, context):
|
54 | 54 |
with mock.patch.object(service, 'remote_execution_pb2_grpc'):
|
55 |
- ac_service = ActionCacheService(server, cache_instances)
|
|
55 |
+ ac_service = ActionCacheService(server)
|
|
56 |
+ |
|
57 |
+ for k, v in cache_instances.items():
|
|
58 |
+ ac_service.add_instance(k, v)
|
|
56 | 59 |
|
57 |
- print(cache_instances)
|
|
58 | 60 |
action_digest = remote_execution_pb2.Digest(hash='sample', size_bytes=4)
|
59 | 61 |
|
60 | 62 |
# Check that before adding the ActionResult, attempting to fetch it fails
|
... | ... | @@ -78,7 +80,8 @@ def test_simple_action_result(cache_instances, context): |
78 | 80 |
def test_disabled_update_action_result(context):
|
79 | 81 |
disabled_push = ActionCache(cas, 50, False)
|
80 | 82 |
with mock.patch.object(service, 'remote_execution_pb2_grpc'):
|
81 |
- ac_service = ActionCacheService(server, {"": disabled_push})
|
|
83 |
+ ac_service = ActionCacheService(server)
|
|
84 |
+ ac_service.add_instance("", disabled_push)
|
|
82 | 85 |
|
83 | 86 |
request = remote_execution_pb2.UpdateActionResultRequest(instance_name='')
|
84 | 87 |
ac_service.UpdateActionResult(request, context)
|
... | ... | @@ -59,7 +59,10 @@ def controller(): |
59 | 59 |
def instance(controller):
|
60 | 60 |
instances = {"": controller.bots_interface}
|
61 | 61 |
with mock.patch.object(service, 'bots_pb2_grpc'):
|
62 |
- yield BotsService(server, instances)
|
|
62 |
+ bots_service = BotsService(server)
|
|
63 |
+ for k, v in instances.items():
|
|
64 |
+ bots_service.add_instance(k, v)
|
|
65 |
+ yield bots_service
|
|
63 | 66 |
|
64 | 67 |
|
65 | 68 |
def test_create_bot_session(bot_session, context, instance):
|
... | ... | @@ -58,9 +58,10 @@ def controller(request): |
58 | 58 |
# Instance to test
|
59 | 59 |
@pytest.fixture
|
60 | 60 |
def instance(controller):
|
61 |
- instances = {"": controller.execution_instance}
|
|
62 | 61 |
with mock.patch.object(service, 'remote_execution_pb2_grpc'):
|
63 |
- yield ExecutionService(server, instances)
|
|
62 |
+ execution_service = ExecutionService(server)
|
|
63 |
+ execution_service.add_instance("", controller.execution_instance)
|
|
64 |
+ yield execution_service
|
|
64 | 65 |
|
65 | 66 |
|
66 | 67 |
@pytest.mark.parametrize("skip_cache_lookup", [True, False])
|
... | ... | @@ -63,9 +63,11 @@ def controller(): |
63 | 63 |
# Instance to test
|
64 | 64 |
@pytest.fixture
|
65 | 65 |
def instance(controller):
|
66 |
- instances = {instance_name: controller.operations_instance}
|
|
67 | 66 |
with mock.patch.object(service, 'operations_pb2_grpc'):
|
68 |
- yield OperationsService(server, instances)
|
|
67 |
+ operation_service = OperationsService(server)
|
|
68 |
+ operation_service.add_instance(instance_name, controller.operations_instance)
|
|
69 |
+ |
|
70 |
+ yield operation_service
|
|
69 | 71 |
|
70 | 72 |
|
71 | 73 |
# Queue an execution, get operation corresponding to that request
|
... | ... | @@ -52,9 +52,10 @@ def cache(cas): |
52 | 52 |
|
53 | 53 |
@pytest.fixture
|
54 | 54 |
def instance(cache):
|
55 |
- instances = {instance_name: cache}
|
|
56 | 55 |
with mock.patch.object(service, 'buildstream_pb2_grpc'):
|
57 |
- yield ReferenceStorageService(server, instances)
|
|
56 |
+ ref_service = ReferenceStorageService(server)
|
|
57 |
+ ref_service.add_instance(instance_name, cache)
|
|
58 |
+ yield ref_service
|
|
58 | 59 |
|
59 | 60 |
|
60 | 61 |
def test_simple_result(instance, context):
|
... | ... | @@ -83,7 +84,8 @@ def test_disabled_update_result(context): |
83 | 84 |
keys = ["rick", "roy", "rach"]
|
84 | 85 |
|
85 | 86 |
with mock.patch.object(service, 'buildstream_pb2_grpc'):
|
86 |
- instance = ReferenceStorageService(server, {'': disabled_push})
|
|
87 |
+ instance = ReferenceStorageService(server)
|
|
88 |
+ instance.add_instance(instance_name, disabled_push)
|
|
87 | 89 |
|
88 | 90 |
# Add an ReferenceResult to the cache
|
89 | 91 |
reference_result = remote_execution_pb2.Digest(hash='deckard')
|
... | ... | @@ -101,7 +103,8 @@ def test_disabled_update_result(context): |
101 | 103 |
def test_status(allow_updates, context):
|
102 | 104 |
cache = ReferenceCache(cas, 5, allow_updates)
|
103 | 105 |
with mock.patch.object(service, 'buildstream_pb2_grpc'):
|
104 |
- instance = ReferenceStorageService(server, {'': cache})
|
|
106 |
+ instance = ReferenceStorageService(server)
|
|
107 |
+ instance.add_instance("", cache)
|
|
105 | 108 |
|
106 | 109 |
request = buildstream_pb2.StatusRequest()
|
107 | 110 |
response = instance.Status(request, context)
|