finnball pushed to branch finn/artifact-cas at BuildGrid / buildgrid
Commits:
-
a931727f
by finn at 2018-08-21T12:42:08Z
4 changed files:
- tests/action_cache.py
- tests/cas/test_storage.py
- tests/integration/action_cache_service.py
- + tests/integration/reference_storage_service.py
Changes:
... | ... | @@ -11,17 +11,16 @@ |
11 | 11 |
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12 | 12 |
# See the License for the specific language governing permissions and
|
13 | 13 |
# limitations under the License.
|
14 |
-#
|
|
15 |
-# Authors:
|
|
16 |
-# Carter Sande <csande bloomberg net>
|
|
14 |
+ |
|
17 | 15 |
|
18 | 16 |
# pylint: disable=redefined-outer-name
|
19 | 17 |
|
20 | 18 |
import pytest
|
21 | 19 |
|
22 |
-from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
|
|
23 |
-from buildgrid.server import action_cache
|
|
24 | 20 |
from buildgrid.server.cas.storage import lru_memory_cache
|
21 |
+from buildgrid.server.execution import action_cache
|
|
22 |
+from buildgrid.server._exceptions import NotFoundError
|
|
23 |
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
|
|
25 | 24 |
|
26 | 25 |
|
27 | 26 |
@pytest.fixture
|
... | ... | @@ -35,8 +34,9 @@ def test_null_action_cache(cas): |
35 | 34 |
action_digest1 = remote_execution_pb2.Digest(hash='alpha', size_bytes=4)
|
36 | 35 |
dummy_result = remote_execution_pb2.ActionResult()
|
37 | 36 |
|
38 |
- cache.put_action_result(action_digest1, dummy_result)
|
|
39 |
- assert cache.get_action_result(action_digest1) is None
|
|
37 |
+ cache.update_action_result(action_digest1, dummy_result)
|
|
38 |
+ with pytest.raises(NotFoundError):
|
|
39 |
+ cache.get_action_result(action_digest1)
|
|
40 | 40 |
|
41 | 41 |
|
42 | 42 |
def test_action_cache_expiry(cas):
|
... | ... | @@ -47,16 +47,18 @@ def test_action_cache_expiry(cas): |
47 | 47 |
action_digest3 = remote_execution_pb2.Digest(hash='charlie', size_bytes=4)
|
48 | 48 |
dummy_result = remote_execution_pb2.ActionResult()
|
49 | 49 |
|
50 |
- cache.put_action_result(action_digest1, dummy_result)
|
|
51 |
- cache.put_action_result(action_digest2, dummy_result)
|
|
50 |
+ cache.update_action_result(action_digest1, dummy_result)
|
|
51 |
+ cache.update_action_result(action_digest2, dummy_result)
|
|
52 | 52 |
|
53 | 53 |
# Get digest 1 (making 2 the least recently used)
|
54 | 54 |
assert cache.get_action_result(action_digest1) is not None
|
55 | 55 |
# Add digest 3 (so 2 gets removed from the cache)
|
56 |
- cache.put_action_result(action_digest3, dummy_result)
|
|
56 |
+ cache.update_action_result(action_digest3, dummy_result)
|
|
57 | 57 |
|
58 | 58 |
assert cache.get_action_result(action_digest1) is not None
|
59 |
- assert cache.get_action_result(action_digest2) is None
|
|
59 |
+ with pytest.raises(NotFoundError):
|
|
60 |
+ cache.get_action_result(action_digest2)
|
|
61 |
+ |
|
60 | 62 |
assert cache.get_action_result(action_digest3) is not None
|
61 | 63 |
|
62 | 64 |
|
... | ... | @@ -67,34 +69,35 @@ def test_action_cache_checks_cas(cas): |
67 | 69 |
action_digest2 = remote_execution_pb2.Digest(hash='bravo', size_bytes=4)
|
68 | 70 |
action_digest3 = remote_execution_pb2.Digest(hash='charlie', size_bytes=4)
|
69 | 71 |
|
70 |
- # Create a tree that references digests in CAS
|
|
72 |
+ # Create a tree that actions digests in CAS
|
|
71 | 73 |
sample_digest = cas.put_message(remote_execution_pb2.Command(arguments=["sample"]))
|
72 | 74 |
tree = remote_execution_pb2.Tree()
|
73 | 75 |
tree.root.files.add().digest.CopyFrom(sample_digest)
|
74 | 76 |
tree.children.add().files.add().digest.CopyFrom(sample_digest)
|
75 | 77 |
tree_digest = cas.put_message(tree)
|
76 | 78 |
|
77 |
- # Add an ActionResult that references real digests to the cache
|
|
79 |
+ # Add an ActionResult that actions real digests to the cache
|
|
78 | 80 |
action_result1 = remote_execution_pb2.ActionResult()
|
79 | 81 |
action_result1.output_directories.add().tree_digest.CopyFrom(tree_digest)
|
80 | 82 |
action_result1.output_files.add().digest.CopyFrom(sample_digest)
|
81 | 83 |
action_result1.stdout_digest.CopyFrom(sample_digest)
|
82 | 84 |
action_result1.stderr_digest.CopyFrom(sample_digest)
|
83 |
- cache.put_action_result(action_digest1, action_result1)
|
|
85 |
+ cache.update_action_result(action_digest1, action_result1)
|
|
84 | 86 |
|
85 |
- # Add ActionResults that reference fake digests to the cache
|
|
87 |
+ # Add ActionResults that action fake digests to the cache
|
|
86 | 88 |
action_result2 = remote_execution_pb2.ActionResult()
|
87 | 89 |
action_result2.output_directories.add().tree_digest.hash = "nonexistent"
|
88 | 90 |
action_result2.output_directories[0].tree_digest.size_bytes = 8
|
89 |
- cache.put_action_result(action_digest2, action_result2)
|
|
91 |
+ cache.update_action_result(action_digest2, action_result2)
|
|
90 | 92 |
|
91 | 93 |
action_result3 = remote_execution_pb2.ActionResult()
|
92 | 94 |
action_result3.stdout_digest.hash = "nonexistent"
|
93 | 95 |
action_result3.stdout_digest.size_bytes = 8
|
94 |
- cache.put_action_result(action_digest3, action_result3)
|
|
96 |
+ cache.update_action_result(action_digest3, action_result3)
|
|
95 | 97 |
|
96 | 98 |
# Verify we can get the first ActionResult but not the others
|
97 | 99 |
fetched_result1 = cache.get_action_result(action_digest1)
|
98 | 100 |
assert fetched_result1.output_directories[0].tree_digest.hash == tree_digest.hash
|
99 |
- assert cache.get_action_result(action_digest2) is None
|
|
100 |
- assert cache.get_action_result(action_digest3) is None
|
|
101 |
+ with pytest.raises(NotFoundError):
|
|
102 |
+ cache.get_action_result(action_digest2)
|
|
103 |
+ cache.get_action_result(action_digest3)
|
... | ... | @@ -20,9 +20,10 @@ |
20 | 20 |
import tempfile
|
21 | 21 |
|
22 | 22 |
import boto3
|
23 |
-from moto import mock_s3
|
|
24 | 23 |
import pytest
|
25 | 24 |
|
25 |
+from moto import mock_s3
|
|
26 |
+ |
|
26 | 27 |
from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import Digest
|
27 | 28 |
from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
|
28 | 29 |
from buildgrid.server.cas.storage.disk import DiskStorage
|
... | ... | @@ -23,10 +23,10 @@ import grpc |
23 | 23 |
from grpc._server import _Context
|
24 | 24 |
import pytest
|
25 | 25 |
|
26 |
+ |
|
26 | 27 |
from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
|
27 |
-from buildgrid.server import action_cache
|
|
28 | 28 |
from buildgrid.server.cas.storage import lru_memory_cache
|
29 |
-from buildgrid.server.execution import action_cache_service
|
|
29 |
+from buildgrid.server.execution import action_cache, action_cache_service
|
|
30 | 30 |
|
31 | 31 |
|
32 | 32 |
# Can mock this
|
... | ... | @@ -67,7 +67,8 @@ def test_simple_action_result(cache, context): |
67 | 67 |
|
68 | 68 |
|
69 | 69 |
def test_disabled_update_action_result(cache, context):
|
70 |
- service = action_cache_service.ActionCacheService(cache, False)
|
|
70 |
+ disabled_push = action_cache.ActionCache(cas, 50, False)
|
|
71 |
+ service = action_cache_service.ActionCacheService(disabled_push)
|
|
71 | 72 |
|
72 | 73 |
request = remote_execution_pb2.UpdateActionResultRequest()
|
73 | 74 |
service.UpdateActionResult(request, context)
|
1 |
+# Copyright (C) 2018 Bloomberg LP
|
|
2 |
+#
|
|
3 |
+# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4 |
+# you may not use this file except in compliance with the License.
|
|
5 |
+# You may obtain a copy of the License at
|
|
6 |
+#
|
|
7 |
+# <http://www.apache.org/licenses/LICENSE-2.0>
|
|
8 |
+#
|
|
9 |
+# Unless required by applicable law or agreed to in writing, software
|
|
10 |
+# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11 |
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12 |
+# See the License for the specific language governing permissions and
|
|
13 |
+# limitations under the License.
|
|
14 |
+ |
|
15 |
+# pylint: disable=redefined-outer-name
|
|
16 |
+ |
|
17 |
+from unittest import mock
|
|
18 |
+ |
|
19 |
+import grpc
|
|
20 |
+from grpc._server import _Context
|
|
21 |
+ |
|
22 |
+import pytest
|
|
23 |
+ |
|
24 |
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
|
|
25 |
+from buildgrid._protos.buildstream.v2 import buildstream_pb2
|
|
26 |
+ |
|
27 |
+from buildgrid.server.cas.storage import lru_memory_cache
|
|
28 |
+from buildgrid.server.cas import reference_cache, reference_storage_service
|
|
29 |
+ |
|
30 |
+ |
|
31 |
+# Can mock this
|
|
32 |
+@pytest.fixture
|
|
33 |
+def context():
|
|
34 |
+ yield mock.MagicMock(spec=_Context)
|
|
35 |
+ |
|
36 |
+ |
|
37 |
+@pytest.fixture
|
|
38 |
+def cas():
|
|
39 |
+ yield lru_memory_cache.LRUMemoryCache(1024 * 1024)
|
|
40 |
+ |
|
41 |
+ |
|
42 |
+@pytest.fixture
|
|
43 |
+def cache(cas):
|
|
44 |
+ yield reference_cache.ReferenceCache(cas, 50)
|
|
45 |
+ |
|
46 |
+ |
|
47 |
+def test_simple_result(cache, context):
|
|
48 |
+ keys = ["rick", "roy", "rach"]
|
|
49 |
+ service = reference_storage_service.ReferenceStorageService(cache)
|
|
50 |
+ |
|
51 |
+ # Check that before adding the ReferenceResult, attempting to fetch it fails
|
|
52 |
+ request = buildstream_pb2.GetReferenceRequest(key=keys[0])
|
|
53 |
+ service.GetReference(request, context)
|
|
54 |
+ context.set_code.assert_called_once_with(grpc.StatusCode.NOT_FOUND)
|
|
55 |
+ |
|
56 |
+ # Add an ReferenceResult to the cache
|
|
57 |
+ reference_result = remote_execution_pb2.Digest(hash='deckard')
|
|
58 |
+ request = buildstream_pb2.UpdateReferenceRequest(keys=keys,
|
|
59 |
+ digest=reference_result)
|
|
60 |
+ service.UpdateReference(request, context)
|
|
61 |
+ |
|
62 |
+ # Check that fetching it now works
|
|
63 |
+ for key in keys:
|
|
64 |
+ request = buildstream_pb2.GetReferenceRequest(key=key)
|
|
65 |
+ fetched_result = service.GetReference(request, context)
|
|
66 |
+ assert fetched_result.digest == reference_result
|
|
67 |
+ |
|
68 |
+ |
|
69 |
+def test_disabled_update_result(cache, context):
|
|
70 |
+ disabled_push = reference_cache.ReferenceCache(cas, 50, False)
|
|
71 |
+ keys = ["rick", "roy", "rach"]
|
|
72 |
+ service = reference_storage_service.ReferenceStorageService(disabled_push)
|
|
73 |
+ |
|
74 |
+ # Add an ReferenceResult to the cache
|
|
75 |
+ reference_result = remote_execution_pb2.Digest(hash='deckard')
|
|
76 |
+ request = buildstream_pb2.UpdateReferenceRequest(keys=keys,
|
|
77 |
+ digest=reference_result)
|
|
78 |
+ service.UpdateReference(request, context)
|
|
79 |
+ |
|
80 |
+ request = buildstream_pb2.UpdateReferenceRequest()
|
|
81 |
+ service.UpdateReference(request, context)
|
|
82 |
+ |
|
83 |
+ context.set_code.assert_called_once_with(grpc.StatusCode.UNIMPLEMENTED)
|
|
84 |
+ |
|
85 |
+ |
|
86 |
+@pytest.mark.parametrize("allow_updates", [True, False])
|
|
87 |
+def test_status(allow_updates, context):
|
|
88 |
+ cache = reference_cache.ReferenceCache(cas, 5, allow_updates)
|
|
89 |
+ service = reference_storage_service.ReferenceStorageService(cache)
|
|
90 |
+ |
|
91 |
+ request = buildstream_pb2.StatusRequest()
|
|
92 |
+ response = service.Status(request, context)
|
|
93 |
+ |
|
94 |
+ assert response.allow_updates == allow_updates
|