GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][finn/artifact-cas] Added unittests to support Reference Storage.

Title: GitLab

finnball pushed to branch finn/artifact-cas at BuildGrid / buildgrid

Commits:

4 changed files:

Changes:

  • tests/action_cache.py
    ... ... @@ -11,17 +11,16 @@
    11 11 
     # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 12 
     # See the License for the specific language governing permissions and
    13 13 
     # limitations under the License.
    14 
    -#
    15 
    -# Authors:
    16 
    -#        Carter Sande <csande bloomberg net>
    14 
    +
    17 15
    18 16 
     # pylint: disable=redefined-outer-name
    19 17
    20 18 
     import pytest
    21 19
    22 
    -from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    23 
    -from buildgrid.server import action_cache
    24 20 
     from buildgrid.server.cas.storage import lru_memory_cache
    21 
    +from buildgrid.server.execution import action_cache
    22 
    +from buildgrid.server._exceptions import NotFoundError
    23 
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    25 24
    26 25
    27 26 
     @pytest.fixture
    ... ... @@ -35,8 +34,9 @@ def test_null_action_cache(cas):
    35 34 
         action_digest1 = remote_execution_pb2.Digest(hash='alpha', size_bytes=4)
    36 35 
         dummy_result = remote_execution_pb2.ActionResult()
    37 36
    38 
    -    cache.put_action_result(action_digest1, dummy_result)
    39 
    -    assert cache.get_action_result(action_digest1) is None
    37 
    +    cache.update_action_result(action_digest1, dummy_result)
    38 
    +    with pytest.raises(NotFoundError):
    39 
    +        cache.get_action_result(action_digest1)
    40 40
    41 41
    42 42 
     def test_action_cache_expiry(cas):
    ... ... @@ -47,16 +47,18 @@ def test_action_cache_expiry(cas):
    47 47 
         action_digest3 = remote_execution_pb2.Digest(hash='charlie', size_bytes=4)
    48 48 
         dummy_result = remote_execution_pb2.ActionResult()
    49 49
    50 
    -    cache.put_action_result(action_digest1, dummy_result)
    51 
    -    cache.put_action_result(action_digest2, dummy_result)
    50 
    +    cache.update_action_result(action_digest1, dummy_result)
    51 
    +    cache.update_action_result(action_digest2, dummy_result)
    52 52
    53 53 
         # Get digest 1 (making 2 the least recently used)
    54 54 
         assert cache.get_action_result(action_digest1) is not None
    55 55 
         # Add digest 3 (so 2 gets removed from the cache)
    56 
    -    cache.put_action_result(action_digest3, dummy_result)
    56 
    +    cache.update_action_result(action_digest3, dummy_result)
    57 57
    58 58 
         assert cache.get_action_result(action_digest1) is not None
    59 
    -    assert cache.get_action_result(action_digest2) is None
    59 
    +    with pytest.raises(NotFoundError):
    60 
    +        cache.get_action_result(action_digest2)
    61 
    +
    60 62 
         assert cache.get_action_result(action_digest3) is not None
    61 63
    62 64
    ... ... @@ -67,34 +69,35 @@ def test_action_cache_checks_cas(cas):
    67 69 
         action_digest2 = remote_execution_pb2.Digest(hash='bravo', size_bytes=4)
    68 70 
         action_digest3 = remote_execution_pb2.Digest(hash='charlie', size_bytes=4)
    69 71
    70 
    -    # Create a tree that references digests in CAS
    72 
    +    # Create a tree that actions digests in CAS
    71 73 
         sample_digest = cas.put_message(remote_execution_pb2.Command(arguments=["sample"]))
    72 74 
         tree = remote_execution_pb2.Tree()
    73 75 
         tree.root.files.add().digest.CopyFrom(sample_digest)
    74 76 
         tree.children.add().files.add().digest.CopyFrom(sample_digest)
    75 77 
         tree_digest = cas.put_message(tree)
    76 78
    77 
    -    # Add an ActionResult that references real digests to the cache
    79 
    +    # Add an ActionResult that actions real digests to the cache
    78 80 
         action_result1 = remote_execution_pb2.ActionResult()
    79 81 
         action_result1.output_directories.add().tree_digest.CopyFrom(tree_digest)
    80 82 
         action_result1.output_files.add().digest.CopyFrom(sample_digest)
    81 83 
         action_result1.stdout_digest.CopyFrom(sample_digest)
    82 84 
         action_result1.stderr_digest.CopyFrom(sample_digest)
    83 
    -    cache.put_action_result(action_digest1, action_result1)
    85 
    +    cache.update_action_result(action_digest1, action_result1)
    84 86
    85 
    -    # Add ActionResults that reference fake digests to the cache
    87 
    +    # Add ActionResults that action fake digests to the cache
    86 88 
         action_result2 = remote_execution_pb2.ActionResult()
    87 89 
         action_result2.output_directories.add().tree_digest.hash = "nonexistent"
    88 90 
         action_result2.output_directories[0].tree_digest.size_bytes = 8
    89 
    -    cache.put_action_result(action_digest2, action_result2)
    91 
    +    cache.update_action_result(action_digest2, action_result2)
    90 92
    91 93 
         action_result3 = remote_execution_pb2.ActionResult()
    92 94 
         action_result3.stdout_digest.hash = "nonexistent"
    93 95 
         action_result3.stdout_digest.size_bytes = 8
    94 
    -    cache.put_action_result(action_digest3, action_result3)
    96 
    +    cache.update_action_result(action_digest3, action_result3)
    95 97
    96 98 
         # Verify we can get the first ActionResult but not the others
    97 99 
         fetched_result1 = cache.get_action_result(action_digest1)
    98 100 
         assert fetched_result1.output_directories[0].tree_digest.hash == tree_digest.hash
    99 
    -    assert cache.get_action_result(action_digest2) is None
    100 
    -    assert cache.get_action_result(action_digest3) is None
    101 
    +    with pytest.raises(NotFoundError):
    102 
    +        cache.get_action_result(action_digest2)
    103 
    +        cache.get_action_result(action_digest3)

  • tests/cas/test_storage.py
    ... ... @@ -20,9 +20,10 @@
    20 20 
     import tempfile
    21 21
    22 22 
     import boto3
    23 
    -from moto import mock_s3
    24 23 
     import pytest
    25 24
    25 
    +from moto import mock_s3
    26 
    +
    26 27 
     from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import Digest
    27 28 
     from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
    28 29 
     from buildgrid.server.cas.storage.disk import DiskStorage

  • tests/integration/action_cache_service.py
    ... ... @@ -23,10 +23,10 @@ import grpc
    23 23 
     from grpc._server import _Context
    24 24 
     import pytest
    25 25
    26 
    +
    26 27 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    27 
    -from buildgrid.server import action_cache
    28 28 
     from buildgrid.server.cas.storage import lru_memory_cache
    29 
    -from buildgrid.server.execution import action_cache_service
    29 
    +from buildgrid.server.execution import action_cache, action_cache_service
    30 30
    31 31
    32 32 
     # Can mock this
    ... ... @@ -67,7 +67,8 @@ def test_simple_action_result(cache, context):
    67 67
    68 68
    69 69 
     def test_disabled_update_action_result(cache, context):
    70 
    -    service = action_cache_service.ActionCacheService(cache, False)
    70 
    +    disabled_push = action_cache.ActionCache(cas, 50, False)
    71 
    +    service = action_cache_service.ActionCacheService(disabled_push)
    71 72
    72 73 
         request = remote_execution_pb2.UpdateActionResultRequest()
    73 74 
         service.UpdateActionResult(request, context)

  • tests/integration/reference_storage_service.py
    1 
    +# Copyright (C) 2018 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +# pylint: disable=redefined-outer-name
    16 
    +
    17 
    +from unittest import mock
    18 
    +
    19 
    +import grpc
    20 
    +from grpc._server import _Context
    21 
    +
    22 
    +import pytest
    23 
    +
    24 
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    25 
    +from buildgrid._protos.buildstream.v2 import buildstream_pb2
    26 
    +
    27 
    +from buildgrid.server.cas.storage import lru_memory_cache
    28 
    +from buildgrid.server.cas import reference_cache, reference_storage_service
    29 
    +
    30 
    +
    31 
    +# Can mock this
    32 
    +@pytest.fixture
    33 
    +def context():
    34 
    +    yield mock.MagicMock(spec=_Context)
    35 
    +
    36 
    +
    37 
    +@pytest.fixture
    38 
    +def cas():
    39 
    +    yield lru_memory_cache.LRUMemoryCache(1024 * 1024)
    40 
    +
    41 
    +
    42 
    +@pytest.fixture
    43 
    +def cache(cas):
    44 
    +    yield reference_cache.ReferenceCache(cas, 50)
    45 
    +
    46 
    +
    47 
    +def test_simple_result(cache, context):
    48 
    +    keys = ["rick", "roy", "rach"]
    49 
    +    service = reference_storage_service.ReferenceStorageService(cache)
    50 
    +
    51 
    +    # Check that before adding the ReferenceResult, attempting to fetch it fails
    52 
    +    request = buildstream_pb2.GetReferenceRequest(key=keys[0])
    53 
    +    service.GetReference(request, context)
    54 
    +    context.set_code.assert_called_once_with(grpc.StatusCode.NOT_FOUND)
    55 
    +
    56 
    +    # Add an ReferenceResult to the cache
    57 
    +    reference_result = remote_execution_pb2.Digest(hash='deckard')
    58 
    +    request = buildstream_pb2.UpdateReferenceRequest(keys=keys,
    59 
    +                                                     digest=reference_result)
    60 
    +    service.UpdateReference(request, context)
    61 
    +
    62 
    +    # Check that fetching it now works
    63 
    +    for key in keys:
    64 
    +        request = buildstream_pb2.GetReferenceRequest(key=key)
    65 
    +        fetched_result = service.GetReference(request, context)
    66 
    +        assert fetched_result.digest == reference_result
    67 
    +
    68 
    +
    69 
    +def test_disabled_update_result(cache, context):
    70 
    +    disabled_push = reference_cache.ReferenceCache(cas, 50, False)
    71 
    +    keys = ["rick", "roy", "rach"]
    72 
    +    service = reference_storage_service.ReferenceStorageService(disabled_push)
    73 
    +
    74 
    +    # Add an ReferenceResult to the cache
    75 
    +    reference_result = remote_execution_pb2.Digest(hash='deckard')
    76 
    +    request = buildstream_pb2.UpdateReferenceRequest(keys=keys,
    77 
    +                                                     digest=reference_result)
    78 
    +    service.UpdateReference(request, context)
    79 
    +
    80 
    +    request = buildstream_pb2.UpdateReferenceRequest()
    81 
    +    service.UpdateReference(request, context)
    82 
    +
    83 
    +    context.set_code.assert_called_once_with(grpc.StatusCode.UNIMPLEMENTED)
    84 
    +
    85 
    +
    86 
    +@pytest.mark.parametrize("allow_updates", [True, False])
    87 
    +def test_status(allow_updates, context):
    88 
    +    cache = reference_cache.ReferenceCache(cas, 5, allow_updates)
    89 
    +    service = reference_storage_service.ReferenceStorageService(cache)
    90 
    +
    91 
    +    request = buildstream_pb2.StatusRequest()
    92 
    +    response = service.Status(request, context)
    93 
    +
    94 
    +    assert response.allow_updates == allow_updates

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]