GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][finn/separate-services] 2 commits: Rearranged files into more sensible folders.

Title: GitLab

finn pushed to branch finn/separate-services at BuildGrid / buildgrid

Commits:

26 changed files:

Changes:

  • buildgrid/_app/commands/cmd_server.py
    ... ... @@ -30,7 +30,7 @@ from buildgrid.server.cas.storage.disk import DiskStorage
    30 30 
     from buildgrid.server.cas.storage.lru_memory_cache import LRUMemoryCache
    31 31 
     from buildgrid.server.cas.storage.s3 import S3Storage
    32 32 
     from buildgrid.server.cas.storage.with_cache import WithCacheStorage
    33 
    -from buildgrid.server.execution.action_cache import ActionCache
    33 
    +from buildgrid.server.actioncache.storage import ActionCache
    34 34
    35 35 
     from ..cli import pass_context
    36 36

  • buildgrid/_exceptions.py
    ... ... @@ -39,9 +39,7 @@ class BgdError(Exception):
    39 39
    40 40 
     class ErrorDomain(Enum):
    41 41 
         SERVER = 1
    42 
    -    EXECUTION = 2
    43 
    -    WORKER = 3
    44 
    -    BOT = 4
    42 
    +    BOT = 2
    45 43
    46 44
    47 45 
     class ServerError(BgdError):
    ... ... @@ -49,16 +47,6 @@ class ServerError(BgdError):
    49 47 
             super().__init__(message, detail=detail, domain=ErrorDomain.SERVER, reason=reason)
    50 48
    51 49
    52 
    -class ExecutionError(BgdError):
    53 
    -    def __init__(self, message, detail=None, reason=None):
    54 
    -        super().__init__(message, detail=detail, domain=ErrorDomain.EXECUTION, reason=reason)
    55 
    -
    56 
    -
    57 
    -class WorkerError(BgdError):
    58 
    -    def __init__(self, message, detail=None, reason=None):
    59 
    -        super().__init__(message, detail=detail, domain=ErrorDomain.WORKER, reason=reason)
    60 
    -
    61 
    -
    62 50 
     class BotError(BgdError):
    63 51 
         def __init__(self, message, detail=None, reason=None):
    64 52 
             super().__init__(message, detail=detail, domain=ErrorDomain.BOT, reason=reason)<
 /pre>

  • buildgrid/server/worker/__init__.py → buildgrid/server/actioncache/__init__.py

  • buildgrid/server/execution/action_cache_service.py → buildgrid/server/actioncache/service.py

  • buildgrid/server/execution/action_cache.py → buildgrid/server/actioncache/storage.py
    ... ... @@ -21,7 +21,7 @@ Implements an in-memory action Cache
    21 21 
     """
    22 22
    23 23
    24 
    -from ..cas.reference_cache import ReferenceCache
    24 
    +from ..referencestorage.storage import ReferenceCache
    25 25
    26 26
    27 27 
     class ActionCache(ReferenceCache):

  • buildgrid/server/bots/__init__.py

  • buildgrid/server/worker/bots_interface.py → buildgrid/server/bots/instance.py

  • buildgrid/server/worker/bots_service.py → buildgrid/server/bots/service.py

  • buildgrid/server/buildgrid_server.py
    ... ... @@ -29,13 +29,12 @@ from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_p
    29 29 
     from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2_grpc
    30 30 
     from buildgrid._protos.google.longrunning import operations_pb2_grpc
    31 31
    32 
    -from .buildgrid_instance import BuildGridInstance
    33 
    -from .cas.bytestream_service import ByteStreamService
    34 
    -from .cas.content_addressable_storage_service import ContentAddressableStorageService
    35 
    -from .execution.action_cache_service import ActionCacheService
    36 
    -from .execution.execution_service import ExecutionService
    37 
    -from .execution.operations_service import OperationsService
    38 
    -from .worker.bots_service import BotsService
    32 
    +from .instance import BuildGridInstance
    33 
    +from .cas.service import ByteStreamService, ContentAddressableStorageService
    34 
    +from .actioncache.service import ActionCacheService
    35 
    +from .execution.service import ExecutionService
    36 
    +from .operations.service import OperationsService
    37 
    +from .bots.service import BotsService
    39 38
    40 39
    41 40 
     class BuildGridServer:

  • buildgrid/server/cas/content_addressable_storage_service.py deleted
    1 
    -# Copyright (C) 2018 Bloomberg LP
    2 
    -#
    3 
    -# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    -# you may not use this file except in compliance with the License.
    5 
    -# You may obtain a copy of the License at
    6 
    -#
    7 
    -#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    -#
    9 
    -# Unless required by applicable law or agreed to in writing, software
    10 
    -# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    -# See the License for the specific language governing permissions and
    13 
    -# limitations under the License.
    14 
    -
    15 
    -
    16 
    -"""
    17 
    -ContentAddressableStorageService
    18 
    -==================
    19 
    -
    20 
    -Implements the Content Addressable Storage API, which provides methods
    21 
    -to check for missing CAS blobs and update them in bulk.
    22 
    -"""
    23 
    -
    24 
    -from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2
    25 
    -from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc as re_pb2_grpc
    26 
    -
    27 
    -
    28 
    -class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServicer):
    29 
    -
    30 
    -    def __init__(self, storage):
    31 
    -        self._storage = storage
    32 
    -
    33 
    -    def FindMissingBlobs(self, request, context):
    34 
    -        # Only one instance for now.
    35 
    -        storage = self._storage
    36 
    -        return re_pb2.FindMissingBlobsResponse(
    37 
    -            missing_blob_digests=storage.missing_blobs(request.blob_digests))
    38 
    -
    39 
    -    def BatchUpdateBlobs(self, request, context):
    40 
    -        # Only one instance for now.
    41 
    -        storage = self._storage
    42 
    -        requests = []
    43 
    -        for request_proto in request.requests:
    44 
    -            requests.append((request_proto.digest, request_proto.data))
    45 
    -        response = re_pb2.BatchUpdateBlobsResponse()
    46 
    -        for (digest, _), status in zip(requests, storage.bulk_update_blobs(requests)):
    47 
    -            response_proto = response.responses.add()
    48 
    -            response_proto.digest.CopyFrom(digest)
    49 
    -            response_proto.status.CopyFrom(status)
    50 
    -        return response

  • buildgrid/server/cas/bytestream_service.py → buildgrid/server/cas/service.py
    ... ... @@ -14,21 +14,47 @@
    14 14
    15 15
    16 16 
     """
    17 
    -ByteStreamService
    17 
    +CAS services
    18 18 
     ==================
    19 19
    20 
    -Implements the ByteStream API, which clients can use to read and write
    21 
    -CAS blobs.
    20 
    +Implements the Content Addressable Storage API and ByteStream API.
    22 21 
     """
    23 22
    23 
    +
    24 24 
     import grpc
    25 25
    26 26 
     from buildgrid._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
    27 27 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2
    28 
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2_grpc as re_pb2_grpc
    28 29
    29 30 
     from ...settings import HASH
    30 31
    31 32
    33 
    +class ContentAddressableStorageService(re_pb2_grpc.ContentAddressableStorageServicer):
    34 
    +
    35 
    +    def __init__(self, storage):
    36 
    +        self._storage = storage
    37 
    +
    38 
    +    def FindMissingBlobs(self, request, context):
    39 
    +        # Only one instance for now.
    40 
    +        storage = self._storage
    41 
    +        return re_pb2.FindMissingBlobsResponse(
    42 
    +            missing_blob_digests=storage.missing_blobs(request.blob_digests))
    43 
    +
    44 
    +    def BatchUpdateBlobs(self, request, context):
    45 
    +        # Only one instance for now.
    46 
    +        storage = self._storage
    47 
    +        requests = []
    48 
    +        for request_proto in request.requests:
    49 
    +            requests.append((request_proto.digest, request_proto.data))
    50 
    +        response = re_pb2.BatchUpdateBlobsResponse()
    51 
    +        for (digest, _), status in zip(requests, storage.bulk_update_blobs(requests)):
    52 
    +            response_proto = response.responses.add()
    53 
    +            response_proto.digest.CopyFrom(digest)
    54 
    +            response_proto.status.CopyFrom(status)
    55 
    +        return response
    56 
    +
    57 
    +
    32 58 
     class ByteStreamService(bytestream_pb2_grpc.ByteStreamServicer):
    33 59
    34 60 
         BLOCK_SIZE = 1 * 1024 * 1024  # 1 MB block size

  • buildgrid/server/execution/execution_instance.py → buildgrid/server/execution/instance.py

  • buildgrid/server/execution/execution_service.py → buildgrid/server/execution/service.py

  • buildgrid/server/buildgrid_instance.py → buildgrid/server/instance.py
    ... ... @@ -25,9 +25,9 @@ Contains scheduler, execution instance and an interface to the bots.
    25 25
    26 26 
     import logging
    27 27
    28 
    -from .execution.execution_instance import ExecutionInstance
    28 
    +from .execution.instance import ExecutionInstance
    29 29 
     from .scheduler import Scheduler
    30 
    -from .worker.bots_interface import BotsInterface
    30 
    +from .bots.instance import BotsInterface
    31 31
    32 32
    33 33 
     class BuildGridInstance(ExecutionInstance, BotsInterface):

  • buildgrid/server/operations/__init__.py

  • buildgrid/server/execution/operations_service.py → buildgrid/server/operations/service.py

  • buildgrid/server/referencestorage/__init__.py

  • buildgrid/server/cas/reference_storage_service.py → buildgrid/server/referencestorage/service.py

  • buildgrid/server/cas/reference_cache.py → buildgrid/server/referencestorage/storage.py

  • tests/action_cache.py
    ... ... @@ -17,8 +17,8 @@
    17 17
    18 18 
     import pytest
    19 19
    20 
    +from buildgrid.server.actioncache.storage import ActionCache
    20 21 
     from buildgrid.server.cas.storage import lru_memory_cache
    21 
    -from buildgrid.server.execution import action_cache
    22 22 
     from buildgrid.server._exceptions import NotFoundError
    23 23 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    24 24
    ... ... @@ -28,8 +28,8 @@ def cas():
    28 28 
         return lru_memory_cache.LRUMemoryCache(1024 * 1024)
    29 29
    30 30
    31 
    -def test_null_action_cache(cas):
    32 
    -    cache = action_cache.ActionCache(cas, 0)
    31 
    +def test_null_cas_action_cache(cas):
    32 
    +    cache = ActionCache(cas, 0)
    33 33
    34 34 
         action_digest1 = remote_execution_pb2.Digest(hash='alpha', size_bytes=4)
    35 35 
         dummy_result = remote_execution_pb2.ActionResult()
    ... ... @@ -39,8 +39,8 @@ def test_null_action_cache(cas):
    39 39 
             cache.get_action_result(action_digest1)
    40 40
    41 41
    42 
    -def test_action_cache_expiry(cas):
    43 
    -    cache = action_cache.ActionCache(cas, 2)
    42 
    +def test_expiry(cas):
    43 
    +    cache = ActionCache(cas, 2)
    44 44
    45 45 
         action_digest1 = remote_execution_pb2.Digest(hash='alpha', size_bytes=4)
    46 46 
         action_digest2 = remote_execution_pb2.Digest(hash='bravo', size_bytes=4)
    ... ... @@ -62,8 +62,8 @@ def test_action_cache_expiry(cas):
    62 62 
         assert cache.get_action_result(action_digest3) is not None
    63 63
    64 64
    65 
    -def test_action_cache_checks_cas(cas):
    66 
    -    cache = action_cache.ActionCache(cas, 50)
    65 
    +def test_checks_cas(cas):
    66 
    +    cache = ActionCache(cas, 50)
    67 67
    68 68 
         action_digest1 = remote_execution_pb2.Digest(hash='alpha', size_bytes=4)
    69 69 
         action_digest2 = remote_execution_pb2.Digest(hash='bravo', size_bytes=4)

  • tests/cas/test_services.py
    ... ... @@ -24,8 +24,8 @@ import pytest
    24 24 
     from buildgrid._protos.google.bytestream import bytestream_pb2
    25 25 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 as re_pb2
    26 26 
     from buildgrid.server.cas.storage.storage_abc import StorageABC
    27 
    -from buildgrid.server.cas.bytestream_service import ByteStreamService
    28 
    -from buildgrid.server.cas.content_addressable_storage_service import ContentAddressableStorageService
    27 
    +from buildgrid.server.cas.service import ByteStreamService
    28 
    +from buildgrid.server.cas.service import ContentAddressableStorageService
    29 29 
     from buildgrid.settings import HASH
    30 30
    31 31

  • tests/integration/action_cache_service.py
    ... ... @@ -26,7 +26,8 @@ import pytest
    26 26
    27 27 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    28 28 
     from buildgrid.server.cas.storage import lru_memory_cache
    29 
    -from buildgrid.server.execution import action_cache, action_cache_service
    29 
    +from buildgrid.server.actioncache.storage import ActionCache
    30 
    +from buildgrid.server.actioncache.service import ActionCacheService
    30 31
    31 32
    32 33 
     # Can mock this
    ... ... @@ -42,11 +43,11 @@ def cas():
    42 43
    43 44 
     @pytest.fixture
    44 45 
     def cache(cas):
    45 
    -    yield action_cache.ActionCache(cas, 50)
    46 
    +    yield ActionCache(cas, 50)
    46 47
    47 48
    48 49 
     def test_simple_action_result(cache, context):
    49 
    -    service = action_cache_service.ActionCacheService(cache)
    50 
    +    service = ActionCacheService(cache)
    50 51 
         action_digest = remote_execution_pb2.Digest(hash='sample', size_bytes=4)
    51 52
    52 53 
         # Check that before adding the ActionResult, attempting to fetch it fails
    ... ... @@ -67,8 +68,8 @@ def test_simple_action_result(cache, context):
    67 68
    68 69
    69 70 
     def test_disabled_update_action_result(cache, context):
    70 
    -    disabled_push = action_cache.ActionCache(cas, 50, False)
    71 
    -    service = action_cache_service.ActionCacheService(disabled_push)
    71 
    +    disabled_push = ActionCache(cas, 50, False)
    72 
    +    service = ActionCacheService(disabled_push)
    72 73
    73 74 
         request = remote_execution_pb2.UpdateActionResultRequest()
    74 75 
         service.UpdateActionResult(request, context)

  • tests/integration/bots_service.py
    ... ... @@ -26,9 +26,11 @@ import pytest
    26 26
    27 27 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    28 28 
     from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2
    29 
    -from buildgrid.server import job, buildgrid_instance
    29 
    +from buildgrid.server import job
    30 
    +from buildgrid.server.instance import BuildGridInstance
    30 31 
     from buildgrid.server.job import LeaseState
    31 
    -from buildgrid.server.worker import bots_interface, bots_service
    32 
    +from buildgrid.server.bots.instance import BotsInterface
    33 
    +from buildgrid.server.bots.service import BotsService
    32 34
    33 35
    34 36 
     # GRPC context
    ... ... @@ -53,19 +55,19 @@ def bot_session():
    53 55
    54 56 
     @pytest.fixture
    55 57 
     def buildgrid():
    56 
    -    yield buildgrid_instance.BuildGridInstance()
    58 
    +    yield BuildGridInstance()
    57 59
    58 60
    59 61 
     @pytest.fixture
    60 62 
     def bots(schedule):
    61 
    -    yield bots_interface.BotsInterface(schedule)
    63 
    +    yield BotsInterface(schedule)
    62 64
    63 65
    64 66 
     # Instance to test
    65 67 
     @pytest.fixture
    66 68 
     def instance(buildgrid):
    67 69 
         instances = {"": buildgrid}
    68 
    -    yield bots_service.BotsService(instances)
    70 
    +    yield BotsService(instances)
    69 71
    70 72
    71 73 
     def test_create_bot_session(bot_session, context, instance):

  • tests/integration/execution_service.py
    ... ... @@ -28,9 +28,11 @@ from google.protobuf import any_pb2
    28 28 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    29 29 
     from buildgrid._protos.google.longrunning import operations_pb2
    30 30
    31 
    -from buildgrid.server import job, buildgrid_instance
    31 
    +from buildgrid.server import job
    32 
    +from buildgrid.server.instance import BuildGridInstance
    32 33 
     from buildgrid.server.cas.storage import lru_memory_cache
    33 
    -from buildgrid.server.execution import action_cache, execution_service
    34 
    +from buildgrid.server.actioncache.storage import ActionCache
    35 
    +from buildgrid.server.execution.service import ExecutionService
    34 36
    35 37
    36 38 
     @pytest.fixture
    ... ... @@ -43,18 +45,18 @@ def context():
    43 45 
     def buildgrid(request):
    44 46 
         if request.param == "action-cache":
    45 47 
             storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
    46 
    -        cache = action_cache.ActionCache(storage, 50)
    48 
    +        cache = ActionCache(storage, 50)
    47 49
    48 
    -        return buildgrid_instance.BuildGridInstance(action_cache=cache,
    49 
    -                                                    cas_storage=storage)
    50 
    -    return buildgrid_instance.BuildGridInstance()
    50 
    +        return BuildGridInstance(action_cache=cache,
    51 
    +                                 cas_storage=storage)
    52 
    +    return BuildGridInstance()
    51 53
    52 54
    53 55 
     # Instance to test
    54 56 
     @pytest.fixture
    55 57 
     def instance(buildgrid):
    56 58 
         instances = {"": buildgrid}
    57 
    -    yield execution_service.ExecutionService(instances)
    59 
    +    yield ExecutionService(instances)
    58 60
    59 61
    60 62 
     @pytest.mark.parametrize("skip_cache_lookup", [True, False])

  • tests/integration/operations_service.py
    ... ... @@ -28,10 +28,10 @@ from google.protobuf import any_pb2
    28 28 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    29 29 
     from buildgrid._protos.google.longrunning import operations_pb2
    30 30
    31 
    -from buildgrid.server import buildgrid_instance
    31 
    +from buildgrid.server.instance import BuildGridInstance
    32 32 
     from buildgrid.server._exceptions import InvalidArgumentError
    33 33
    34 
    -from buildgrid.server.execution import operations_service
    34 
    +from buildgrid.server.operations.service import OperationsService
    35 35
    36 36
    37 37 
     instance_name = "blade"
    ... ... @@ -56,14 +56,14 @@ def execute_request():
    56 56
    57 57 
     @pytest.fixture
    58 58 
     def buildgrid():
    59 
    -    yield buildgrid_instance.BuildGridInstance()
    59 
    +    yield BuildGridInstance()
    60 60
    61 61
    62 62 
     # Instance to test
    63 63 
     @pytest.fixture
    64 64 
     def instance(buildgrid):
    65 65 
         instances = {instance_name: buildgrid}
    66 
    -    yield operations_service.OperationsService(instances)
    66 
    +    yield OperationsService(instances)
    67 67
    68 68
    69 69 
     # Queue an execution, get operation corresponding to that request

  • tests/integration/reference_storage_service.py
    ... ... @@ -25,7 +25,8 @@ from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_p
    25 25 
     from buildgrid._protos.buildstream.v2 import buildstream_pb2
    26 26
    27 27 
     from buildgrid.server.cas.storage import lru_memory_cache
    28 
    -from buildgrid.server.cas import reference_cache, reference_storage_service
    28 
    +from buildgrid.server.referencestorage.service import ReferenceStorageService
    29 
    +from buildgrid.server.referencestorage.storage import ReferenceCache
    29 30
    30 31
    31 32 
     # Can mock this
    ... ... @@ -41,12 +42,12 @@ def cas():
    41 42
    42 43 
     @pytest.fixture
    43 44 
     def cache(cas):
    44 
    -    yield reference_cache.ReferenceCache(cas, 50)
    45 
    +    yield ReferenceCache(cas, 50)
    45 46
    46 47
    47 48 
     def test_simple_result(cache, context):
    48 49 
         keys = ["rick", "roy", "rach"]
    49 
    -    service = reference_storage_service.ReferenceStorageService(cache)
    50 
    +    service = ReferenceStorageService(cache)
    50 51
    51 52 
         # Check that before adding the ReferenceResult, attempting to fetch it fails
    52 53 
         request = buildstream_pb2.GetReferenceRequest(key=keys[0])
    ... ... @@ -67,9 +68,9 @@ def test_simple_result(cache, context):
    67 68
    68 69
    69 70 
     def test_disabled_update_result(cache, context):
    70 
    -    disabled_push = reference_cache.ReferenceCache(cas, 50, False)
    71 
    +    disabled_push = ReferenceCache(cas, 50, False)
    71 72 
         keys = ["rick", "roy", "rach"]
    72 
    -    service = reference_storage_service.ReferenceStorageService(disabled_push)
    73 
    +    service = ReferenceStorageService(disabled_push)
    73 74
    74 75 
         # Add an ReferenceResult to the cache
    75 76 
         reference_result = remote_execution_pb2.Digest(hash='deckard')
    ... ... @@ -85,8 +86,8 @@ def test_disabled_update_result(cache, context):
    85 86
    86 87 
     @pytest.mark.parametrize("allow_updates", [True, False])
    87 88 
     def test_status(allow_updates, context):
    88 
    -    cache = reference_cache.ReferenceCache(cas, 5, allow_updates)
    89 
    -    service = reference_storage_service.ReferenceStorageService(cache)
    89 
    +    cache = ReferenceCache(cas, 5, allow_updates)
    90 
    +    service = ReferenceStorageService(cache)
    90 91
    91 92 
         request = buildstream_pb2.StatusRequest()
    92 93 
         response = service.Status(request, context)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]