GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][finn/instances] Instance name support unittests

Title: GitLab

finn pushed to branch finn/instances at BuildGrid / buildgrid

Commits:

3 changed files:

Changes:

  • tests/integration/bots_service.py
    ... ... @@ -18,7 +18,6 @@
    18 18 
     # pylint: disable=redefined-outer-name
    19 19
    20 20 
     import copy
    21 
    -import uuid
    22 21 
     from unittest import mock
    23 22
    24 23 
     import grpc
    ... ... @@ -27,7 +26,7 @@ import pytest
    27 26
    28 27 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    29 28 
     from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2
    30 
    -from buildgrid.server import scheduler, job
    29 
    +from buildgrid.server import job, buildgrid_instance
    31 30 
     from buildgrid.server.job import LeaseState
    32 31 
     from buildgrid.server.worker import bots_interface, bots_service
    33 32
    ... ... @@ -53,8 +52,8 @@ def bot_session():
    53 52
    54 53
    55 54 
     @pytest.fixture
    56 
    -def schedule():
    57 
    -    yield scheduler.Scheduler()
    55 
    +def buildgrid():
    56 
    +    yield buildgrid_instance.BuildGridInstance()
    58 57
    59 58
    60 59 
     @pytest.fixture
    ... ... @@ -64,19 +63,17 @@ def bots(schedule):
    64 63
    65 64 
     # Instance to test
    66 65 
     @pytest.fixture
    67 
    -def instance(bots):
    68 
    -    yield bots_service.BotsService(bots)
    66 
    +def instance(buildgrid):
    67 
    +    instances = {"": buildgrid}
    68 
    +    yield bots_service.BotsService(instances)
    69 69
    70 70
    71 71 
     def test_create_bot_session(bot_session, context, instance):
    72 
    -    parent = 'rach'
    73 
    -    request = bots_pb2.CreateBotSessionRequest(parent=parent,
    74 
    -                                               bot_session=bot_session)
    72 
    +    request = bots_pb2.CreateBotSessionRequest(bot_session=bot_session)
    75 73
    76 74 
         response = instance.CreateBotSession(request, context)
    77 75
    78 76 
         assert isinstance(response, bots_pb2.BotSession)
    79 
    -    assert uuid.UUID(response.name, version=4)
    80 77 
         assert bot_session.bot_id == response.bot_id
    81 78
    82 79
    ... ... @@ -92,8 +89,7 @@ def test_create_bot_session_bot_id_fail(context, instance):
    92 89
    93 90
    94 91 
     def test_update_bot_session(bot_session, context, instance):
    95 
    -    request = bots_pb2.CreateBotSessionRequest(parent='',
    96 
    -                                               bot_session=bot_session)
    92 
    +    request = bots_pb2.CreateBotSessionRequest(bot_session=bot_session)
    97 93 
         bot = instance.CreateBotSession(request, context)
    98 94
    99 95 
         request = bots_pb2.UpdateBotSessionRequest(name=bot.name,
    ... ... @@ -106,8 +102,7 @@ def test_update_bot_session(bot_session, context, instance):
    106 102
    107 103
    108 104 
     def test_update_bot_session_zombie(bot_session, context, instance):
    109 
    -    request = bots_pb2.CreateBotSessionRequest(parent='',
    110 
    -                                               bot_session=bot_session)
    105 
    +    request = bots_pb2.CreateBotSessionRequest(bot_session=bot_session)
    111 106 
         bot = instance.CreateBotSession(request, context)
    112 107 
         # Update server with incorrect UUID by rotating it
    113 108 
         bot.name = bot.name[len(bot.name): 0]
    ... ... @@ -121,8 +116,7 @@ def test_update_bot_session_zombie(bot_session, context, instance):
    121 116
    122 117
    123 118 
     def test_update_bot_session_bot_id_fail(bot_session, context, instance):
    124 
    -    request = bots_pb2.UpdateBotSessionRequest(name='ana',
    125 
    -                                               bot_session=bot_session)
    119 
    +    request = bots_pb2.UpdateBotSessionRequest(bot_session=bot_session)
    126 120
    127 121 
         instance.UpdateBotSession(request, context)
    128 122
    ... ... @@ -131,17 +125,15 @@ def test_update_bot_session_bot_id_fail(bot_session, context, instance):
    131 125
    132 126 
     @pytest.mark.parametrize("number_of_jobs", [0, 1, 3, 500])
    133 127 
     def test_number_of_leases(number_of_jobs, bot_session, context, instance):
    134 
    -    request = bots_pb2.CreateBotSessionRequest(parent='',
    135 
    -                                               bot_session=bot_session)
    128 
    +    request = bots_pb2.CreateBotSessionRequest(bot_session=bot_session)
    136 129 
         # Inject work
    137 130 
         for _ in range(0, number_of_jobs):
    138 131 
             action_digest = remote_execution_pb2.Digest()
    139 
    -        instance._instance._scheduler.append_job(job.Job(action_digest))
    132 
    +        instance._instances[""].execute(action_digest, True)
    140 133
    141 134 
         response = instance.CreateBotSession(request, context)
    142 135
    143 136 
         assert len(response.leases) == number_of_jobs
    144 
    -    assert isinstance(response, bots_pb2.BotSession)
    145 137
    146 138
    147 139 
     def test_update_leases_with_work(bot_session, context, instance):
    ... ... @@ -149,7 +141,7 @@ def test_update_leases_with_work(bot_session, context, instance):
    149 141 
                                                    bot_session=bot_session)
    150 142 
         # Inject work
    151 143 
         action_digest = remote_execution_pb2.Digest(hash='gaff')
    152 
    -    instance._instance._scheduler.append_job(job.Job(action_digest))
    144 
    +    instance._instances[""].execute(action_digest, True)
    153 145
    154 146 
         response = instance.CreateBotSession(request, context)
    155 147
    ... ... @@ -159,7 +151,6 @@ def test_update_leases_with_work(bot_session, context, instance):
    159 151
    160 152 
         assert isinstance(response, bots_pb2.BotSession)
    161 153 
         assert response.leases[0].state == LeaseState.PENDING.value
    162 
    -    assert uuid.UUID(response.leases[0].id, version=4)
    163 154 
         assert response_action == action_digest
    164 155
    165 156
    ... ... @@ -172,7 +163,7 @@ def test_update_leases_work_complete(bot_session, context, instance):
    172 163
    173 164 
         # Inject work
    174 165 
         action_digest = remote_execution_pb2.Digest(hash='gaff')
    175 
    -    instance._instance._scheduler.append_job(job.Job(action_digest))
    166 
    +    instance._instances[""].execute(action_digest, True)
    176 167
    177 168 
         request = bots_pb2.UpdateBotSessionRequest(name=response.name,
    178 169 
                                                    bot_session=response)
    ... ... @@ -200,7 +191,7 @@ def test_work_rejected_by_bot(bot_session, context, instance):
    200 191 
                                                    bot_session=bot_session)
    201 192 
         # Inject work
    202 193 
         action_digest = remote_execution_pb2.Digest(hash='gaff')
    203 
    -    instance._instance._scheduler.append_job(job.Job(action_digest))
    194 
    +    instance._instances[""].execute(action_digest, True)
    204 195
    205 196 
         # Simulated the severed binding between client and server
    206 197 
         response = copy.deepcopy(instance.CreateBotSession(request, context))
    ... ... @@ -222,7 +213,8 @@ def test_work_out_of_sync_from_pending(state, bot_session, context, instance):
    222 213 
                                                    bot_session=bot_session)
    223 214 
         # Inject work
    224 215 
         action_digest = remote_execution_pb2.Digest(hash='gaff')
    225 
    -    instance._instance._scheduler.append_job(job.Job(action_digest))
    216 
    +    instance._instances[""].execute(action_digest, True)
    217 
    +
    226 218 
         # Simulated the severed binding between client and server
    227 219 
         response = copy.deepcopy(instance.CreateBotSession(request, context))
    228 220
    ... ... @@ -242,7 +234,8 @@ def test_work_out_of_sync_from_active(state, bot_session, context, instance):
    242 234 
                                                    bot_session=bot_session)
    243 235 
         # Inject work
    244 236 
         action_digest = remote_execution_pb2.Digest(hash='gaff')
    245 
    -    instance._instance._scheduler.append_job(job.Job(action_digest))
    237 
    +    instance._instances[""].execute(action_digest, True)
    238 
    +
    246 239 
         # Simulated the severed binding between client and server
    247 240 
         response = copy.deepcopy(instance.CreateBotSession(request, context))
    248 241
    ... ... @@ -268,7 +261,8 @@ def test_work_active_to_active(bot_session, context, instance):
    268 261 
                                                    bot_session=bot_session)
    269 262 
         # Inject work
    270 263 
         action_digest = remote_execution_pb2.Digest(hash='gaff')
    271 
    -    instance._instance._scheduler.append_job(job.Job(action_digest))
    264 
    +    instance._instances[""].execute(action_digest, True)
    265 
    +
    272 266 
         # Simulated the severed binding between client and server
    273 267 
         response = copy.deepcopy(instance.CreateBotSession(request, context))
    274 268

  • tests/integration/execution_service.py
    ... ... @@ -20,15 +20,17 @@
    20 20 
     import uuid
    21 21 
     from unittest import mock
    22 22
    23 
    +import grpc
    23 24 
     from grpc._server import _Context
    24 25 
     import pytest
    26 
    +from google.protobuf import any_pb2
    25 27
    26 28 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    27 29 
     from buildgrid._protos.google.longrunning import operations_pb2
    28 30
    29 
    -from buildgrid.server import scheduler, job
    31 
    +from buildgrid.server import job, buildgrid_instance
    30 32 
     from buildgrid.server.cas.storage import lru_memory_cache
    31 
    -from buildgrid.server.execution import action_cache, execution_instance, execution_service
    33 
    +from buildgrid.server.execution import action_cache, execution_service
    32 34
    33 35
    34 36 
     @pytest.fixture
    ... ... @@ -38,19 +40,21 @@ def context():
    38 40
    39 41
    40 42 
     @pytest.fixture(params=["action-cache", "no-action-cache"])
    41 
    -def execution(request):
    43 
    +def buildgrid(request):
    42 44 
         if request.param == "action-cache":
    43 45 
             storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
    44 46 
             cache = action_cache.ActionCache(storage, 50)
    45 
    -        schedule = scheduler.Scheduler(cache)
    46 
    -        return execution_instance.ExecutionInstance(schedule, storage)
    47 
    -    return execution_instance.ExecutionInstance(scheduler.Scheduler())
    47 
    +
    48 
    +        return buildgrid_instance.BuildGridInstance(action_cache=cache,
    49 
    +                                                    cas_storage=storage)
    50 
    +    return buildgrid_instance.BuildGridInstance()
    48 51
    49 52
    50 53 
     # Instance to test
    51 54 
     @pytest.fixture
    52 
    -def instance(execution):
    53 
    -    yield execution_service.ExecutionService(execution)
    55 
    +def instance(buildgrid):
    56 
    +    instances = {"": buildgrid}
    57 
    +    yield execution_service.ExecutionService(instances)
    54 58
    55 59
    56 60 
     @pytest.mark.parametrize("skip_cache_lookup", [True, False])
    ... ... @@ -72,23 +76,45 @@ def test_execute(skip_cache_lookup, instance, context):
    72 76 
         assert result.done is False
    73 77
    74 78
    75 
    -# def test_wait_execution(instance, context):
    76 
    -    # TODO: Figure out why next(response) hangs on the .get()
    77 
    -    # method when running in pytest.
    78 
    -#     action_digest = remote_execution_pb2.Digest()
    79 
    -#     action_digest.hash = 'zhora'
    79 
    +def test_wrong_execute_instance(instance, context):
    80 
    +    request = remote_execution_pb2.ExecuteRequest(instance_name='blade')
    81 
    +    response = instance.Execute(request, context)
    82 
    +
    83 
    +    next(response)
    84 
    +    context.set_code.assert_called_once_with(grpc.StatusCode.INVALID_ARGUMENT)
    85 
    +
    86 
    +
    87 
    +def test_wait_execution(instance, buildgrid, context):
    88 
    +    action_digest = remote_execution_pb2.Digest()
    89 
    +    action_digest.hash = 'zhora'
    90 
    +
    91 
    +    j = job.Job(action_digest, None)
    92 
    +    j._operation.done = True
    93 
    +
    94 
    +    request = remote_execution_pb2.WaitExecutionRequest(name="{}/{}".format('', j.name))
    80 95
    81 
    -#     j = job.Job(action_digest, None)
    82 
    -#     j._operation.done = True
    96 
    +    buildgrid._scheduler.jobs[j.name] = j
    83 97
    84 
    -#     request = remote_execution_pb2.WaitExecutionRequest(name=j.name)
    98 
    +    action_result_any = any_pb2.Any()
    99 
    +    action_result = remote_execution_pb2.ActionResult()
    100 
    +    action_result_any.Pack(action_result)
    85 101
    86 
    -#     instance._instance._scheduler.jobs[j.name] = j
    102 
    +    j.update_execute_stage(job.ExecuteStage.COMPLETED)
    103 
    +
    104 
    +    response = instance.WaitExecution(request, context)
    105 
    +
    106 
    +    result = next(response)
    107 
    +
    108 
    +    assert isinstance(result, operations_pb2.Operation)
    109 
    +    metadata = remote_execution_pb2.ExecuteOperationMetadata()
    110 
    +    result.metadata.Unpack(metadata)
    111 
    +    assert metadata.stage == job.ExecuteStage.COMPLETED.value
    112 
    +    assert uuid.UUID(result.name, version=4)
    113 
    +    assert result.done is True
    87 114
    88 
    -#     action_result_any = any_pb2.Any()
    89 
    -#     action_result = remote_execution_pb2.ActionResult()
    90 
    -#     action_result_any.Pack(action_result)
    91 115
    92 
    -#     instance._instance._scheduler._update_execute_stage(j, job.ExecuteStage.COMPLETED)
    116 
    +def test_wrong_instance_wait_execution(instance, buildgrid, context):
    117 
    +    request = remote_execution_pb2.WaitExecutionRequest(name="blade")
    118 
    +    next(instance.WaitExecution(request, context))
    93 119
    94 
    -#     response = instance.WaitExecution(request, context)
    120 
    +    context.set_code.assert_called_once_with(grpc.StatusCode.INVALID_ARGUMENT)

  • tests/integration/operations_service.py
    ... ... @@ -28,10 +28,13 @@ from google.protobuf import any_pb2
    28 28 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    29 29 
     from buildgrid._protos.google.longrunning import operations_pb2
    30 30
    31 
    -from buildgrid.server import scheduler
    31 
    +from buildgrid.server import buildgrid_instance
    32 32 
     from buildgrid.server._exceptions import InvalidArgumentError
    33 33
    34 
    -from buildgrid.server.execution import execution_instance, operations_service
    34 
    +from buildgrid.server.execution import operations_service
    35 
    +
    36 
    +
    37 
    +instance_name = "blade"
    35 38
    36 39
    37 40 
     # Can mock this
    ... ... @@ -52,29 +55,25 @@ def execute_request():
    52 55
    53 56
    54 57 
     @pytest.fixture
    55 
    -def schedule():
    56 
    -    yield scheduler.Scheduler()
    57 
    -
    58 
    -
    59 
    -@pytest.fixture
    60 
    -def execution(schedule):
    61 
    -    yield execution_instance.ExecutionInstance(schedule)
    58 
    +def buildgrid():
    59 
    +    yield buildgrid_instance.BuildGridInstance()
    62 60
    63 61
    64 62 
     # Instance to test
    65 63 
     @pytest.fixture
    66 
    -def instance(execution):
    67 
    -    yield operations_service.OperationsService(execution)
    64 
    +def instance(buildgrid):
    65 
    +    instances = {instance_name: buildgrid}
    66 
    +    yield operations_service.OperationsService(instances)
    68 67
    69 68
    70 69 
     # Queue an execution, get operation corresponding to that request
    71 
    -def test_get_operation(instance, execute_request, context):
    72 
    -    response_execute = instance._instance.execute(execute_request.action_digest,
    73 
    -                                                  execute_request.skip_cache_lookup)
    70 
    +def test_get_operation(instance, buildgrid, execute_request, context):
    71 
    +    response_execute = buildgrid.execute(execute_request.action_digest,
    72 
    +                                         execute_request.skip_cache_lookup)
    74 73
    75 74 
         request = operations_pb2.GetOperationRequest()
    76 75
    77 
    -    request.name = response_execute.name
    76 
    +    request.name = "{}/{}".format(instance_name, response_execute.name)
    78 77
    79 78 
         response = instance.GetOperation(request, context)
    80 79 
         assert response is response_execute
    ... ... @@ -82,35 +81,54 @@ def test_get_operation(instance, execute_request, context):
    82 81
    83 82 
     def test_get_operation_fail(instance, context):
    84 83 
         request = operations_pb2.GetOperationRequest()
    84 
    +    request.name = "{}/{}".format(instance_name, "runner")
    85 85 
         instance.GetOperation(request, context)
    86 86
    87 87 
         context.set_code.assert_called_once_with(grpc.StatusCode.INVALID_ARGUMENT)
    88 88
    89 89
    90 
    -def test_list_operations(instance, execute_request, context):
    91 
    -    response_execute = instance._instance.execute(execute_request.action_digest,
    92 
    -                                                  execute_request.skip_cache_lookup)
    90 
    +def test_get_operation_instance_fail(instance, context):
    91 
    +    request = operations_pb2.GetOperationRequest()
    92 
    +    instance.GetOperation(request, context)
    93 
    +
    94 
    +    context.set_code.assert_called_once_with(grpc.StatusCode.INVALID_ARGUMENT)
    95 
    +
    96 
    +
    97 
    +def test_list_operations(instance, buildgrid, execute_request, context):
    98 
    +    response_execute = buildgrid.execute(execute_request.action_digest,
    99 
    +                                         execute_request.skip_cache_lookup)
    100 
    +
    101 
    +    request = operations_pb2.ListOperationsRequest(name=instance_name)
    102 
    +    response = instance.ListOperations(request, context)
    103 
    +
    104 
    +    assert response.operations[0].name.split('/')[-1] == response_execute.name
    105 
    +
    106 
    +
    107 
    +def test_list_operations_instance_fail(instance, buildgrid, execute_request, context):
    108 
    +    response_execute = buildgrid.execute(execute_request.action_digest,
    109 
    +                                         execute_request.skip_cache_lookup)
    93 110
    94 111 
         request = operations_pb2.ListOperationsRequest()
    95 112 
         response = instance.ListOperations(request, context)
    96 113
    97 
    -    assert response.operations[0].name == response_execute.name
    114 
    +    context.set_code.assert_called_once_with(grpc.StatusCode.INVALID_ARGUMENT)
    98 115
    99 116
    100 
    -def test_list_operations_with_result(instance, execute_request, context):
    101 
    -    response_execute = instance._instance.execute(execute_request.action_digest,
    102 
    -                                                  execute_request.skip_cache_lookup)
    117 
    +def test_list_operations_with_result(instance, buildgrid, execute_request, context):
    118 
    +    response_execute = buildgrid.execute(execute_request.action_digest,
    119 
    +                                         execute_request.skip_cache_lookup)
    103 120
    104 121 
         action_result = remote_execution_pb2.ActionResult()
    105 122 
         output_file = remote_execution_pb2.OutputFile(path='unicorn')
    106 123 
         action_result.output_files.extend([output_file])
    107 124
    108 
    -    instance._instance._scheduler.job_complete(response_execute.name, _pack_any(action_result))
    125 
    +    buildgrid._scheduler.job_complete(response_execute.name,
    126 
    +                                      _pack_any(action_result))
    109 127
    110 
    -    request = operations_pb2.ListOperationsRequest()
    128 
    +    request = operations_pb2.ListOperationsRequest(name=instance_name)
    111 129 
         response = instance.ListOperations(request, context)
    112 130
    113 
    -    assert response.operations[0].name == response_execute.name
    131 
    +    assert response.operations[0].name.split('/')[-1] == response_execute.name
    114 132
    115 133 
         execute_response = remote_execution_pb2.ExecuteResponse()
    116 134 
         response.operations[0].response.Unpack(execute_response)
    ... ... @@ -118,7 +136,7 @@ def test_list_operations_with_result(instance, execute_request, context):
    118 136
    119 137
    120 138 
     def test_list_operations_empty(instance, context):
    121 
    -    request = operations_pb2.ListOperationsRequest()
    139 
    +    request = operations_pb2.ListOperationsRequest(name=instance_name)
    122 140
    123 141 
         response = instance.ListOperations(request, context)
    124 142
    ... ... @@ -126,21 +144,23 @@ def test_list_operations_empty(instance, context):
    126 144
    127 145
    128 146 
     # Send execution off, delete, try to find operation should fail
    129 
    -def test_delete_operation(instance, execute_request, context):
    130 
    -    response_execute = instance._instance.execute(execute_request.action_digest,
    131 
    -                                                  execute_request.skip_cache_lookup)
    147 
    +def test_delete_operation(instance, buildgrid, execute_request, context):
    148 
    +    response_execute = buildgrid.execute(execute_request.action_digest,
    149 
    +                                         execute_request.skip_cache_lookup)
    132 150 
         request = operations_pb2.DeleteOperationRequest()
    133 
    -    request.name = response_execute.name
    151 
    +    request.name = "{}/{}".format(instance_name, response_execute.name)
    134 152 
         instance.DeleteOperation(request, context)
    135 153
    136 154 
         request = operations_pb2.GetOperationRequest()
    137 
    -    request.name = response_execute.name
    155 
    +    request.name = "{}/{}".format(instance_name, response_execute.name)
    156 
    +
    138 157 
         with pytest.raises(InvalidArgumentError):
    139 
    -        instance._instance.get_operation(response_execute.name)
    158 
    +        buildgrid.get_operation(response_execute.name)
    140 159
    141 160
    142 
    -def test_delete_operation_fail(instance, execute_request, context):
    161 
    +def test_delete_operation_fail(instance, context):
    143 162 
         request = operations_pb2.DeleteOperationRequest()
    163 
    +    request.name = "{}/{}".format(instance_name, "runner")
    144 164 
         instance.DeleteOperation(request, context)
    145 165
    146 166 
         context.set_code.assert_called_once_with(grpc.StatusCode.INVALID_ARGUMENT)
    ... ... @@ -148,11 +168,19 @@ def test_delete_operation_fail(instance, execute_request, context):
    148 168
    149 169 
     def test_cancel_operation(instance, context):
    150 170 
         request = operations_pb2.CancelOperationRequest()
    171 
    +    request.name = "{}/{}".format(instance_name, "runner")
    151 172 
         instance.CancelOperation(request, context)
    152 173
    153 174 
         context.set_code.assert_called_once_with(grpc.StatusCode.UNIMPLEMENTED)
    154 175
    155 176
    177 
    +def test_cancel_operation_instance_fail(instance, context):
    178 
    +    request = operations_pb2.CancelOperationRequest()
    179 
    +    instance.CancelOperation(request, context)
    180 
    +
    181 
    +    context.set_code.assert_called_once_with(grpc.StatusCode.INVALID_ARGUMENT)
    182 
    +
    183 
    +
    156 184 
     def _pack_any(pack):
    157 185 
         some_any = any_pb2.Any()
    158 186 
         some_any.Pack(pack)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]