GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][mablanch/117-job-scheduler-refactoring] 4 commits: Fixed minor issue in execute command.

Title: GitLab

Martin Blanchard pushed to branch mablanch/117-job-scheduler-refactoring at BuildGrid / buildgrid

Commits:

9 changed files:

Changes:

  • buildgrid/_app/bots/buildbox.py
    ... ... @@ -17,8 +17,6 @@ import os
    17 17 
     import subprocess
    18 18 
     import tempfile
    19 19
    20 
    -from google.protobuf import any_pb2
    21 
    -
    22 20 
     from buildgrid.client.cas import download, upload
    23 21 
     from buildgrid._exceptions import BotError
    24 22 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    ... ... @@ -29,13 +27,14 @@ from buildgrid.utils import read_file, write_file
    29 27 
     def work_buildbox(context, lease):
    30 28 
         """Executes a lease for a build action, using buildbox.
    31 29 
         """
    32 
    -
    33 30 
         local_cas_directory = context.local_cas
    34 31 
         # instance_name = context.parent
    35 32 
         logger = context.logger
    36 33
    37 34 
         action_digest = remote_execution_pb2.Digest()
    35 
    +
    38 36 
         lease.payload.Unpack(action_digest)
    37 
    +    lease.result.Clear()
    39 38
    40 39 
         with download(context.cas_channel) as downloader:
    41 40 
             action = downloader.get_message(action_digest,
    ... ... @@ -131,10 +130,7 @@ def work_buildbox(context, lease):
    131 130
    132 131 
                 action_result.output_directories.extend([output_directory])
    133 132
    134 
    -            action_result_any = any_pb2.Any()
    135 
    -            action_result_any.Pack(action_result)
    136 
    -
    137 
    -            lease.result.CopyFrom(action_result_any)
    133 
    +            lease.result.Pack(action_result)
    138 134
    139 135 
         return lease
    140 136

  • buildgrid/_app/bots/host.py
    ... ... @@ -17,8 +17,6 @@ import os
    17 17 
     import subprocess
    18 18 
     import tempfile
    19 19
    20 
    -from google.protobuf import any_pb2
    21 
    -
    22 20 
     from buildgrid.client.cas import download, upload
    23 21 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    24 22 
     from buildgrid.utils import output_file_maker, output_directory_maker
    ... ... @@ -27,12 +25,13 @@ from buildgrid.utils import output_file_maker, output_directory_maker
    27 25 
     def work_host_tools(context, lease):
    28 26 
         """Executes a lease for a build action, using host tools.
    29 27 
         """
    30 
    -
    31 28 
         instance_name = context.parent
    32 29 
         logger = context.logger
    33 30
    34 31 
         action_digest = remote_execution_pb2.Digest()
    32 
    +
    35 33 
         lease.payload.Unpack(action_digest)
    34 
    +    lease.result.Clear()
    36 35
    37 36 
         with tempfile.TemporaryDirectory() as temp_directory:
    38 37 
             with download(context.cas_channel, instance=instance_name) as downloader:
    ... ... @@ -122,9 +121,6 @@ def work_host_tools(context, lease):
    122 121
    123 122 
                 action_result.output_directories.extend(output_directories)
    124 123
    125 
    -        action_result_any = any_pb2.Any()
    126 
    -        action_result_any.Pack(action_result)
    127 
    -
    128 
    -        lease.result.CopyFrom(action_result_any)
    124 
    +        lease.result.Pack(action_result)
    129 125
    130 126 
         return lease

  • buildgrid/_app/commands/cmd_execute.py
    ... ... @@ -169,6 +169,7 @@ def run_command(context, input_root, commands, output_file, output_directory):
    169 169
    170 170 
                 downloader.download_file(output_file_response.digest, path)
    171 171
    172 
    -            if output_file_response.path in output_executeables:
    173 
    -                st = os.stat(path)
    174 
    -                os.chmod(path, st.st_mode | stat.S_IXUSR)
    172 
    +    for output_file_response in execute_response.result.output_files:
    173 
    +        if output_file_response.path in output_executeables:
    174 
    +            st = os.stat(path)
    175 
    +            os.chmod(path, st.st_mode | stat.S_IXUSR)

  • buildgrid/server/bots/instance.py
    ... ... @@ -109,7 +109,7 @@ class BotsInterface:
    109 109 
             if server_state == LeaseState.PENDING:
    110 110
    111 111 
                 if client_state == LeaseState.ACTIVE:
    112 
    -                self._scheduler.update_job_lease_state(client_lease.id, client_lease.state)
    112 
    +                self._scheduler.update_job_lease_state(client_lease.id, client_state)
    113 113 
                 elif client_state == LeaseState.COMPLETED:
    114 114 
                     # TODO: Lease was rejected
    115 115 
                     raise NotImplementedError("'Not Accepted' is unsupported")
    ... ... @@ -122,8 +122,7 @@ class BotsInterface:
    122 122 
                     pass
    123 123
    124 124 
                 elif client_state == LeaseState.COMPLETED:
    125 
    -                self._scheduler.update_job_lease_state(client_lease.id, client_lease.state)
    126 
    -                self._scheduler.job_complete(client_lease.id, client_lease.result, client_lease.status)
    125 
    +                self._scheduler.update_job_lease_state(client_lease.id, client_state, lease_status=client_lease.status, lease_result=client_lease.result)
    127 126 
                     return None
    128 127
    129 128 
                 else:

  • buildgrid/server/execution/instance.py
    ... ... @@ -53,7 +53,7 @@ class ExecutionInstance:
    53 53
    54 54 
             self._scheduler.append_job(job, skip_cache_lookup)
    55 55
    56 
    -        return job.get_operation()
    56 
    +        return job.operation
    57 57
    58 58 
         def register_message_client(self, name, queue):
    59 59 
             try:

  • buildgrid/server/job.py
    ... ... @@ -19,57 +19,33 @@ import logging
    19 19 
     import uuid
    20 20 
     from enum import Enum
    21 21
    22 
    -from google.protobuf import any_pb2
    23 
    -
    24 22 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    25 23 
     from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2
    26 24 
     from buildgrid._protos.google.longrunning import operations_pb2
    27 25
    28 26
    29 
    -class ExecuteStage(Enum):
    27 
    +class OperationStage(Enum):
    28 
    +    # Initially unknown stage.
    30 29 
         UNKNOWN = remote_execution_pb2.ExecuteOperationMetadata.Stage.Value('UNKNOWN')
    31 
    -
    32 30 
         # Checking the result against the cache.
    33 31 
         CACHE_CHECK = remote_execution_pb2.ExecuteOperationMetadata.Stage.Value('CACHE_CHECK')
    34 
    -
    35 32 
         # Currently idle, awaiting a free machine to execute.
    36 33 
         QUEUED = remote_execution_pb2.ExecuteOperationMetadata.Stage.Value('QUEUED')
    37 
    -
    38 34 
         # Currently being executed by a worker.
    39 35 
         EXECUTING = remote_execution_pb2.ExecuteOperationMetadata.Stage.Value('EXECUTING')
    40 
    -
    41 36 
         # Finished execution.
    42 37 
         COMPLETED = remote_execution_pb2.ExecuteOperationMetadata.Stage.Value('COMPLETED')
    43 38
    44 39
    45 
    -class BotStatus(Enum):
    46 
    -    BOT_STATUS_UNSPECIFIED = bots_pb2.BotStatus.Value('BOT_STATUS_UNSPECIFIED')
    47 
    -
    48 
    -    # The bot is healthy, and will accept leases as normal.
    49 
    -    OK = bots_pb2.BotStatus.Value('OK')
    50 
    -
    51 
    -    # The bot is unhealthy and will not accept new leases.
    52 
    -    UNHEALTHY = bots_pb2.BotStatus.Value('UNHEALTHY')
    53 
    -
    54 
    -    # The bot has been asked to reboot the host.
    55 
    -    HOST_REBOOTING = bots_pb2.BotStatus.Value('HOST_REBOOTING')
    56 
    -
    57 
    -    # The bot has been asked to shut down.
    58 
    -    BOT_TERMINATING = bots_pb2.BotStatus.Value('BOT_TERMINATING')
    59 
    -
    60 
    -
    61 40 
     class LeaseState(Enum):
    41 
    +    # Initially unknown state.
    62 42 
         LEASE_STATE_UNSPECIFIED = bots_pb2.LeaseState.Value('LEASE_STATE_UNSPECIFIED')
    63 
    -
    64 43 
         # The server expects the bot to accept this lease.
    65 44 
         PENDING = bots_pb2.LeaseState.Value('PENDING')
    66 
    -
    67 45 
         # The bot has accepted this lease.
    68 46 
         ACTIVE = bots_pb2.LeaseState.Value('ACTIVE')
    69 
    -
    70 47 
         # The bot is no longer leased.
    71 48 
         COMPLETED = bots_pb2.LeaseState.Value('COMPLETED')
    72 
    -
    73 49 
         # The bot should immediately release all resources associated with the lease.
    74 50 
         CANCELLED = bots_pb2.LeaseState.Value('CANCELLED')
    75 51
    ... ... @@ -77,18 +53,23 @@ class LeaseState(Enum):
    77 53 
     class Job:
    78 54
    79 55 
         def __init__(self, action_digest, do_not_cache=False, message_queue=None):
    80 
    -        self.lease = None
    81 56 
             self.logger = logging.getLogger(__name__)
    82 
    -        self.n_tries = 0
    83 
    -        self.result = None
    84 
    -        self.result_cached = False
    85 57
    58 
    +        self._name = str(uuid.uuid4())
    86 59 
             self._action_digest = action_digest
    87 60 
             self._do_not_cache = do_not_cache
    88 
    -        self._execute_stage = ExecuteStage.UNKNOWN
    89 
    -        self._name = str(uuid.uuid4())
    90 
    -        self._operation = operations_pb2.Operation(name=self._name)
    61 
    +        self._operation = operations_pb2.Operation()
    62 
    +        self._lease = None
    63 
    +        self._n_tries = 0
    64 
    +
    65 
    +        self.__operation_metadata = remote_execution_pb2.ExecuteOperationMetadata()
    66 
    +
    67 
    +        self.__operation_metadata.action_digest.CopyFrom(self._action_digest)
    68 
    +        self.__operation_metadata.stage = OperationStage.UNKNOWN.value
    69 
    +
    91 70 
             self._operation_update_queues = []
    71 
    +        self._operation.name = self._name
    72 
    +        self._operation.done = False
    92 73
    93 74 
             if message_queue is not None:
    94 75 
                 self.register_client(message_queue)
    ... ... @@ -105,57 +86,72 @@ class Job:
    105 86 
         def do_not_cache(self):
    106 87 
             return self._do_not_cache
    107 88
    108 
    -    def check_job_finished(self):
    109 
    -        if not self._operation_update_queues:
    110 
    -            return self._operation.done
    111 
    -        return False
    89 
    +    @property
    90 
    +    def operation(self):
    91 
    +        return self._operation
    92 
    +
    93 
    +    @property
    94 
    +    def operation_stage(self):
    95 
    +        return OperationStage(self.__operation_metadata.state)
    96 
    +
    97 
    +    @property
    98 
    +    def lease(self):
    99 
    +        return self._lease
    100 
    +
    101 
    +    @property
    102 
    +    def lease_state(self):
    103 
    +        return LeaseState(self._lease.state)
    104 
    +
    105 
    +    @property
    106 
    +    def n_tries(self):
    107 
    +        return self._n_tries
    112 108
    113 109 
         def register_client(self, queue):
    114 110 
             self._operation_update_queues.append(queue)
    115 
    -        queue.put(self.get_operation())
    111 
    +        queue.put(self._operation)
    116 112
    117 113 
         def unregister_client(self, queue):
    118 114 
             self._operation_update_queues.remove(queue)
    119 115
    120 
    -    def get_operation(self):
    121 
    -        self._operation.metadata.CopyFrom(self._pack_any(self.get_operation_meta()))
    122 
    -        if self.result is not None:
    123 
    -            self._operation.done = True
    124 
    -            response = remote_execution_pb2.ExecuteResponse(result=self.result,
    125 
    -                                                            cached_result=self.result_cached)
    116 
    +    def create_lease(self):
    117 
    +        self._lease = bots_pb2.Lease()
    118 
    +        self._lease.id = self._name
    119 
    +        self._lease.payload.Pack(self._action_digest)
    120 
    +        self._lease.state = LeaseState.PENDING.value
    126 121
    127 
    -            if not self.result_cached:
    128 
    -                response.status.CopyFrom(self.lease.status)
    122 
    +        return self._lease
    129 123
    130 
    -            self._operation.response.CopyFrom(self._pack_any(response))
    124 
    +    def update_lease_state(self, state, status=None, result=None):
    125 
    +        if state.value == self._lease.state:
    126 
    +            return
    131 127
    132 
    -        return self._operation
    128 
    +        self._lease.state = state.value
    133 129
    134 
    -    def get_operation_meta(self):
    135 
    -        meta = remote_execution_pb2.ExecuteOperationMetadata()
    136 
    -        meta.stage = self._execute_stage.value
    137 
    -        meta.action_digest.CopyFrom(self._action_digest)
    130 
    +        if self._lease.state == LeaseState.COMPLETED.value:
    131 
    +            response = remote_execution_pb2.ExecuteResponse()
    132 
    +            response.result.CopyFrom(result)
    133 
    +            response.cached_result = False
    134 
    +            response.status.CopyFrom(status)
    138 135
    139 
    -        return meta
    136 
    +            self._operation.response.Pack(response)
    137 
    +            self._operation.done = True
    140 138
    141 
    -    def create_lease(self):
    142 
    -        action_digest = self._pack_any(self._action_digest)
    139 
    +    def cancel_lease(self):
    140 
    +        pass
    141 
    +
    142 
    +    def update_operation_stage(self, stage):
    143 
    +        if stage.value == self.__operation_metadata.stage:
    144 
    +            return
    145 
    +
    146 
    +        self.__operation_metadata.stage = stage.value
    143 147
    144 
    -        lease = bots_pb2.Lease(id=self.name,
    145 
    -                               payload=action_digest,
    146 
    -                               state=LeaseState.PENDING.value)
    147 
    -        self.lease = lease
    148 
    -        return lease
    148 
    +        if self.__operation_metadata.stage == OperationStage.QUEUED.value:
    149 
    +            self._n_tries += 1
    149 150
    150 
    -    def get_operations(self):
    151 
    -        return operations_pb2.ListOperationsResponse(operations=[self.get_operation()])
    151 
    +        self._operation.metadata.Pack(self.__operation_metadata)
    152 152
    153 
    -    def update_execute_stage(self, stage):
    154 
    -        self._execute_stage = stage
    155 153 
             for queue in self._operation_update_queues:
    156 
    -            queue.put(self.get_operation())
    154 
    +            queue.put(self._operation)
    157 155
    158 
    -    def _pack_any(self, pack):
    159 
    -        some_any = any_pb2.Any()
    160 
    -        some_any.Pack(pack)
    161 
    -        return some_any
    156 
    +    def cancel_operation(self):
    157 
    +        pass

  • buildgrid/server/scheduler.py
    ... ... @@ -27,7 +27,7 @@ from buildgrid._exceptions import NotFoundError
    27 27 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    28 28 
     from buildgrid._protos.google.longrunning import operations_pb2
    29 29
    30 
    -from .job import ExecuteStage, LeaseState
    30 
    +from .job import OperationStage, LeaseState
    31 31
    32 32
    33 33 
     class Scheduler:
    ... ... @@ -45,7 +45,7 @@ class Scheduler:
    45 45 
         def unregister_client(self, name, queue):
    46 46 
             job = self.jobs[name]
    47 47 
             job.unregister_client(queue)
    48 
    -        if job.check_job_finished():
    48 
    +        if job.operation.done:
    49 49 
                 del self.jobs[name]
    50 50
    51 51 
         def append_job(self, job, skip_cache_lookup=False):
    ... ... @@ -55,64 +55,55 @@ class Scheduler:
    55 55 
                     cached_result = self._action_cache.get_action_result(job.action_digest)
    56 56 
                 except NotFoundError:
    57 57 
                     self.queue.append(job)
    58 
    -                job.update_execute_stage(ExecuteStage.QUEUED)
    58 
    +                job.update_operation_stage(OperationStage.QUEUED)
    59 59
    60 60 
                 else:
    61 
    -                job.result = cached_result
    62 
    -                job.result_cached = True
    63 
    -                job.update_execute_stage(ExecuteStage.COMPLETED)
    61 
    +                job.update_operation_stage(OperationStage.COMPLETED)
    64 62
    65 63 
             else:
    66 64 
                 self.queue.append(job)
    67 
    -            job.update_execute_stage(ExecuteStage.QUEUED)
    65 
    +            job.update_operation_stage(OperationStage.QUEUED)
    68 66
    69 67 
         def retry_job(self, name):
    70 68 
             if name in self.jobs:
    71 69 
                 job = self.jobs[name]
    72 70 
                 if job.n_tries >= self.MAX_N_TRIES:
    73 71 
                     # TODO: Decide what to do with these jobs
    74 
    -                job.update_execute_stage(ExecuteStage.COMPLETED)
    72 
    +                job.update_operation_stage(OperationStage.COMPLETED)
    75 73 
                     # TODO: Mark these jobs as done
    76 74 
                 else:
    77 
    -                job.update_execute_stage(ExecuteStage.QUEUED)
    78 
    -                job.n_tries += 1
    75 
    +                job.update_operation_stage(OperationStage.QUEUED)
    79 76 
                     self.queue.appendleft(job)
    80 77
    81 
    -    def job_complete(self, name, result, status):
    82 
    -        job = self.jobs[name]
    83 
    -        job.lease.status.CopyFrom(status)
    84 
    -        action_result = remote_execution_pb2.ActionResult()
    85 
    -        result.Unpack(action_result)
    86 
    -        job.result = action_result
    87 
    -        if not job.do_not_cache and self._action_cache is not None:
    88 
    -            if not job.lease.status.code:
    89 
    -                self._action_cache.update_action_result(job.action_digest, action_result)
    90 
    -        job.update_execute_stage(ExecuteStage.COMPLETED)
    91 
    -
    92 78 
         def get_operations(self):
    93 79 
             response = operations_pb2.ListOperationsResponse()
    94 80 
             for v in self.jobs.values():
    95 
    -            response.operations.extend([v.get_operation()])
    81 
    +            response.operations.extend([v.operation])
    96 82 
             return response
    97 83
    98 
    -    def update_job_lease_state(self, name, state):
    99 
    -        job = self.jobs[name]
    100 
    -        job.lease.state = state
    84 
    +    def update_job_lease_state(self, job_name, lease_state, lease_status=None, lease_result=None):
    85 
    +        job = self.jobs[job_name]
    86 
    +        if lease_state != LeaseState.COMPLETED:
    87 
    +            job.update_lease_state(lease_state)
    88 
    +        else:
    89 
    +            action_result = remote_execution_pb2.ActionResult()
    90 
    +            lease_result.Unpack(action_result)
    91 
    +
    92 
    +            job.update_lease_state(lease_state, status=lease_status, result=action_result)
    93 
    +
    94 
    +            if not job.do_not_cache and self._action_cache is not None:
    95 
    +                if not job.lease.status.code:
    96 
    +                    self._action_cache.update_action_result(job.action_digest, action_result)
    97 
    +
    98 
    +            job.update_operation_stage(OperationStage.COMPLETED)
    101 99
    102 100 
         def get_job_lease(self, name):
    103 101 
             return self.jobs[name].lease
    104 102
    105 
    -    def cancel_session(self, name):
    106 
    -        job = self.jobs[name]
    107 
    -        state = job.lease.state
    108 
    -        if state in (LeaseState.PENDING.value, LeaseState.ACTIVE.value):
    109 
    -            self.retry_job(name)
    110 
    -
    111 103 
         def create_lease(self):
    112 104 
             if self.queue:
    113 105 
                 job = self.queue.popleft()
    114 
    -            job.update_execute_stage(ExecuteStage.EXECUTING)
    106 
    +            job.update_operation_stage(OperationStage.EXECUTING)
    115 107 
                 job.create_lease()
    116 
    -            job.lease.state = LeaseState.PENDING.value
    117 108 
                 return job.lease
    118 109 
             return None

  • tests/integration/execution_service.py
    ... ... @@ -82,7 +82,7 @@ def test_execute(skip_cache_lookup, instance, context):
    82 82 
         assert isinstance(result, operations_pb2.Operation)
    83 83 
         metadata = remote_execution_pb2.ExecuteOperationMetadata()
    84 84 
         result.metadata.Unpack(metadata)
    85 
    -    assert metadata.stage == job.ExecuteStage.QUEUED.value
    85 
    +    assert metadata.stage == job.OperationStage.QUEUED.value
    86 86 
         assert uuid.UUID(result.name, version=4)
    87 87 
         assert result.done is False
    88 88
    ... ... @@ -116,7 +116,7 @@ def test_wait_execution(instance, controller, context):
    116 116 
         action_result = remote_execution_pb2.ActionResult()
    117 117 
         action_result_any.Pack(action_result)
    118 118
    119 
    -    j.update_execute_stage(job.ExecuteStage.COMPLETED)
    119 
    +    j.update_operation_stage(job.OperationStage.COMPLETED)
    120 120
    121 121 
         response = instance.WaitExecution(request, context)
    122 122
    ... ... @@ -125,7 +125,7 @@ def test_wait_execution(instance, controller, context):
    125 125 
         assert isinstance(result, operations_pb2.Operation)
    126 126 
         metadata = remote_execution_pb2.ExecuteOperationMetadata()
    127 127 
         result.metadata.Unpack(metadata)
    128 
    -    assert metadata.stage == job.ExecuteStage.COMPLETED.value
    128 
    +    assert metadata.stage == job.OperationStage.COMPLETED.value
    129 129 
         assert uuid.UUID(result.name, version=4)
    130 130 
         assert result.done is True
    131 131

  • tests/integration/operations_service.py
    ... ... @@ -30,6 +30,7 @@ from buildgrid._protos.google.longrunning import operations_pb2
    30 30 
     from buildgrid._protos.google.rpc import status_pb2
    31 31 
     from buildgrid.server.cas.storage import lru_memory_cache
    32 32 
     from buildgrid.server.controller import ExecutionController
    33 
    +from buildgrid.server.job import OperationStage
    33 34 
     from buildgrid.server.operations import service
    34 35 
     from buildgrid.server.operations.service import OperationsService
    35 36 
     from buildgrid.utils import create_digest
    ... ... @@ -131,9 +132,10 @@ def test_list_operations_with_result(instance, controller, execute_request, cont
    131 132 
         action_result.output_files.extend([output_file])
    132 133
    133 134 
         controller.operations_instance._scheduler.jobs[response_execute.name].create_lease()
    134 
    -    controller.operations_instance._scheduler.job_complete(response_execute.name,
    135 
    -                                                           _pack_any(action_result),
    136 
    -                                                           status_pb2.Status())
    135 
    +    controller.operations_instance._scheduler.update_job_lease_state(response_execute.name,
    136 
    +                                                                     OperationStage.COMPLETED,
    137 
    +                                                                     lease_status=status_pb2.Status(),
    138 
    +                                                                     lease_result=_pack_any(action_result))
    137 139
    138 140 
         request = operations_pb2.ListOperationsRequest(name=instance_name)
    139 141 
         response = instance.ListOperations(request, context)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]