GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][mablanch/83-executed-action-metadata] 3 commits: job.py: Implement server-side ExecutedActionMetadata

Title: GitLab

Martin Blanchard pushed to branch mablanch/83-executed-action-metadata at BuildGrid / buildgrid

Commits:

5 changed files:

Changes:

  • .pylintrc
    ... ... @@ -184,7 +184,8 @@ ignore-on-opaque-inference=yes
    184 184 
     # List of class names for which member attributes should not be checked (useful
    185 185 
     # for classes with dynamically set attributes). This supports the use of
    186 186 
     # qualified names.
    187 
    -ignored-classes=google.protobuf.any_pb2.Any
    187 
    +ignored-classes=google.protobuf.any_pb2.Any,
    188 
    +                google.protobuf.timestamp_pb2.Timestamp
    188 189
    189 190 
     # List of module names for which member attributes should not be checked
    190 191 
     # (useful for modules/projects where namespaces are manipulated during runtime

  • buildgrid/_app/bots/dummy.py
    ... ... @@ -24,10 +24,23 @@ def work_dummy(context, lease):
    24 24 
         """
    25 25 
         lease.result.Clear()
    26 26
    27 
    -    time.sleep(random.randint(1, 5))
    28 
    -
    29 27 
         action_result = remote_execution_pb2.ActionResult()
    30 28
    29 
    +    # Simulation input-downloading phase:
    30 
    +    action_result.execution_metadata.input_fetch_start_timestamp.GetCurrentTime()
    31 
    +    time.sleep(random.random())
    32 
    +    action_result.execution_metadata.input_fetch_completed_timestamp.GetCurrentTime()
    33 
    +
    34 
    +    # Simulation execution phase:
    35 
    +    action_result.execution_metadata.execution_start_timestamp.GetCurrentTime()
    36 
    +    time.sleep(random.random())
    37 
    +    action_result.execution_metadata.execution_completed_timestamp.GetCurrentTime()
    38 
    +
    39 
    +    # Simulation output-uploading phase:
    40 
    +    action_result.execution_metadata.output_upload_start_timestamp.GetCurrentTime()
    41 
    +    time.sleep(random.random())
    42 
    +    action_result.execution_metadata.output_upload_completed_timestamp.GetCurrentTime()
    43 
    +
    31 44 
         lease.result.Pack(action_result)
    32 45
    33 46 
         return lease

  • buildgrid/_app/bots/host.py
    ... ... @@ -29,11 +29,14 @@ def work_host_tools(context, lease):
    29 29 
         logger = context.logger
    30 30
    31 31 
         action_digest = remote_execution_pb2.Digest()
    32 
    +    action_result = remote_execution_pb2.ActionResult()
    32 33
    33 34 
         lease.payload.Unpack(action_digest)
    34 35 
         lease.result.Clear()
    35 36
    36 37 
         with tempfile.TemporaryDirectory() as temp_directory:
    38 
    +        action_result.execution_metadata.input_fetch_start_timestamp.GetCurrentTime()
    39 
    +
    37 40 
             with download(context.cas_channel, instance=instance_name) as downloader:
    38 41 
                 action = downloader.get_message(action_digest,
    39 42 
                                                 remote_execution_pb2.Action())
    ... ... @@ -45,6 +48,8 @@ def work_host_tools(context, lease):
    45 48
    46 49 
                 downloader.download_directory(action.input_root_digest, temp_directory)
    47 50
    51 
    +        action_result.execution_metadata.input_fetch_completed_timestamp.GetCurrentTime()
    52 
    +
    48 53 
             environment = os.environ.copy()
    49 54 
             for variable in command.environment_variables:
    50 55 
                 if variable.name not in ['PATH', 'PWD']:
    ... ... @@ -70,6 +75,8 @@ def work_host_tools(context, lease):
    70 75
    71 76 
             logger.debug(' '.join(command_line))
    72 77
    78 
    +        action_result.execution_metadata.execution_start_timestamp.GetCurrentTime()
    79 
    +
    73 80 
             process = subprocess.Popen(command_line,
    74 81 
                                        cwd=working_directory,
    75 82 
                                        env=environment,
    ... ... @@ -80,7 +87,8 @@ def work_host_tools(context, lease):
    80 87 
             stdout, stderr = process.communicate()
    81 88 
             returncode = process.returncode
    82 89
    83 
    -        action_result = remote_execution_pb2.ActionResult()
    90 
    +        action_result.execution_metadata.execution_completed_timestamp.GetCurrentTime()
    91 
    +
    84 92 
             # TODO: Upload to CAS or output RAW
    85 93 
             # For now, just pass raw
    86 94 
             # https://gitlab.com/BuildGrid/buildgrid/issues/90
    ... ... @@ -92,6 +100,8 @@ def work_host_tools(context, lease):
    92 100 
             logger.debug("Command stdout: [{}]".format(stdout))
    93 101 
             logger.debug("Command exit code: [{}]".format(returncode))
    94 102
    103 
    +        action_result.execution_metadata.output_upload_start_timestamp.GetCurrentTime()
    104 
    +
    95 105 
             with upload(context.cas_channel, instance=instance_name) as uploader:
    96 106 
                 output_files, output_directories = [], []
    97 107
    ... ... @@ -121,6 +131,8 @@ def work_host_tools(context, lease):
    121 131
    122 132 
                 action_result.output_directories.extend(output_directories)
    123 133
    134 
    +        action_result.execution_metadata.output_upload_completed_timestamp.GetCurrentTime()
    135 
    +
    124 136 
             lease.result.Pack(action_result)
    125 137
    126 138 
         return lease

  • buildgrid/server/job.py
    ... ... @@ -17,6 +17,8 @@ import logging
    17 17 
     import uuid
    18 18 
     from enum import Enum
    19 19
    20 
    +from google.protobuf import timestamp_pb2
    21 
    +
    20 22 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    21 23 
     from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2
    22 24 
     from buildgrid._protos.google.longrunning import operations_pb2
    ... ... @@ -60,6 +62,9 @@ class Job:
    60 62
    61 63 
             self.__execute_response = None
    62 64 
             self.__operation_metadata = remote_execution_pb2.ExecuteOperationMetadata()
    65 
    +        self.__queued_timestamp = timestamp_pb2.Timestamp()
    66 
    +        self.__worker_start_timestamp = timestamp_pb2.Timestamp()
    67 
    +        self.__worker_completed_timestamp = timestamp_pb2.Timestamp()
    63 68
    64 69 
             self.__operation_metadata.action_digest.CopyFrom(action_digest)
    65 70 
             self.__operation_metadata.stage = OperationStage.UNKNOWN.value
    ... ... @@ -177,10 +182,18 @@ class Job:
    177 182 
             self._lease.state = state.value
    178 183
    179 184 
             if self._lease.state == LeaseState.PENDING.value:
    185 
    +            self.__worker_start_timestamp.Clear()
    186 
    +            self.__worker_completed_timestamp.Clear()
    187 
    +
    180 188 
                 self._lease.status.Clear()
    181 189 
                 self._lease.result.Clear()
    182 190
    191 
    +        elif self._lease.state == LeaseState.ACTIVE.value:
    192 
    +            self.__worker_start_timestamp.GetCurrentTime()
    193 
    +
    183 194 
             elif self._lease.state == LeaseState.COMPLETED.value:
    195 
    +            self.__worker_completed_timestamp.GetCurrentTime()
    196 
    +
    184 197 
                 action_result = remote_execution_pb2.ActionResult()
    185 198
    186 199 
                 # TODO: Make a distinction between build and bot failures!
    ... ... @@ -191,6 +204,11 @@ class Job:
    191 204 
                     assert result.Is(action_result.DESCRIPTOR)
    192 205 
                     result.Unpack(action_result)
    193 206
    207 
    +            action_metadata = action_result.execution_metadata
    208 
    +            action_metadata.queued_timestamp.CopyFrom(self.__worker_start_timestamp)
    209 
    +            action_metadata.worker_start_timestamp.CopyFrom(self.__worker_start_timestamp)
    210 
    +            action_metadata.worker_completed_timestamp.CopyFrom(self.__worker_completed_timestamp)
    211 
    +
    194 212 
                 self.__execute_response = remote_execution_pb2.ExecuteResponse()
    195 213 
                 self.__execute_response.result.CopyFrom(action_result)
    196 214 
                 self.__execute_response.cached_result = False
    ... ... @@ -208,6 +226,8 @@ class Job:
    208 226 
             self.__operation_metadata.stage = stage.value
    209 227
    210 228 
             if self.__operation_metadata.stage == OperationStage.QUEUED.value:
    229 
    +            if self.__queued_timestamp.ByteSize() == 0:
    230 
    +                self.__queued_timestamp.GetCurrentTime()
    211 231 
                 self._n_tries += 1
    212 232
    213 233 
             elif self.__operation_metadata.stage == OperationStage.COMPLETED.value:

  • tests/integration/operations_service.py
    ... ... @@ -144,7 +144,8 @@ def test_list_operations_with_result(instance, controller, execute_request, cont
    144 144
    145 145 
         execute_response = remote_execution_pb2.ExecuteResponse()
    146 146 
         response.operations[0].response.Unpack(execute_response)
    147 
    -    assert execute_response.result == action_result
    147 
    +
    148 
    +    assert execute_response.result.output_files == action_result.output_files
    148 149
    149 150
    150 151 
     def test_list_operations_empty(instance, context):

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]