finn pushed to branch finn/84-bot-errors at BuildGrid / buildgrid
Commits:
2 changed files:
Changes:
| ... | ... | @@ -19,7 +19,9 @@ import tempfile |
| 19 | 19 |
|
| 20 | 20 |
from google.protobuf import any_pb2
|
| 21 | 21 |
|
| 22 |
+from buildgrid.settings import HASH_LENGTH
|
|
| 22 | 23 |
from buildgrid.client.cas import upload
|
| 24 |
+from buildgrid._exceptions import BotError
|
|
| 23 | 25 |
from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
|
| 24 | 26 |
from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
|
| 25 | 27 |
from buildgrid.utils import read_file, write_file, parse_to_pb2_from_fetch
|
| ... | ... | @@ -87,17 +89,28 @@ def work_buildbox(context, lease): |
| 87 | 89 |
|
| 88 | 90 |
command_line = subprocess.Popen(command_line,
|
| 89 | 91 |
stdin=subprocess.PIPE,
|
| 90 |
- stdout=subprocess.PIPE)
|
|
| 91 |
- # TODO: Should return the stdout and stderr to the user.
|
|
| 92 |
- command_line.communicate()
|
|
| 92 |
+ stdout=subprocess.PIPE,
|
|
| 93 |
+ stderr=subprocess.PIPE)
|
|
| 94 |
+ stdout, stderr = command_line.communicate()
|
|
| 95 |
+ action_result = remote_execution_pb2.ActionResult()
|
|
| 96 |
+ # TODO: Upload to CAS or output RAW
|
|
| 97 |
+ # For now, just pass raw
|
|
| 98 |
+ action_result.stdout_raw = stdout
|
|
| 99 |
+ |
|
| 100 |
+ if stderr:
|
|
| 101 |
+ # TODO: Upload to CAS or output RAW
|
|
| 102 |
+ # For now, just pass raw
|
|
| 103 |
+ logger.error("Bot error: [{}]".format(stderr))
|
|
| 104 |
+ raise BotError(stderr, detail=stdout, reason="Captured stderr")
|
|
| 93 | 105 |
|
| 94 | 106 |
output_digest = remote_execution_pb2.Digest()
|
| 95 | 107 |
output_digest.ParseFromString(read_file(output_digest_file.name))
|
| 96 | 108 |
|
| 97 | 109 |
logger.debug("Output root digest: {}".format(output_digest))
|
| 98 | 110 |
|
| 99 |
- if len(output_digest.hash) < 64:
|
|
| 100 |
- logger.warning("Buildbox command failed - no output root digest present.")
|
|
| 111 |
+ if len(output_digest.hash) < HASH_LENGTH:
|
|
| 112 |
+ msg = "Buildbox command failed."
|
|
| 113 |
+ raise BotError(msg, detail=stdout, reason="No output root digest present.")
|
|
| 101 | 114 |
|
| 102 | 115 |
# TODO: Have BuildBox helping us creating the Tree instance here
|
| 103 | 116 |
# See https://gitlab.com/BuildStream/buildbox/issues/7 for details
|
| ... | ... | @@ -110,7 +123,6 @@ def work_buildbox(context, lease): |
| 110 | 123 |
output_directory.tree_digest.CopyFrom(output_tree_digest)
|
| 111 | 124 |
output_directory.path = os.path.relpath(working_directory, start='/')
|
| 112 | 125 |
|
| 113 |
- action_result = remote_execution_pb2.ActionResult()
|
|
| 114 | 126 |
action_result.output_directories.extend([output_directory])
|
| 115 | 127 |
|
| 116 | 128 |
action_result_any = any_pb2.Any()
|
| ... | ... | @@ -12,6 +12,9 @@ |
| 12 | 12 |
# See the License for the specific language governing permissions and
|
| 13 | 13 |
# limitations under the License.
|
| 14 | 14 |
|
| 15 |
+# Disable broad exception catch
|
|
| 16 |
+# pylint: disable=broad-except
|
|
| 17 |
+ |
|
| 15 | 18 |
|
| 16 | 19 |
"""
|
| 17 | 20 |
Bot Session
|
| ... | ... | @@ -23,10 +26,14 @@ import asyncio |
| 23 | 26 |
import logging
|
| 24 | 27 |
import platform
|
| 25 | 28 |
import uuid
|
| 26 |
- |
|
| 27 | 29 |
from enum import Enum
|
| 28 | 30 |
|
| 31 |
+import grpc
|
|
| 32 |
+from google.protobuf import any_pb2
|
|
| 33 |
+ |
|
| 29 | 34 |
from buildgrid._protos.google.devtools.remoteworkers.v1test2 import bots_pb2, worker_pb2
|
| 35 |
+from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
|
|
| 36 |
+from buildgrid._exceptions import BotError
|
|
| 30 | 37 |
|
| 31 | 38 |
|
| 32 | 39 |
class BotStatus(Enum):
|
| ... | ... | @@ -142,13 +149,35 @@ class BotSession: |
| 142 | 149 |
|
| 143 | 150 |
async def create_work(self, lease):
|
| 144 | 151 |
self.logger.debug("Work created: [{}]".format(lease.id))
|
| 145 |
- |
|
| 152 |
+ input_lease = lease
|
|
| 146 | 153 |
loop = asyncio.get_event_loop()
|
| 147 |
- lease = await loop.run_in_executor(None, self._work, self._context, lease)
|
|
| 154 |
+ |
|
| 155 |
+ try:
|
|
| 156 |
+ lease = await loop.run_in_executor(None, self._work, self._context, lease)
|
|
| 157 |
+ |
|
| 158 |
+ except BotError as e:
|
|
| 159 |
+ self.logger.error("Bot error thrown: [{}]".format(e))
|
|
| 160 |
+ lease = self._lease_error(input_lease, e)
|
|
| 161 |
+ |
|
| 162 |
+ except grpc.RpcError as e:
|
|
| 163 |
+ self.logger.error("Connection error thrown: [{}]".format(e))
|
|
| 164 |
+ lease = self._lease_error(input_lease, e)
|
|
| 165 |
+ |
|
| 166 |
+ except Exception as e:
|
|
| 167 |
+ self.logger.error("Connection error thrown: [{}]".format(e))
|
|
| 168 |
+ lease = self._lease_error(input_lease, e)
|
|
| 148 | 169 |
|
| 149 | 170 |
self.logger.debug("Work complete: [{}]".format(lease.id))
|
| 150 | 171 |
self.lease_completed(lease)
|
| 151 | 172 |
|
| 173 |
+ def _lease_error(self, lease, error):
|
|
| 174 |
+ action_result = remote_execution_pb2.ActionResult()
|
|
| 175 |
+ action_result.stderr_raw = str(error)
|
|
| 176 |
+ action_result_any = any_pb2.Any()
|
|
| 177 |
+ action_result_any.Pack(action_result)
|
|
| 178 |
+ lease.result.CopyFrom(action_result_any)
|
|
| 179 |
+ return lease
|
|
| 180 |
+ |
|
| 152 | 181 |
|
| 153 | 182 |
class Worker:
|
| 154 | 183 |
def __init__(self, properties=None, configs=None):
|