GNOME.org
 

[Notes] [Git][BuildStream/buildstream][mablanch/668-remote-build-failure] 5 commits: sandbox.py: Stop caching get_virtual_directory if get_directory is used

Title: GitLab

Martin Blanchard pushed to branch mablanch/668-remote-build-failure at BuildStream / buildstream

Commits:

6 changed files:

Changes:

  • buildstream/_protos/google/rpc/code.proto
    1 
    +// Copyright 2017 Google Inc.
    2 
    +//
    3 
    +// Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +// you may not use this file except in compliance with the License.
    5 
    +// You may obtain a copy of the License at
    6 
    +//
    7 
    +//     http://www.apache.org/licenses/LICENSE-2.0
    8 
    +//
    9 
    +// Unless required by applicable law or agreed to in writing, software
    10 
    +// distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +// See the License for the specific language governing permissions and
    13 
    +// limitations under the License.
    14 
    +
    15 
    +syntax = "proto3";
    16 
    +
    17 
    +package google.rpc;
    18 
    +
    19 
    +option go_package = "google.golang.org/genproto/googleapis/rpc/code;code";
    20 
    +option java_multiple_files = true;
    21 
    +option java_outer_classname = "CodeProto";
    22 
    +option java_package = "com.google.rpc";
    23 
    +option objc_class_prefix = "RPC";
    24 
    +
    25 
    +
    26 
    +// The canonical error codes for Google APIs.
    27 
    +//
    28 
    +//
    29 
    +// Sometimes multiple error codes may apply.  Services should return
    30 
    +// the most specific error code that applies.  For example, prefer
    31 
    +// `OUT_OF_RANGE` over `FAILED_PRECONDITION` if both codes apply.
    32 
    +// Similarly prefer `NOT_FOUND` or `ALREADY_EXISTS` over `FAILED_PRECONDITION`.
    33 
    +enum Code {
    34 
    +  // Not an error; returned on success
    35 
    +  //
    36 
    +  // HTTP Mapping: 200 OK
    37 
    +  OK = 0;
    38 
    +
    39 
    +  // The operation was cancelled, typically by the caller.
    40 
    +  //
    41 
    +  // HTTP Mapping: 499 Client Closed Request
    42 
    +  CANCELLED = 1;
    43 
    +
    44 
    +  // Unknown error.  For example, this error may be returned when
    45 
    +  // a `Status` value received from another address space belongs to
    46 
    +  // an error space that is not known in this address space.  Also
    47 
    +  // errors raised by APIs that do not return enough error information
    48 
    +  // may be converted to this error.
    49 
    +  //
    50 
    +  // HTTP Mapping: 500 Internal Server Error
    51 
    +  UNKNOWN = 2;
    52 
    +
    53 
    +  // The client specified an invalid argument.  Note that this differs
    54 
    +  // from `FAILED_PRECONDITION`.  `INVALID_ARGUMENT` indicates arguments
    55 
    +  // that are problematic regardless of the state of the system
    56 
    +  // (e.g., a malformed file name).
    57 
    +  //
    58 
    +  // HTTP Mapping: 400 Bad Request
    59 
    +  INVALID_ARGUMENT = 3;
    60 
    +
    61 
    +  // The deadline expired before the operation could complete. For operations
    62 
    +  // that change the state of the system, this error may be returned
    63 
    +  // even if the operation has completed successfully.  For example, a
    64 
    +  // successful response from a server could have been delayed long
    65 
    +  // enough for the deadline to expire.
    66 
    +  //
    67 
    +  // HTTP Mapping: 504 Gateway Timeout
    68 
    +  DEADLINE_EXCEEDED = 4;
    69 
    +
    70 
    +  // Some requested entity (e.g., file or directory) was not found.
    71 
    +  //
    72 
    +  // Note to server developers: if a request is denied for an entire class
    73 
    +  // of users, such as gradual feature rollout or undocumented whitelist,
    74 
    +  // `NOT_FOUND` may be used. If a request is denied for some users within
    75 
    +  // a class of users, such as user-based access control, `PERMISSION_DENIED`
    76 
    +  // must be used.
    77 
    +  //
    78 
    +  // HTTP Mapping: 404 Not Found
    79 
    +  NOT_FOUND = 5;
    80 
    +
    81 
    +  // The entity that a client attempted to create (e.g., file or directory)
    82 
    +  // already exists.
    83 
    +  //
    84 
    +  // HTTP Mapping: 409 Conflict
    85 
    +  ALREADY_EXISTS = 6;
    86 
    +
    87 
    +  // The caller does not have permission to execute the specified
    88 
    +  // operation. `PERMISSION_DENIED` must not be used for rejections
    89 
    +  // caused by exhausting some resource (use `RESOURCE_EXHAUSTED`
    90 
    +  // instead for those errors). `PERMISSION_DENIED` must not be
    91 
    +  // used if the caller can not be identified (use `UNAUTHENTICATED`
    92 
    +  // instead for those errors). This error code does not imply the
    93 
    +  // request is valid or the requested entity exists or satisfies
    94 
    +  // other pre-conditions.
    95 
    +  //
    96 
    +  // HTTP Mapping: 403 Forbidden
    97 
    +  PERMISSION_DENIED = 7;
    98 
    +
    99 
    +  // The request does not have valid authentication credentials for the
    100 
    +  // operation.
    101 
    +  //
    102 
    +  // HTTP Mapping: 401 Unauthorized
    103 
    +  UNAUTHENTICATED = 16;
    104 
    +
    105 
    +  // Some resource has been exhausted, perhaps a per-user quota, or
    106 
    +  // perhaps the entire file system is out of space.
    107 
    +  //
    108 
    +  // HTTP Mapping: 429 Too Many Requests
    109 
    +  RESOURCE_EXHAUSTED = 8;
    110 
    +
    111 
    +  // The operation was rejected because the system is not in a state
    112 
    +  // required for the operation's execution.  For example, the directory
    113 
    +  // to be deleted is non-empty, an rmdir operation is applied to
    114 
    +  // a non-directory, etc.
    115 
    +  //
    116 
    +  // Service implementors can use the following guidelines to decide
    117 
    +  // between `FAILED_PRECONDITION`, `ABORTED`, and `UNAVAILABLE`:
    118 
    +  //  (a) Use `UNAVAILABLE` if the client can retry just the failing call.
    119 
    +  //  (b) Use `ABORTED` if the client should retry at a higher level
    120 
    +  //      (e.g., when a client-specified test-and-set fails, indicating the
    121 
    +  //      client should restart a read-modify-write sequence).
    122 
    +  //  (c) Use `FAILED_PRECONDITION` if the client should not retry until
    123 
    +  //      the system state has been explicitly fixed.  E.g., if an "rmdir"
    124 
    +  //      fails because the directory is non-empty, `FAILED_PRECONDITION`
    125 
    +  //      should be returned since the client should not retry unless
    126 
    +  //      the files are deleted from the directory.
    127 
    +  //
    128 
    +  // HTTP Mapping: 400 Bad Request
    129 
    +  FAILED_PRECONDITION = 9;
    130 
    +
    131 
    +  // The operation was aborted, typically due to a concurrency issue such as
    132 
    +  // a sequencer check failure or transaction abort.
    133 
    +  //
    134 
    +  // See the guidelines above for deciding between `FAILED_PRECONDITION`,
    135 
    +  // `ABORTED`, and `UNAVAILABLE`.
    136 
    +  //
    137 
    +  // HTTP Mapping: 409 Conflict
    138 
    +  ABORTED = 10;
    139 
    +
    140 
    +  // The operation was attempted past the valid range.  E.g., seeking or
    141 
    +  // reading past end-of-file.
    142 
    +  //
    143 
    +  // Unlike `INVALID_ARGUMENT`, this error indicates a problem that may
    144 
    +  // be fixed if the system state changes. For example, a 32-bit file
    145 
    +  // system will generate `INVALID_ARGUMENT` if asked to read at an
    146 
    +  // offset that is not in the range [0,2^32-1], but it will generate
    147 
    +  // `OUT_OF_RANGE` if asked to read from an offset past the current
    148 
    +  // file size.
    149 
    +  //
    150 
    +  // There is a fair bit of overlap between `FAILED_PRECONDITION` and
    151 
    +  // `OUT_OF_RANGE`.  We recommend using `OUT_OF_RANGE` (the more specific
    152 
    +  // error) when it applies so that callers who are iterating through
    153 
    +  // a space can easily look for an `OUT_OF_RANGE` error to detect when
    154 
    +  // they are done.
    155 
    +  //
    156 
    +  // HTTP Mapping: 400 Bad Request
    157 
    +  OUT_OF_RANGE = 11;
    158 
    +
    159 
    +  // The operation is not implemented or is not supported/enabled in this
    160 
    +  // service.
    161 
    +  //
    162 
    +  // HTTP Mapping: 501 Not Implemented
    163 
    +  UNIMPLEMENTED = 12;
    164 
    +
    165 
    +  // Internal errors.  This means that some invariants expected by the
    166 
    +  // underlying system have been broken.  This error code is reserved
    167 
    +  // for serious errors.
    168 
    +  //
    169 
    +  // HTTP Mapping: 500 Internal Server Error
    170 
    +  INTERNAL = 13;
    171 
    +
    172 
    +  // The service is currently unavailable.  This is most likely a
    173 
    +  // transient condition, which can be corrected by retrying with
    174 
    +  // a backoff.
    175 
    +  //
    176 
    +  // See the guidelines above for deciding between `FAILED_PRECONDITION`,
    177 
    +  // `ABORTED`, and `UNAVAILABLE`.
    178 
    +  //
    179 
    +  // HTTP Mapping: 503 Service Unavailable
    180 
    +  UNAVAILABLE = 14;
    181 
    +
    182 
    +  // Unrecoverable data loss or corruption.
    183 
    +  //
    184 
    +  // HTTP Mapping: 500 Internal Server Error
    185 
    +  DATA_LOSS = 15;
    186 
    +}
    \ No newline at end of file

  • buildstream/_protos/google/rpc/code_pb2.py
    1 
    +# Generated by the protocol buffer compiler.  DO NOT EDIT!
    2 
    +# source: google/rpc/code.proto
    3 
    +
    4 
    +import sys
    5 
    +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
    6 
    +from google.protobuf.internal import enum_type_wrapper
    7 
    +from google.protobuf import descriptor as _descriptor
    8 
    +from google.protobuf import message as _message
    9 
    +from google.protobuf import reflection as _reflection
    10 
    +from google.protobuf import symbol_database as _symbol_database
    11 
    +# @@protoc_insertion_point(imports)
    12 
    +
    13 
    +_sym_db = _symbol_database.Default()
    14 
    +
    15 
    +
    16 
    +
    17 
    +
    18 
    +DESCRIPTOR = _descriptor.FileDescriptor(
    19 
    +  name='google/rpc/code.proto',
    20 
    +  package='google.rpc',
    21 
    +  syntax='proto3',
    22 
    +  serialized_options=_b('\n\016com.google.rpcB\tCodeProtoP\001Z3google.golang.org/genproto/googleapis/rpc/code;code\242\002\003RPC'),
    23 
    +  serialized_pb=_b('\n\x15google/rpc/code.proto\x12\ngoogle.rpc*\xb7\x02\n\x04\x43ode\x12\x06\n\x02OK\x10\x00\x12\r\n\tCANCELLED\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x14\n\x10INVALID_ARGUMENT\x10\x03\x12\x15\n\x11\x44\x45\x41\x44LINE_EXCEEDED\x10\x04\x12\r\n\tNOT_FOUND\x10\x05\x12\x12\n\x0e\x41LREADY_EXISTS\x10\x06\x12\x15\n\x11PERMISSION_DENIED\x10\x07\x12\x13\n\x0fUNAUTHENTICATED\x10\x10\x12\x16\n\x12RESOURCE_EXHAUSTED\x10\x08\x12\x17\n\x13\x46\x41ILED_PRECONDITION\x10\t\x12\x0b\n\x07\x41\x42ORTED\x10\n\x12\x10\n\x0cOUT_OF_RANGE\x10\x0b\x12\x11\n\rUNIMPLEMENTED\x10\x0c\x12\x0c\n\x08INTERNAL\x10\r\x12\x0f\n\x0bUNAVAILABLE\x10\x0e\x12\r\n\tDATA_LOSS\x10\x0f\x42X\n\x0e\x63om.google.rpcB\tCodeProtoP\x01Z3google.golang.org/genproto/googleapis/rpc/code;code\xa2\x02\x03RPCb\x06proto3')
    24 
    +)
    25 
    +
    26 
    +_CODE = _descriptor.EnumDescriptor(
    27 
    +  name='Code',
    28 
    +  full_name='google.rpc.Code',
    29 
    +  filename=None,
    30 
    +  file=DESCRIPTOR,
    31 
    +  values=[
    32 
    +    _descriptor.EnumValueDescriptor(
    33 
    +      name='OK', index=0, number=0,
    34 
    +      serialized_options=None,
    35 
    +      type=None),
    36 
    +    _descriptor.EnumValueDescriptor(
    37 
    +      name='CANCELLED', index=1, number=1,
    38 
    +      serialized_options=None,
    39 
    +      type=None),
    40 
    +    _descriptor.EnumValueDescriptor(
    41 
    +      name='UNKNOWN', index=2, number=2,
    42 
    +      serialized_options=None,
    43 
    +      type=None),
    44 
    +    _descriptor.EnumValueDescriptor(
    45 
    +      name='INVALID_ARGUMENT', index=3, number=3,
    46 
    +      serialized_options=None,
    47 
    +      type=None),
    48 
    +    _descriptor.EnumValueDescriptor(
    49 
    +      name='DEADLINE_EXCEEDED', index=4, number=4,
    50 
    +      serialized_options=None,
    51 
    +      type=None),
    52 
    +    _descriptor.EnumValueDescriptor(
    53 
    +      name='NOT_FOUND', index=5, number=5,
    54 
    +      serialized_options=None,
    55 
    +      type=None),
    56 
    +    _descriptor.EnumValueDescriptor(
    57 
    +      name='ALREADY_EXISTS', index=6, number=6,
    58 
    +      serialized_options=None,
    59 
    +      type=None),
    60 
    +    _descriptor.EnumValueDescriptor(
    61 
    +      name='PERMISSION_DENIED', index=7, number=7,
    62 
    +      serialized_options=None,
    63 
    +      type=None),
    64 
    +    _descriptor.EnumValueDescriptor(
    65 
    +      name='UNAUTHENTICATED', index=8, number=16,
    66 
    +      serialized_options=None,
    67 
    +      type=None),
    68 
    +    _descriptor.EnumValueDescriptor(
    69 
    +      name='RESOURCE_EXHAUSTED', index=9, number=8,
    70 
    +      serialized_options=None,
    71 
    +      type=None),
    72 
    +    _descriptor.EnumValueDescriptor(
    73 
    +      name='FAILED_PRECONDITION', index=10, number=9,
    74 
    +      serialized_options=None,
    75 
    +      type=None),
    76 
    +    _descriptor.EnumValueDescriptor(
    77 
    +      name='ABORTED', index=11, number=10,
    78 
    +      serialized_options=None,
    79 
    +      type=None),
    80 
    +    _descriptor.EnumValueDescriptor(
    81 
    +      name='OUT_OF_RANGE', index=12, number=11,
    82 
    +      serialized_options=None,
    83 
    +      type=None),
    84 
    +    _descriptor.EnumValueDescriptor(
    85 
    +      name='UNIMPLEMENTED', index=13, number=12,
    86 
    +      serialized_options=None,
    87 
    +      type=None),
    88 
    +    _descriptor.EnumValueDescriptor(
    89 
    +      name='INTERNAL', index=14, number=13,
    90 
    +      serialized_options=None,
    91 
    +      type=None),
    92 
    +    _descriptor.EnumValueDescriptor(
    93 
    +      name='UNAVAILABLE', index=15, number=14,
    94 
    +      serialized_options=None,
    95 
    +      type=None),
    96 
    +    _descriptor.EnumValueDescriptor(
    97 
    +      name='DATA_LOSS', index=16, number=15,
    98 
    +      serialized_options=None,
    99 
    +      type=None),
    100 
    +  ],
    101 
    +  containing_type=None,
    102 
    +  serialized_options=None,
    103 
    +  serialized_start=38,
    104 
    +  serialized_end=349,
    105 
    +)
    106 
    +_sym_db.RegisterEnumDescriptor(_CODE)
    107 
    +
    108 
    +Code = enum_type_wrapper.EnumTypeWrapper(_CODE)
    109 
    +OK = 0
    110 
    +CANCELLED = 1
    111 
    +UNKNOWN = 2
    112 
    +INVALID_ARGUMENT = 3
    113 
    +DEADLINE_EXCEEDED = 4
    114 
    +NOT_FOUND = 5
    115 
    +ALREADY_EXISTS = 6
    116 
    +PERMISSION_DENIED = 7
    117 
    +UNAUTHENTICATED = 16
    118 
    +RESOURCE_EXHAUSTED = 8
    119 
    +FAILED_PRECONDITION = 9
    120 
    +ABORTED = 10
    121 
    +OUT_OF_RANGE = 11
    122 
    +UNIMPLEMENTED = 12
    123 
    +INTERNAL = 13
    124 
    +UNAVAILABLE = 14
    125 
    +DATA_LOSS = 15
    126 
    +
    127 
    +
    128 
    +DESCRIPTOR.enum_types_by_name['Code'] = _CODE
    129 
    +_sym_db.RegisterFileDescriptor(DESCRIPTOR)
    130 
    +
    131 
    +
    132 
    +DESCRIPTOR._options = None
    133 
    +# @@protoc_insertion_point(module_scope)

  • buildstream/_protos/google/rpc/code_pb2_grpc.py
    1 
    +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
    2 
    +import grpc
    3 
    +

  • buildstream/element.py
    ... ... @@ -1532,8 +1532,6 @@ class Element(Plugin):
    1532 1532 
                 with _signals.terminator(cleanup_rootdir), \
    1533 1533 
                     self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox:  # nopep8
    1534 1534
    1535 
    -                sandbox_vroot = sandbox.get_virtual_directory()
    1536 
    -
    1537 1535 
                     # By default, the dynamic public data is the same as the static public data.
    1538 1536 
                     # The plugin's assemble() method may modify this, though.
    1539 1537 
                     self.__dynamic_public = _yaml.node_copy(self.__public)
    ... ... @@ -1581,7 +1579,6 @@ class Element(Plugin):
    1581 1579 
                     finally:
    1582 1580 
                         if collect is not None:
    1583 1581 
                             try:
    1584 
    -                            # Sandbox will probably have replaced its virtual directory, so get it again
    1585 1582 
                                 sandbox_vroot = sandbox.get_virtual_directory()
    1586 1583 
                                 collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
    1587 1584 
                             except VirtualDirectoryError:
    ... ... @@ -1606,6 +1603,7 @@ class Element(Plugin):
    1606 1603 
                             collectvdir.export_files(filesdir, can_link=True)
    1607 1604
    1608 1605 
                         try:
    1606 
    +                        sandbox_vroot = sandbox.get_virtual_directory()
    1609 1607 
                             sandbox_build_dir = sandbox_vroot.descend(
    1610 1608 
                                 self.get_variable('build-root').lstrip(os.sep).split(os.sep))
    1611 1609 
                             # Hard link files from build-root dir to buildtreedir directory

  • buildstream/sandbox/_sandboxremote.py
    ... ... @@ -27,6 +27,7 @@ from . import Sandbox
    27 27 
     from ..storage._filebaseddirectory import FileBasedDirectory
    28 28 
     from ..storage._casbaseddirectory import CasBasedDirectory
    29 29 
     from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    30 
    +from .._protos.google.rpc import code_pb2
    30 31 
     from .._platform import Platform
    31 32
    32 33
    ... ... @@ -207,15 +208,23 @@ class SandboxRemote(Sandbox):
    207 208
    208 209 
             operation.response.Unpack(execution_response)
    209 210
    210 
    -        if execution_response.status.code != 0:
    211 
    -            # A normal error during the build: the remote execution system
    212 
    -            # has worked correctly but the command failed.
    213 
    -            # execution_response.error also contains 'message' (str) and
    214 
    -            # 'details' (iterator of Any) which we ignore at the moment.
    215 
    -            return execution_response.status.code
    211 
    +        if execution_response.status.code != code_pb2.OK:
    212 
    +            # An unexpected error during execution: the remote execution
    213 
    +            # system failed at processing the execution request.
    214 
    +            if execution_response.status.message:
    215 
    +                raise SandboxError(execution_response.status.message)
    216 
    +            else:
    217 
    +                raise SandboxError("Remote server failed at executing the build request.")
    216 218
    217 219 
             action_result = execution_response.result
    218 220
    221 
    +        if action_result.exit_code != 0:
    222 
    +            # A normal error during the build: the remote execution system
    223 
    +            # has worked correctly but the command failed.
    224 
    +            # action_result.stdout and action_result.stderr also contains
    225 
    +            # build command outputs which we ignore at the moment.
    226 
    +            return action_result.exit_code
    227 
    +
    219 228 
             self.process_job_output(action_result.output_directories, action_result.output_files)
    220 229
    221 230 
             return 0

  • buildstream/sandbox/sandbox.py
    ... ... @@ -110,6 +110,10 @@ class Sandbox():
    110 110 
                 os.makedirs(directory_, exist_ok=True)
    111 111 
             self._vdir = None
    112 112
    113 
    +        # This is set if anyone requests access to the underlying
    114 
    +        # directory via get_directory.
    115 
    +        self._never_cache_vdirs = False
    116 
    +
    113 117 
         def get_directory(self):
    114 118 
             """Fetches the sandbox root directory
    115 119
    ... ... @@ -122,24 +126,28 @@ class Sandbox():
    122 126
    123 127 
             """
    124 128 
             if self.__allow_real_directory:
    129 
    +            self._never_cache_vdirs = True
    125 130 
                 return self._root
    126 131 
             else:
    127 132 
                 raise BstError("You can't use get_directory")
    128 133
    129 134 
         def get_virtual_directory(self):
    130 
    -        """Fetches the sandbox root directory
    135 
    +        """Fetches the sandbox root directory as a virtual Directory.
    131 136
    132 137 
             The root directory is where artifacts for the base
    133 
    -        runtime environment should be staged. Only works if
    134 
    -        BST_VIRTUAL_DIRECTORY is not set.
    138 
    +        runtime environment should be staged.
    139 
    +
    140 
    +        Use caution if you use get_directory and
    141 
    +        get_virtual_directory.  If you alter the contents of the
    142 
    +        directory returned by get_directory, all objects returned by
    143 
    +        get_virtual_directory or derived from them are invalid and you
    144 
    +        must call get_virtual_directory again to get a new copy.
    135 145
    136 146 
             Returns:
    137 
    -           (str): The sandbox root directory
    147 
    +           (Directory): The sandbox root directory
    138 148
    139 149 
             """
    140 
    -        if not self._vdir:
    141 
    -            # BST_CAS_DIRECTORIES is a deliberately hidden environment variable which
    142 
    -            # can be used to switch on CAS-based directories for testing.
    150 
    +        if self._vdir is None or self._never_cache_vdirs:
    143 151 
                 if 'BST_CAS_DIRECTORIES' in os.environ:
    144 152 
                     self._vdir = CasBasedDirectory(self.__context, ref=None)
    145 153 
                 else:

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]