GNOME.org
 

[Notes] [Git][BuildStream/buildstream][valentindavid/cache_server_fill_up] 25 commits: _project.py: Validate nodes early in Project._load

Title: GitLab

Valentin David pushed to branch valentindavid/cache_server_fill_up at BuildStream / buildstream

Commits:

26 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -79,32 +79,46 @@ source_dist:
    79 79 
       - cd ../..
    80 80 
       - mkdir -p coverage-linux/
    81 81 
       - cp dist/buildstream/.coverage coverage-linux/coverage."${CI_JOB_NAME}"
    82 
    -  except:
    83 
    -  - schedules
    84 82 
       artifacts:
    85 83 
         paths:
    86 84 
         - coverage-linux/
    87 85
    88 86 
     tests-debian-9:
    89 
    -  image: buildstream/testsuite-debian:9-master-119-552f5fc6
    87 
    +  image: buildstream/testsuite-debian:9-master-123-7ce6581b
    90 88 
       <<: *linux-tests
    89 
    +  except:
    90 
    +  - schedules
    91 91
    92 92 
     tests-fedora-27:
    93 
    -  image: buildstream/testsuite-fedora:27-master-119-552f5fc6
    93 
    +  image: buildstream/testsuite-fedora:27-master-123-7ce6581b
    94 94 
       <<: *linux-tests
    95 
    +  except:
    96 
    +  - schedules
    95 97
    96 98 
     tests-fedora-28:
    97 
    -  image: buildstream/testsuite-fedora:28-master-119-552f5fc6
    99 
    +  image: buildstream/testsuite-fedora:28-master-123-7ce6581b
    98 100 
       <<: *linux-tests
    101 
    +  except:
    102 
    +  - schedules
    99 103
    100 104 
     tests-ubuntu-18.04:
    101 
    -  image: buildstream/testsuite-ubuntu:18.04-master-119-552f5fc6
    105 
    +  image: buildstream/testsuite-ubuntu:18.04-master-123-7ce6581b
    102 106 
       <<: *linux-tests
    107 
    +  except:
    108 
    +  - schedules
    109 
    +
    110 
    +overnight-fedora-28-aarch64:
    111 
    +  image: buildstream/testsuite-fedora:aarch64-28-master-123-7ce6581b
    112 
    +  tags:
    113 
    +    - aarch64
    114 
    +  <<: *linux-tests
    115 
    +  only:
    116 
    +  - schedules
    103 117
    104 118 
     tests-unix:
    105 119 
       # Use fedora here, to a) run a test on fedora and b) ensure that we
    106 120 
       # can get rid of ostree - this is not possible with debian-8
    107 
    -  image: buildstream/testsuite-fedora:27-master-119-552f5fc6
    121 
    +  image: buildstream/testsuite-fedora:27-master-123-7ce6581b
    108 122 
       stage: test
    109 123 
       variables:
    110 124 
         BST_FORCE_BACKEND: "unix"

  • buildstream/_artifactcache/cascache.py
    ... ... @@ -25,6 +25,7 @@ import stat
    25 25 
     import tempfile
    26 26 
     import uuid
    27 27 
     import errno
    28 
    +import contextlib
    28 29 
     from urllib.parse import urlparse
    29 30
    30 31 
     import grpc
    ... ... @@ -43,6 +44,13 @@ from .._exceptions import CASError
    43 44 
     _MAX_PAYLOAD_BYTES = 1024 * 1024
    44 45
    45 46
    47 
    +class BlobNotFound(CASError):
    48 
    +
    49 
    +    def __init__(self, blob, msg):
    50 
    +        self.blob = blob
    51 
    +        super().__init__(msg)
    52 
    +
    53 
    +
    46 54 
     # A CASCache manages a CAS repository as specified in the Remote Execution API.
    47 55 
     #
    48 56 
     # Args:
    ... ... @@ -219,6 +227,8 @@ class CASCache():
    219 227 
                     raise CASError("Failed to pull ref {}: {}".format(ref, e)) from e
    220 228 
                 else:
    221 229 
                     return False
    230 
    +        except BlobNotFound as e:
    231 
    +            return False
    222 232
    223 233 
         # pull_tree():
    224 234 
         #
    ... ... @@ -391,13 +401,14 @@ class CASCache():
    391 401 
         #     digest (Digest): An optional Digest object to populate
    392 402 
         #     path (str): Path to file to add
    393 403 
         #     buffer (bytes): Byte buffer to add
    404 
    +    #     link_directly (bool): Whether file given by path can be linked
    394 405 
         #
    395 406 
         # Returns:
    396 407 
         #     (Digest): The digest of the added object
    397 408 
         #
    398 409 
         # Either `path` or `buffer` must be passed, but not both.
    399 410 
         #
    400 
    -    def add_object(self, *, digest=None, path=None, buffer=None):
    411 
    +    def add_object(self, *, digest=None, path=None, buffer=None, link_directly=False):
    401 412 
             # Exactly one of the two parameters has to be specified
    402 413 
             assert (path is None) != (buffer is None)
    403 414
    ... ... @@ -407,28 +418,34 @@ class CASCache():
    407 418 
             try:
    408 419 
                 h = hashlib.sha256()
    409 420 
                 # Always write out new file to avoid corruption if input file is modified
    410 
    -            with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
    411 
    -                # Set mode bits to 0644
    412 
    -                os.chmod(out.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
    413 
    -
    414 
    -                if path:
    415 
    -                    with open(path, 'rb') as f:
    416 
    -                        for chunk in iter(lambda: f.read(4096), b""):
    417 
    -                            h.update(chunk)
    418 
    -                            out.write(chunk)
    421 
    +            with contextlib.ExitStack() as stack:
    422 
    +                if path is not None and link_directly:
    423 
    +                    tmp = stack.enter_context(open(path, 'rb'))
    424 
    +                    for chunk in iter(lambda: tmp.read(4096), b""):
    425 
    +                        h.update(chunk)
    419 426 
                     else:
    420 
    -                    h.update(buffer)
    421 
    -                    out.write(buffer)
    427 
    +                    tmp = stack.enter_context(tempfile.NamedTemporaryFile(dir=self.tmpdir))
    428 
    +                    # Set mode bits to 0644
    429 
    +                    os.chmod(tmp.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
    422 430
    423 
    -                out.flush()
    431 
    +                    if path:
    432 
    +                        with open(path, 'rb') as f:
    433 
    +                            for chunk in iter(lambda: f.read(4096), b""):
    434 
    +                                h.update(chunk)
    435 
    +                                tmp.write(chunk)
    436 
    +                    else:
    437 
    +                        h.update(buffer)
    438 
    +                        tmp.write(buffer)
    439 
    +
    440 
    +                    tmp.flush()
    424 441
    425 442 
                     digest.hash = h.hexdigest()
    426 
    -                digest.size_bytes = os.fstat(out.fileno()).st_size
    443 
    +                digest.size_bytes = os.fstat(tmp.fileno()).st_size
    427 444
    428 445 
                     # Place file at final location
    429 446 
                     objpath = self.objpath(digest)
    430 447 
                     os.makedirs(os.path.dirname(objpath), exist_ok=True)
    431 
    -                os.link(out.name, objpath)
    448 
    +                os.link(tmp.name, objpath)
    432 449
    433 450 
             except FileExistsError as e:
    434 451 
                 # We can ignore the failed link() if the object is already in the repo.
    ... ... @@ -526,6 +543,41 @@ class CASCache():
    526 543 
             # first ref of this list will be the file modified earliest.
    527 544 
             return [ref for _, ref in sorted(zip(mtimes, refs))]
    528 545
    546 
    +    # list_objects():
    547 
    +    #
    548 
    +    # List cached objects in Least Recently Modified (LRM) order.
    549 
    +    #
    550 
    +    # Returns:
    551 
    +    #     (list) - A list of objects and timestamps in LRM order
    552 
    +    #
    553 
    +    def list_objects(self):
    554 
    +        objs = []
    555 
    +        mtimes = []
    556 
    +
    557 
    +        for root, _, files in os.walk(os.path.join(self.casdir, 'objects')):
    558 
    +            for filename in files:
    559 
    +                obj_path = os.path.join(root, filename)
    560 
    +                try:
    561 
    +                    mtimes.append(os.path.getmtime(obj_path))
    562 
    +                except FileNotFoundError:
    563 
    +                    pass
    564 
    +                else:
    565 
    +                    objs.append(obj_path)
    566 
    +
    567 
    +        # NOTE: Sorted will sort from earliest to latest, thus the
    568 
    +        # first element of this list will be the file modified earliest.
    569 
    +        return sorted(zip(mtimes, objs))
    570 
    +
    571 
    +    def clean_up_refs_until(self, time):
    572 
    +        ref_heads = os.path.join(self.casdir, 'refs', 'heads')
    573 
    +
    574 
    +        for root, _, files in os.walk(ref_heads):
    575 
    +            for filename in files:
    576 
    +                ref_path = os.path.join(root, filename)
    577 
    +                # Obtain the mtime (the time a file was last modified)
    578 
    +                if os.path.getmtime(ref_path) < time:
    579 
    +                    os.unlink(ref_path)
    580 
    +
    529 581 
         # remove():
    530 582 
         #
    531 583 
         # Removes the given symbolic ref from the repo.
    ... ... @@ -559,7 +611,12 @@ class CASCache():
    559 611 
         #
    560 612 
         # Prune unreachable objects from the repo.
    561 613 
         #
    562 
    -    def prune(self):
    614 
    +    # Args:
    615 
    +    #    keep_after (int|None): timestamp after which unreachable objects
    616 
    +    #                           are kept. None if no unreachable object
    617 
    +    #                           should be kept.
    618 
    +    #
    619 
    +    def prune(self, keep_after=None):
    563 620 
             ref_heads = os.path.join(self.casdir, 'refs', 'heads')
    564 621
    565 622 
             pruned = 0
    ... ... @@ -580,11 +637,19 @@ class CASCache():
    580 637 
                     objhash = os.path.basename(root) + filename
    581 638 
                     if objhash not in reachable:
    582 639 
                         obj_path = os.path.join(root, filename)
    640 
    +                    if keep_after:
    641 
    +                        st = os.stat(obj_path)
    642 
    +                        if st.st_mtime >= keep_after:
    643 
    +                            continue
    583 644 
                         pruned += os.stat(obj_path).st_size
    584 645 
                         os.unlink(obj_path)
    585 646
    586 647 
             return pruned
    587 648
    649 
    +    def update_tree_mtime(self, tree):
    650 
    +        reachable = set()
    651 
    +        self._reachable_refs_dir(reachable, tree, update_mtime=True)
    652 
    +
    588 653 
         ################################################
    589 654 
         #             Local Private Methods            #
    590 655 
         ################################################
    ... ... @@ -729,10 +794,13 @@ class CASCache():
    729 794 
                     a += 1
    730 795 
                     b += 1
    731 796
    732 
    -    def _reachable_refs_dir(self, reachable, tree):
    797 
    +    def _reachable_refs_dir(self, reachable, tree, update_mtime=False):
    733 798 
             if tree.hash in reachable:
    734 799 
                 return
    735 800
    801 
    +        if update_mtime:
    802 
    +            os.utime(self.objpath(tree))
    803 
    +
    736 804 
             reachable.add(tree.hash)
    737 805
    738 806 
             directory = remote_execution_pb2.Directory()
    ... ... @@ -741,10 +809,12 @@ class CASCache():
    741 809 
                 directory.ParseFromString(f.read())
    742 810
    743 811 
             for filenode in directory.files:
    812 
    +            if update_mtime:
    813 
    +                os.utime(self.objpath(filenode.digest))
    744 814 
                 reachable.add(filenode.digest.hash)
    745 815
    746 816 
             for dirnode in directory.directories:
    747 
    -            self._reachable_refs_dir(reachable, dirnode.digest)
    817 
    +            self._reachable_refs_dir(reachable, dirnode.digest, update_mtime=update_mtime)
    748 818
    749 819 
         def _required_blobs(self, directory_digest):
    750 820 
             # parse directory, and recursively add blobs
    ... ... @@ -798,7 +868,7 @@ class CASCache():
    798 868 
             with tempfile.NamedTemporaryFile(dir=self.tmpdir) as f:
    799 869 
                 self._fetch_blob(remote, digest, f)
    800 870
    801 
    -            added_digest = self.add_object(path=f.name)
    871 
    +            added_digest = self.add_object(path=f.name, link_directly=True)
    802 872 
                 assert added_digest.hash == digest.hash
    803 873
    804 874 
             return objpath
    ... ... @@ -809,7 +879,7 @@ class CASCache():
    809 879 
                     f.write(data)
    810 880 
                     f.flush()
    811 881
    812 
    -                added_digest = self.add_object(path=f.name)
    882 
    +                added_digest = self.add_object(path=f.name, link_directly=True)
    813 883 
                     assert added_digest.hash == digest.hash
    814 884
    815 885 
         # Helper function for _fetch_directory().
    ... ... @@ -1113,6 +1183,9 @@ class _CASBatchRead():
    1113 1183 
             batch_response = self._remote.cas.BatchReadBlobs(self._request)
    1114 1184
    1115 1185 
             for response in batch_response.responses:
    1186 
    +            if response.status.code == code_pb2.NOT_FOUND:
    1187 
    +                raise BlobNotFound(response.digest.hash, "Failed to download blob {}: {}".format(
    1188 
    +                    response.digest.hash, response.status.code))
    1116 1189 
                 if response.status.code != code_pb2.OK:
    1117 1190 
                     raise CASError("Failed to download blob {}: {}".format(
    1118 1191 
                         response.digest.hash, response.status.code))

  • buildstream/_artifactcache/casserver.py
    ... ... @@ -24,6 +24,9 @@ import signal
    24 24 
     import sys
    25 25 
     import tempfile
    26 26 
     import uuid
    27 
    +import errno
    28 
    +import ctypes
    29 
    +import threading
    27 30
    28 31 
     import click
    29 32 
     import grpc
    ... ... @@ -31,6 +34,7 @@ import grpc
    31 34 
     from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    32 35 
     from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
    33 36 
     from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
    37 
    +from .._protos.google.rpc import code_pb2
    34 38
    35 39 
     from .._exceptions import CASError
    36 40
    ... ... @@ -41,6 +45,10 @@ from .cascache import CASCache
    41 45 
     # Limit payload to 1 MiB to leave sufficient headroom for metadata.
    42 46 
     _MAX_PAYLOAD_BYTES = 1024 * 1024
    43 47
    48 
    +# The minimum age in seconds for objects before they can be cleaned
    49 
    +# up.
    50 
    +_OBJECT_MIN_AGE = 6 * 60 * 60
    51 
    +
    44 52
    45 53 
     # Trying to push an artifact that is too large
    46 54 
     class ArtifactTooLargeException(Exception):
    ... ... @@ -55,18 +63,22 @@ class ArtifactTooLargeException(Exception):
    55 63 
     #     repo (str): Path to CAS repository
    56 64 
     #     enable_push (bool): Whether to allow blob uploads and artifact updates
    57 65 
     #
    58 
    -def create_server(repo, *, enable_push):
    66 
    +def create_server(repo, *, enable_push,
    67 
    +                  max_head_size=int(10e9),
    68 
    +                  min_head_size=int(2e9)):
    59 69 
         cas = CASCache(os.path.abspath(repo))
    60 70
    61 71 
         # Use max_workers default from Python 3.5+
    62 72 
         max_workers = (os.cpu_count() or 1) * 5
    63 73 
         server = grpc.server(futures.ThreadPoolExecutor(max_workers))
    64 74
    75 
    +    cache_cleaner = _CacheCleaner(cas, max_head_size, min_head_size)
    76 
    +
    65 77 
         bytestream_pb2_grpc.add_ByteStreamServicer_to_server(
    66 
    -        _ByteStreamServicer(cas, enable_push=enable_push), server)
    78 
    +        _ByteStreamServicer(cas, cache_cleaner, enable_push=enable_push), server)
    67 79
    68 80 
         remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server(
    69 
    -        _ContentAddressableStorageServicer(cas, enable_push=enable_push), server)
    81 
    +        _ContentAddressableStorageServicer(cas, cache_cleaner, enable_push=enable_push), server)
    70 82
    71 83 
         remote_execution_pb2_grpc.add_CapabilitiesServicer_to_server(
    72 84 
             _CapabilitiesServicer(), server)
    ... ... @@ -84,9 +96,19 @@ def create_server(repo, *, enable_push):
    84 96 
     @click.option('--client-certs', help="Public client certificates for TLS (PEM-encoded)")
    85 97 
     @click.option('--enable-push', default=False, is_flag=True,
    86 98 
                   help="Allow clients to upload blobs and update artifact cache")
    99 
    +@click.option('--head-room-min', type=click.INT,
    100 
    +              help="Disk head room minimum in bytes",
    101 
    +              default=2e9)
    102 
    +@click.option('--head-room-max', type=click.INT,
    103 
    +              help="Disk head room maximum in bytes",
    104 
    +              default=10e9)
    87 105 
     @click.argument('repo')
    88 
    -def server_main(repo, port, server_key, server_cert, client_certs, enable_push):
    89 
    -    server = create_server(repo, enable_push=enable_push)
    106 
    +def server_main(repo, port, server_key, server_cert, client_certs, enable_push,
    107 
    +                head_room_min, head_room_max):
    108 
    +    server = create_server(repo,
    109 
    +                           max_head_size=head_room_max,
    110 
    +                           min_head_size=head_room_min,
    111 
    +                           enable_push=enable_push)
    90 112
    91 113 
         use_tls = bool(server_key)
    92 114
    ... ... @@ -127,11 +149,43 @@ def server_main(repo, port, server_key, server_cert, client_certs, enable_push):
    127 149 
             server.stop(0)
    128 150
    129 151
    152 
    +class _FallocateCall:
    153 
    +
    154 
    +    FALLOC_FL_KEEP_SIZE = 1
    155 
    +    FALLOC_FL_PUNCH_HOLE = 2
    156 
    +    FALLOC_FL_NO_HIDE_STALE = 4
    157 
    +    FALLOC_FL_COLLAPSE_RANGE = 8
    158 
    +    FALLOC_FL_ZERO_RANGE = 16
    159 
    +    FALLOC_FL_INSERT_RANGE = 32
    160 
    +    FALLOC_FL_UNSHARE_RANGE = 64
    161 
    +
    162 
    +    def __init__(self):
    163 
    +        self.libc = ctypes.CDLL("libc.so.6", use_errno=True)
    164 
    +        try:
    165 
    +            self.fallocate64 = self.libc.fallocate64
    166 
    +        except AttributeError:
    167 
    +            self.fallocate = self.libc.fallocate
    168 
    +
    169 
    +    def __call__(self, fd, mode, offset, length):
    170 
    +        if hasattr(self, 'fallocate64'):
    171 
    +            ret = self.fallocate64(ctypes.c_int(fd), ctypes.c_int(mode),
    172 
    +                                   ctypes.c_int64(offset), ctypes.c_int64(length))
    173 
    +        else:
    174 
    +            ret = self.fallocate(ctypes.c_int(fd), ctypes.c_int(mode),
    175 
    +                                 ctypes.c_int(offset), ctypes.c_int(length))
    176 
    +        if ret == -1:
    177 
    +            err = ctypes.get_errno()
    178 
    +            raise OSError(errno, os.strerror(err))
    179 
    +        return ret
    180 
    +
    181 
    +
    130 182 
     class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
    131 
    -    def __init__(self, cas, *, enable_push):
    183 
    +    def __init__(self, cas, cache_cleaner, *, enable_push):
    132 184 
             super().__init__()
    133 185 
             self.cas = cas
    134 186 
             self.enable_push = enable_push
    187 
    +        self.fallocate = _FallocateCall()
    188 
    +        self.cache_cleaner = cache_cleaner
    135 189
    136 190 
         def Read(self, request, context):
    137 191 
             resource_name = request.resource_name
    ... ... @@ -189,25 +243,44 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
    189 243 
                             context.set_code(grpc.StatusCode.NOT_FOUND)
    190 244 
                             return response
    191 245
    192 
    -                    try:
    193 
    -                        _clean_up_cache(self.cas, client_digest.size_bytes)
    194 
    -                    except ArtifactTooLargeException as e:
    195 
    -                        context.set_code(grpc.StatusCode.RESOURCE_EXHAUSTED)
    196 
    -                        context.set_details(str(e))
    197 
    -                        return response
    246 
    +                    while True:
    247 
    +                        if client_digest.size_bytes == 0:
    248 
    +                            break
    249 
    +                        try:
    250 
    +                            self.cache_cleaner.clean_up(client_digest.size_bytes)
    251 
    +                        except ArtifactTooLargeException as e:
    252 
    +                            context.set_code(grpc.StatusCode.RESOURCE_EXHAUSTED)
    253 
    +                            context.set_details(str(e))
    254 
    +                            return response
    255 
    +
    256 
    +                        try:
    257 
    +                            self.fallocate(out.fileno(), 0, 0, client_digest.size_bytes)
    258 
    +                            break
    259 
    +                        except OSError as e:
    260 
    +                            # Multiple upload can happen in the same time
    261 
    +                            if e.errno != errno.ENOSPC:
    262 
    +                                raise
    263 
    +
    198 264 
                     elif request.resource_name:
    199 265 
                         # If it is set on subsequent calls, it **must** match the value of the first request.
    200 266 
                         if request.resource_name != resource_name:
    201 267 
                             context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
    202 268 
                             return response
    269 
    +
    270 
    +                if (offset + len(request.data)) > client_digest.size_bytes:
    271 
    +                    context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
    272 
    +                    return response
    273 
    +
    203 274 
                     out.write(request.data)
    275 
    +
    204 276 
                     offset += len(request.data)
    277 
    +
    205 278 
                     if request.finish_write:
    206 279 
                         if client_digest.size_bytes != offset:
    207 280 
                             context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
    208 281 
                             return response
    209 282 
                         out.flush()
    210 
    -                    digest = self.cas.add_object(path=out.name)
    283 
    +                    digest = self.cas.add_object(path=out.name, link_directly=True)
    211 284 
                         if digest.hash != client_digest.hash:
    212 285 
                             context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
    213 286 
                             return response
    ... ... @@ -220,18 +293,26 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
    220 293
    221 294
    222 295 
     class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddressableStorageServicer):
    223 
    -    def __init__(self, cas, *, enable_push):
    296 
    +    def __init__(self, cas, cache_cleaner, *, enable_push):
    224 297 
             super().__init__()
    225 298 
             self.cas = cas
    226 299 
             self.enable_push = enable_push
    300 
    +        self.cache_cleaner = cache_cleaner
    227 301
    228 302 
         def FindMissingBlobs(self, request, context):
    229 303 
             response = remote_execution_pb2.FindMissingBlobsResponse()
    230 304 
             for digest in request.blob_digests:
    231 
    -            if not _has_object(self.cas, digest):
    232 
    -                d = response.missing_blob_digests.add()
    233 
    -                d.hash = digest.hash
    234 
    -                d.size_bytes = digest.size_bytes
    305 
    +            objpath = self.cas.objpath(digest)
    306 
    +            try:
    307 
    +                os.utime(objpath)
    308 
    +            except OSError as e:
    309 
    +                if e.errno != errno.ENOENT:
    310 
    +                    raise
    311 
    +                else:
    312 
    +                    d = response.missing_blob_digests.add()
    313 
    +                    d.hash = digest.hash
    314 
    +                    d.size_bytes = digest.size_bytes
    315 
    +
    235 316 
             return response
    236 317
    237 318 
         def BatchReadBlobs(self, request, context):
    ... ... @@ -250,12 +331,12 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
    250 331 
                 try:
    251 332 
                     with open(self.cas.objpath(digest), 'rb') as f:
    252 333 
                         if os.fstat(f.fileno()).st_size != digest.size_bytes:
    253 
    -                        blob_response.status.code = grpc.StatusCode.NOT_FOUND
    334 
    +                        blob_response.status.code = code_pb2.NOT_FOUND
    254 335 
                             continue
    255 336
    256 337 
                         blob_response.data = f.read(digest.size_bytes)
    257 338 
                 except FileNotFoundError:
    258 
    -                blob_response.status.code = grpc.StatusCode.NOT_FOUND
    339 
    +                blob_response.status.code = code_pb2.NOT_FOUND
    259 340
    260 341 
             return response
    261 342
    ... ... @@ -285,7 +366,7 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
    285 366 
                     continue
    286 367
    287 368 
                 try:
    288 
    -                _clean_up_cache(self.cas, digest.size_bytes)
    369 
    +                self.cache_cleaner.clean_up(digest.size_bytes)
    289 370
    290 371 
                     with tempfile.NamedTemporaryFile(dir=self.cas.tmpdir) as out:
    291 372 
                         out.write(blob_request.data)
    ... ... @@ -328,6 +409,12 @@ class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
    328 409
    329 410 
             try:
    330 411 
                 tree = self.cas.resolve_ref(request.key, update_mtime=True)
    412 
    +            try:
    413 
    +                self.cas.update_tree_mtime(tree)
    414 
    +            except FileNotFoundError:
    415 
    +                self.cas.remove(request.key, defer_prune=True)
    416 
    +                context.set_code(grpc.StatusCode.NOT_FOUND)
    417 
    +                return response
    331 418
    332 419 
                 response.digest.hash = tree.hash
    333 420 
                 response.digest.size_bytes = tree.size_bytes
    ... ... @@ -400,60 +487,80 @@ def _digest_from_upload_resource_name(resource_name):
    400 487 
             return None
    401 488
    402 489
    403 
    -def _has_object(cas, digest):
    404 
    -    objpath = cas.objpath(digest)
    405 
    -    return os.path.exists(objpath)
    490 
    +class _CacheCleaner:
    406 491
    492 
    +    __cleanup_cache_lock = threading.Lock()
    407 493
    408 
    -# _clean_up_cache()
    409 
    -#
    410 
    -# Keep removing Least Recently Pushed (LRP) artifacts in a cache until there
    411 
    -# is enough space for the incoming artifact
    412 
    -#
    413 
    -# Args:
    414 
    -#   cas: CASCache object
    415 
    -#   object_size: The size of the object being received in bytes
    416 
    -#
    417 
    -# Returns:
    418 
    -#   int: The total bytes removed on the filesystem
    419 
    -#
    420 
    -def _clean_up_cache(cas, object_size):
    421 
    -    # Determine the available disk space, in bytes, of the file system
    422 
    -    # which mounts the repo
    423 
    -    stats = os.statvfs(cas.casdir)
    424 
    -    buffer_ = int(2e9)                # Add a 2 GB buffer
    425 
    -    free_disk_space = (stats.f_bfree * stats.f_bsize) - buffer_
    426 
    -    total_disk_space = (stats.f_blocks * stats.f_bsize) - buffer_
    427 
    -
    428 
    -    if object_size > total_disk_space:
    429 
    -        raise ArtifactTooLargeException("Artifact of size: {} is too large for "
    430 
    -                                        "the filesystem which mounts the remote "
    431 
    -                                        "cache".format(object_size))
    432 
    -
    433 
    -    if object_size <= free_disk_space:
    434 
    -        # No need to clean up
    435 
    -        return 0
    436 
    -
    437 
    -    # obtain a list of LRP artifacts
    438 
    -    LRP_artifacts = cas.list_refs()
    439 
    -
    440 
    -    removed_size = 0  # in bytes
    441 
    -    while object_size - removed_size > free_disk_space:
    442 
    -        try:
    443 
    -            to_remove = LRP_artifacts.pop(0)  # The first element in the list is the LRP artifact
    444 
    -        except IndexError:
    445 
    -            # This exception is caught if there are no more artifacts in the list
    446 
    -            # LRP_artifacts. This means the the artifact is too large for the filesystem
    447 
    -            # so we abort the process
    448 
    -            raise ArtifactTooLargeException("Artifact of size {} is too large for "
    449 
    -                                            "the filesystem which mounts the remote "
    450 
    -                                            "cache".format(object_size))
    494 
    +    def __init__(self, cas, max_head_size, min_head_size=int(2e9)):
    495 
    +        self.__cas = cas
    496 
    +        self.__max_head_size = max_head_size
    497 
    +        self.__min_head_size = min_head_size
    451 498
    452 
    -        removed_size += cas.remove(to_remove, defer_prune=False)
    499 
    +    def __has_space(self, object_size):
    500 
    +        stats = os.statvfs(self.__cas.casdir)
    501 
    +        free_disk_space = (stats.f_bavail * stats.f_bsize) - self.__min_head_size
    502 
    +        total_disk_space = (stats.f_blocks * stats.f_bsize) - self.__min_head_size
    453 503
    454 
    -    if removed_size > 0:
    455 
    -        logging.info("Successfully removed {} bytes from the cache".format(removed_size))
    456 
    -    else:
    457 
    -        logging.info("No artifacts were removed from the cache.")
    504 
    +        if object_size > total_disk_space:
    505 
    +            raise ArtifactTooLargeException("Artifact of size: {} is too large for "
    506 
    +                                            "the filesystem which mounts the remote "
    507 
    +                                            "cache".format(object_size))
    458 508
    459 
    -    return removed_size
    509 
    +        return object_size <= free_disk_space
    510 
    +
    511 
    +    # _clean_up_cache()
    512 
    +    #
    513 
    +    # Keep removing Least Recently Pushed (LRP) artifacts in a cache until there
    514 
    +    # is enough space for the incoming artifact
    515 
    +    #
    516 
    +    # Args:
    517 
    +    #   object_size: The size of the object being received in bytes
    518 
    +    #
    519 
    +    # Returns:
    520 
    +    #   int: The total bytes removed on the filesystem
    521 
    +    #
    522 
    +    def clean_up(self, object_size):
    523 
    +        if self.__has_space(object_size):
    524 
    +            return 0
    525 
    +
    526 
    +        with _CacheCleaner.__cleanup_cache_lock:
    527 
    +            if self.__has_space(object_size):
    528 
    +                # Another thread has done the cleanup for us
    529 
    +                return 0
    530 
    +
    531 
    +            stats = os.statvfs(self.__cas.casdir)
    532 
    +            target_disk_space = (stats.f_bavail * stats.f_bsize) - self.__max_head_size
    533 
    +
    534 
    +            # obtain a list of LRP artifacts
    535 
    +            LRP_objects = self.__cas.list_objects()
    536 
    +
    537 
    +            removed_size = 0  # in bytes
    538 
    +
    539 
    +            last_mtime = 0
    540 
    +
    541 
    +            while object_size - removed_size > target_disk_space:
    542 
    +                try:
    543 
    +                    last_mtime, to_remove = LRP_objects.pop(0)  # The first element in the list is the LRP artifact
    544 
    +                except IndexError:
    545 
    +                    # This exception is caught if there are no more artifacts in the list
    546 
    +                    # LRP_artifacts. This means the the artifact is too large for the filesystem
    547 
    +                    # so we abort the process
    548 
    +                    raise ArtifactTooLargeException("Artifact of size {} is too large for "
    549 
    +                                                    "the filesystem which mounts the remote "
    550 
    +                                                    "cache".format(object_size))
    551 
    +
    552 
    +                try:
    553 
    +                    size = os.stat(to_remove).st_size
    554 
    +                    os.unlink(to_remove)
    555 
    +                    removed_size += size
    556 
    +                except FileNotFoundError:
    557 
    +                    pass
    558 
    +
    559 
    +            self.__cas.clean_up_refs_until(last_mtime)
    560 
    +
    561 
    +            if removed_size > 0:
    562 
    +                logging.info("Successfully removed {} bytes from the cache".format(removed_size))
    563 
    +            else:
    564 
    +                logging.info("No artifacts were removed from the cache.")
    565 
    +
    566 
    +            return removed_size

  • buildstream/_platform/linux.py
    ... ... @@ -18,9 +18,9 @@
    18 18 
     #        Tristan Maat <tristan maat codethink co uk>
    19 19
    20 20 
     import os
    21 
    -import shutil
    22 21 
     import subprocess
    23 22
    23 
    +from .. import _site
    24 24 
     from .. import utils
    25 25 
     from ..sandbox import SandboxDummy
    26 26
    ... ... @@ -38,16 +38,18 @@ class Linux(Platform):
    38 38
    39 39 
             self._have_fuse = os.path.exists("/dev/fuse")
    40 40
    41 
    -        bwrap_version = self._get_bwrap_version()
    41 
    +        bwrap_version = _site.get_bwrap_version()
    42 42
    43 43 
             if bwrap_version is None:
    44 44 
                 self._bwrap_exists = False
    45 45 
                 self._have_good_bwrap = False
    46 46 
                 self._die_with_parent_available = False
    47 
    +            self._json_status_available = False
    47 48 
             else:
    48 49 
                 self._bwrap_exists = True
    49 50 
                 self._have_good_bwrap = (0, 1, 2) <= bwrap_version
    50 51 
                 self._die_with_parent_available = (0, 1, 8) <= bwrap_version
    52 
    +            self._json_status_available = (0, 3, 2) <= bwrap_version
    51 53
    52 54 
             self._local_sandbox_available = self._have_fuse and self._have_good_bwrap
    53 55
    ... ... @@ -97,6 +99,7 @@ class Linux(Platform):
    97 99 
             # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
    98 100 
             kwargs['user_ns_available'] = self._user_ns_available
    99 101 
             kwargs['die_with_parent_available'] = self._die_with_parent_available
    102 
    +        kwargs['json_status_available'] = self._json_status_available
    100 103 
             return SandboxBwrap(*args, **kwargs)
    101 104
    102 105 
         def _check_user_ns_available(self):
    ... ... @@ -119,21 +122,3 @@ class Linux(Platform):
    119 122 
                 output = ''
    120 123
    121 124 
             return output == 'root'
    122 
    -
    123 
    -    def _get_bwrap_version(self):
    124 
    -        # Get the current bwrap version
    125 
    -        #
    126 
    -        # returns None if no bwrap was found
    127 
    -        # otherwise returns a tuple of 3 int: major, minor, patch
    128 
    -        bwrap_path = shutil.which('bwrap')
    129 
    -
    130 
    -        if not bwrap_path:
    131 
    -            return None
    132 
    -
    133 
    -        cmd = [bwrap_path, "--version"]
    134 
    -        try:
    135 
    -            version = str(subprocess.check_output(cmd).split()[1], "utf-8")
    136 
    -        except subprocess.CalledProcessError:
    137 
    -            return None
    138 
    -
    139 
    -        return tuple(int(x) for x in version.split("."))

  • buildstream/_project.py
    ... ... @@ -219,6 +219,19 @@ class Project():
    219 219
    220 220 
             return self._cache_key
    221 221
    222 
    +    def _validate_node(self, node):
    223 
    +        _yaml.node_validate(node, [
    224 
    +            'format-version',
    225 
    +            'element-path', 'variables',
    226 
    +            'environment', 'environment-nocache',
    227 
    +            'split-rules', 'elements', 'plugins',
    228 
    +            'aliases', 'name',
    229 
    +            'artifacts', 'options',
    230 
    +            'fail-on-overlap', 'shell', 'fatal-warnings',
    231 
    +            'ref-storage', 'sandbox', 'mirrors', 'remote-execution',
    232 
    +            'sources', '(@)'
    233 
    +        ])
    234 
    +
    222 235 
         # create_element()
    223 236 
         #
    224 237 
         # Instantiate and return an element
    ... ... @@ -402,6 +415,8 @@ class Project():
    402 415 
                     "Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
    403 416 
                     .format(format_version, major, minor, BST_FORMAT_VERSION))
    404 417
    418 
    +        self._validate_node(pre_config_node)
    419 
    +
    405 420 
             # FIXME:
    406 421 
             #
    407 422 
             #   Performing this check manually in the absense
    ... ... @@ -467,16 +482,7 @@ class Project():
    467 482
    468 483 
             self._load_pass(config, self.config)
    469 484
    470 
    -        _yaml.node_validate(config, [
    471 
    -            'format-version',
    472 
    -            'element-path', 'variables',
    473 
    -            'environment', 'environment-nocache',
    474 
    -            'split-rules', 'elements', 'plugins',
    475 
    -            'aliases', 'name',
    476 
    -            'artifacts', 'options',
    477 
    -            'fail-on-overlap', 'shell', 'fatal-warnings',
    478 
    -            'ref-storage', 'sandbox', 'mirrors', 'remote-execution'
    479 
    -        ])
    485 
    +        self._validate_node(config)
    480 486
    481 487 
             #
    482 488 
             # Now all YAML composition is done, from here on we just load

  • buildstream/_site.py
    ... ... @@ -18,6 +18,8 @@
    18 18 
     #        Tristan Van Berkom <tristan vanberkom codethink co uk>
    19 19
    20 20 
     import os
    21 
    +import shutil
    22 
    +import subprocess
    21 23
    22 24 
     #
    23 25 
     # Private module declaring some info about where the buildstream
    ... ... @@ -44,3 +46,22 @@ build_all_template = os.path.join(root, 'data', 'build-all.sh.in')
    44 46
    45 47 
     # Module building script template
    46 48 
     build_module_template = os.path.join(root, 'data', 'build-module.sh.in')
    49 
    +
    50 
    +
    51 
    +def get_bwrap_version():
    52 
    +    # Get the current bwrap version
    53 
    +    #
    54 
    +    # returns None if no bwrap was found
    55 
    +    # otherwise returns a tuple of 3 int: major, minor, patch
    56 
    +    bwrap_path = shutil.which('bwrap')
    57 
    +
    58 
    +    if not bwrap_path:
    59 
    +        return None
    60 
    +
    61 
    +    cmd = [bwrap_path, "--version"]
    62 
    +    try:
    63 
    +        version = str(subprocess.check_output(cmd).split()[1], "utf-8")
    64 
    +    except subprocess.CalledProcessError:
    65 
    +        return None
    66 
    +
    67 
    +    return tuple(int(x) for x in version.split("."))

  • buildstream/sandbox/_sandboxbwrap.py
    ... ... @@ -17,6 +17,8 @@
    17 17 
     #  Authors:
    18 18 
     #        Andrew Leeming <andrew leeming codethink co uk>
    19 19 
     #        Tristan Van Berkom <tristan vanberkom codethink co uk>
    20 
    +import collections
    21 
    +import json
    20 22 
     import os
    21 23 
     import sys
    22 24 
     import time
    ... ... @@ -24,7 +26,8 @@ import errno
    24 26 
     import signal
    25 27 
     import subprocess
    26 28 
     import shutil
    27 
    -from contextlib import ExitStack
    29 
    +from contextlib import ExitStack, suppress
    30 
    +from tempfile import TemporaryFile
    28 31
    29 32 
     import psutil
    30 33
    ... ... @@ -53,6 +56,7 @@ class SandboxBwrap(Sandbox):
    53 56 
             super().__init__(*args, **kwargs)
    54 57 
             self.user_ns_available = kwargs['user_ns_available']
    55 58 
             self.die_with_parent_available = kwargs['die_with_parent_available']
    59 
    +        self.json_status_available = kwargs['json_status_available']
    56 60
    57 61 
         def run(self, command, flags, *, cwd=None, env=None):
    58 62 
             stdout, stderr = self._get_output()
    ... ... @@ -160,24 +164,31 @@ class SandboxBwrap(Sandbox):
    160 164 
                     gid = self._get_config().build_gid
    161 165 
                     bwrap_command += ['--uid', str(uid), '--gid', str(gid)]
    162 166
    163 
    -        # Add the command
    164 
    -        bwrap_command += command
    165 
    -
    166 
    -        # bwrap might create some directories while being suid
    167 
    -        # and may give them to root gid, if it does, we'll want
    168 
    -        # to clean them up after, so record what we already had
    169 
    -        # there just in case so that we can safely cleanup the debris.
    170 
    -        #
    171 
    -        existing_basedirs = {
    172 
    -            directory: os.path.exists(os.path.join(root_directory, directory))
    173 
    -            for directory in ['tmp', 'dev', 'proc']
    174 
    -        }
    175 
    -
    176 
    -        # Use the MountMap context manager to ensure that any redirected
    177 
    -        # mounts through fuse layers are in context and ready for bwrap
    178 
    -        # to mount them from.
    179 
    -        #
    180 167 
             with ExitStack() as stack:
    168 
    +            pass_fds = ()
    169 
    +            # Improve error reporting with json-status if available
    170 
    +            if self.json_status_available:
    171 
    +                json_status_file = stack.enter_context(TemporaryFile())
    172 
    +                pass_fds = (json_status_file.fileno(),)
    173 
    +                bwrap_command += ['--json-status-fd', str(json_status_file.fileno())]
    174 
    +
    175 
    +            # Add the command
    176 
    +            bwrap_command += command
    177 
    +
    178 
    +            # bwrap might create some directories while being suid
    179 
    +            # and may give them to root gid, if it does, we'll want
    180 
    +            # to clean them up after, so record what we already had
    181 
    +            # there just in case so that we can safely cleanup the debris.
    182 
    +            #
    183 
    +            existing_basedirs = {
    184 
    +                directory: os.path.exists(os.path.join(root_directory, directory))
    185 
    +                for directory in ['tmp', 'dev', 'proc']
    186 
    +            }
    187 
    +
    188 
    +            # Use the MountMap context manager to ensure that any redirected
    189 
    +            # mounts through fuse layers are in context and ready for bwrap
    190 
    +            # to mount them from.
    191 
    +            #
    181 192 
                 stack.enter_context(mount_map.mounted(self))
    182 193
    183 194 
                 # If we're interactive, we want to inherit our stdin,
    ... ... @@ -190,7 +201,7 @@ class SandboxBwrap(Sandbox):
    190 201
    191 202 
                 # Run bubblewrap !
    192 203 
                 exit_code = self.run_bwrap(bwrap_command, stdin, stdout, stderr,
    193 
    -                                       (flags & SandboxFlags.INTERACTIVE))
    204 
    +                                       (flags & SandboxFlags.INTERACTIVE), pass_fds)
    194 205
    195 206 
                 # Cleanup things which bwrap might have left behind, while
    196 207 
                 # everything is still mounted because bwrap can be creating
    ... ... @@ -238,10 +249,27 @@ class SandboxBwrap(Sandbox):
    238 249 
                             # a bug, bwrap mounted a tempfs here and when it exits, that better be empty.
    239 250 
                             pass
    240 251
    252 
    +            if self.json_status_available:
    253 
    +                json_status_file.seek(0, 0)
    254 
    +                child_exit_code = None
    255 
    +                # The JSON status file's output is a JSON object per line
    256 
    +                # with the keys present identifying the type of message.
    257 
    +                # The only message relevant to us now is the exit-code of the subprocess.
    258 
    +                for line in json_status_file:
    259 
    +                    with suppress(json.decoder.JSONDecodeError):
    260 
    +                        o = json.loads(line)
    261 
    +                        if isinstance(o, collections.abc.Mapping) and 'exit-code' in o:
    262 
    +                            child_exit_code = o['exit-code']
    263 
    +                            break
    264 
    +                if child_exit_code is None:
    265 
    +                    raise SandboxError("`bwrap' terminated during sandbox setup with exitcode {}".format(exit_code),
    266 
    +                                       reason="bwrap-sandbox-fail")
    267 
    +                exit_code = child_exit_code
    268 
    +
    241 269 
             self._vdir._mark_changed()
    242 270 
             return exit_code
    243 271
    244 
    -    def run_bwrap(self, argv, stdin, stdout, stderr, interactive):
    272 
    +    def run_bwrap(self, argv, stdin, stdout, stderr, interactive, pass_fds):
    245 273 
             # Wrapper around subprocess.Popen() with common settings.
    246 274 
             #
    247 275 
             # This function blocks until the subprocess has terminated.
    ... ... @@ -317,6 +345,7 @@ class SandboxBwrap(Sandbox):
    317 345 
                     # The default is to share file descriptors from the parent process
    318 346 
                     # to the subprocess, which is rarely good for sandboxing.
    319 347 
                     close_fds=True,
    348 
    +                pass_fds=pass_fds,
    320 349 
                     stdin=stdin,
    321 350 
                     stdout=stdout,
    322 351 
                     stderr=stderr,

  • tests/cachekey/cachekey.py
    ... ... @@ -36,7 +36,7 @@
    36 36 
     # the result.
    37 37 
     #
    38 38 
     from tests.testutils.runcli import cli
    39 
    -from tests.testutils.site import HAVE_BZR, HAVE_GIT, HAVE_OSTREE, IS_LINUX
    39 
    +from tests.testutils.site import HAVE_BZR, HAVE_GIT, HAVE_OSTREE, IS_LINUX, MACHINE_ARCH
    40 40 
     from buildstream.plugin import CoreWarnings
    41 41 
     from buildstream import _yaml
    42 42 
     import os
    ... ... @@ -144,6 +144,8 @@ DATA_DIR = os.path.join(
    144 144 
     # The cache key test uses a project which exercises all plugins,
    145 145 
     # so we cant run it at all if we dont have them installed.
    146 146 
     #
    147 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    148 
    +                    reason='Cache keys depend on architecture')
    147 149 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    148 150 
     @pytest.mark.skipif(HAVE_BZR is False, reason="bzr is not available")
    149 151 
     @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")

  • tests/examples/autotools.py
    ... ... @@ -3,7 +3,7 @@ import pytest
    3 3
    4 4 
     from tests.testutils import cli_integration as cli
    5 5 
     from tests.testutils.integration import assert_contains
    6 
    -from tests.testutils.site import IS_LINUX
    6 
    +from tests.testutils.site import IS_LINUX, MACHINE_ARCH
    7 7
    8 8 
     pytestmark = pytest.mark.integration
    9 9
    ... ... @@ -13,6 +13,8 @@ DATA_DIR = os.path.join(
    13 13
    14 14
    15 15 
     # Tests a build of the autotools amhello project on a alpine-linux base runtime
    16 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    17 
    +                    reason='Examples are writtent for x86_64')
    16 18 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    17 19 
     @pytest.mark.datafiles(DATA_DIR)
    18 20 
     def test_autotools_build(cli, tmpdir, datafiles):
    ... ... @@ -36,6 +38,8 @@ def test_autotools_build(cli, tmpdir, datafiles):
    36 38
    37 39
    38 40 
     # Test running an executable built with autotools.
    41 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    42 
    +                    reason='Examples are writtent for x86_64')
    39 43 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    40 44 
     @pytest.mark.datafiles(DATA_DIR)
    41 45 
     def test_autotools_run(cli, tmpdir, datafiles):

  • tests/examples/developing.py
    ... ... @@ -4,7 +4,7 @@ import pytest
    4 4 
     import tests.testutils.patch as patch
    5 5 
     from tests.testutils import cli_integration as cli
    6 6 
     from tests.testutils.integration import assert_contains
    7 
    -from tests.testutils.site import IS_LINUX
    7 
    +from tests.testutils.site import IS_LINUX, MACHINE_ARCH
    8 8
    9 9 
     pytestmark = pytest.mark.integration
    10 10
    ... ... @@ -14,6 +14,8 @@ DATA_DIR = os.path.join(
    14 14
    15 15
    16 16 
     # Test that the project builds successfully
    17 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    18 
    +                    reason='Examples are writtent for x86_64')
    17 19 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    18 20 
     @pytest.mark.datafiles(DATA_DIR)
    19 21 
     def test_autotools_build(cli, tmpdir, datafiles):
    ... ... @@ -35,6 +37,8 @@ def test_autotools_build(cli, tmpdir, datafiles):
    35 37
    36 38
    37 39 
     # Test the unmodified hello command works as expected.
    40 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    41 
    +                    reason='Examples are writtent for x86_64')
    38 42 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    39 43 
     @pytest.mark.datafiles(DATA_DIR)
    40 44 
     def test_run_unmodified_hello(cli, tmpdir, datafiles):
    ... ... @@ -66,6 +70,8 @@ def test_open_workspace(cli, tmpdir, datafiles):
    66 70
    67 71
    68 72 
     # Test making a change using the workspace
    73 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    74 
    +                    reason='Examples are writtent for x86_64')
    69 75 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    70 76 
     @pytest.mark.datafiles(DATA_DIR)
    71 77 
     def test_make_change_in_workspace(cli, tmpdir, datafiles):

  • tests/examples/flatpak-autotools.py
    ... ... @@ -3,7 +3,7 @@ import pytest
    3 3
    4 4 
     from tests.testutils import cli_integration as cli
    5 5 
     from tests.testutils.integration import assert_contains
    6 
    -from tests.testutils.site import IS_LINUX
    6 
    +from tests.testutils.site import IS_LINUX, MACHINE_ARCH
    7 7
    8 8
    9 9 
     pytestmark = pytest.mark.integration
    ... ... @@ -32,6 +32,8 @@ def workaround_setuptools_bug(project):
    32 32
    33 33 
     # Test that a build upon flatpak runtime 'works' - we use the autotools sample
    34 34 
     # amhello project for this.
    35 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    36 
    +                    reason='Examples are writtent for x86_64')
    35 37 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    36 38 
     @pytest.mark.datafiles(DATA_DIR)
    37 39 
     def test_autotools_build(cli, tmpdir, datafiles):
    ... ... @@ -55,6 +57,8 @@ def test_autotools_build(cli, tmpdir, datafiles):
    55 57
    56 58
    57 59 
     # Test running an executable built with autotools
    60 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    61 
    +                    reason='Examples are writtent for x86_64')
    58 62 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    59 63 
     @pytest.mark.datafiles(DATA_DIR)
    60 64 
     def test_autotools_run(cli, tmpdir, datafiles):

  • tests/examples/integration-commands.py
    ... ... @@ -3,7 +3,7 @@ import pytest
    3 3
    4 4 
     from tests.testutils import cli_integration as cli
    5 5 
     from tests.testutils.integration import assert_contains
    6 
    -from tests.testutils.site import IS_LINUX
    6 
    +from tests.testutils.site import IS_LINUX, MACHINE_ARCH
    7 7
    8 8
    9 9 
     pytestmark = pytest.mark.integration
    ... ... @@ -12,6 +12,8 @@ DATA_DIR = os.path.join(
    12 12 
     )
    13 13
    14 14
    15 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    16 
    +                    reason='Examples are writtent for x86_64')
    15 17 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    16 18 
     @pytest.mark.datafiles(DATA_DIR)
    17 19 
     def test_integration_commands_build(cli, tmpdir, datafiles):
    ... ... @@ -23,6 +25,8 @@ def test_integration_commands_build(cli, tmpdir, datafiles):
    23 25
    24 26
    25 27 
     # Test running the executable
    28 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    29 
    +                    reason='Examples are writtent for x86_64')
    26 30 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    27 31 
     @pytest.mark.datafiles(DATA_DIR)
    28 32 
     def test_integration_commands_run(cli, tmpdir, datafiles):

  • tests/examples/junctions.py
    ... ... @@ -3,7 +3,7 @@ import pytest
    3 3
    4 4 
     from tests.testutils import cli_integration as cli
    5 5 
     from tests.testutils.integration import assert_contains
    6 
    -from tests.testutils.site import IS_LINUX
    6 
    +from tests.testutils.site import IS_LINUX, MACHINE_ARCH
    7 7
    8 8 
     pytestmark = pytest.mark.integration
    9 9
    ... ... @@ -13,6 +13,8 @@ DATA_DIR = os.path.join(
    13 13
    14 14
    15 15 
     # Test that the project builds successfully
    16 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    17 
    +                    reason='Examples are writtent for x86_64')
    16 18 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    17 19 
     @pytest.mark.datafiles(DATA_DIR)
    18 20 
     def test_build(cli, tmpdir, datafiles):
    ... ... @@ -23,6 +25,8 @@ def test_build(cli, tmpdir, datafiles):
    23 25
    24 26
    25 27 
     # Test the callHello script works as expected.
    28 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    29 
    +                    reason='Examples are writtent for x86_64')
    26 30 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    27 31 
     @pytest.mark.datafiles(DATA_DIR)
    28 32 
     def test_shell_call_hello(cli, tmpdir, datafiles):

  • tests/examples/running-commands.py
    ... ... @@ -3,7 +3,7 @@ import pytest
    3 3
    4 4 
     from tests.testutils import cli_integration as cli
    5 5 
     from tests.testutils.integration import assert_contains
    6 
    -from tests.testutils.site import IS_LINUX
    6 
    +from tests.testutils.site import IS_LINUX, MACHINE_ARCH
    7 7
    8 8
    9 9 
     pytestmark = pytest.mark.integration
    ... ... @@ -12,6 +12,8 @@ DATA_DIR = os.path.join(
    12 12 
     )
    13 13
    14 14
    15 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    16 
    +                    reason='Examples are writtent for x86_64')
    15 17 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    16 18 
     @pytest.mark.datafiles(DATA_DIR)
    17 19 
     def test_running_commands_build(cli, tmpdir, datafiles):
    ... ... @@ -23,6 +25,8 @@ def test_running_commands_build(cli, tmpdir, datafiles):
    23 25
    24 26
    25 27 
     # Test running the executable
    28 
    +@pytest.mark.skipif(MACHINE_ARCH != 'x86_64',
    29 
    +                    reason='Examples are writtent for x86_64')
    26 30 
     @pytest.mark.skipif(not IS_LINUX, reason='Only available on linux')
    27 31 
     @pytest.mark.datafiles(DATA_DIR)
    28 32 
     def test_running_commands_run(cli, tmpdir, datafiles):

  • tests/format/list-directive-type-error/project.conf
    ... ... @@ -4,4 +4,4 @@ options:
    4 4 
       arch:
    5 5 
         type: arch
    6 6 
         description: Example architecture option
    7 
    -    values: [ x86_32, x86_64 ]
    7 
    +    values: [ x86_32, x86_64, aarch64 ]

  • tests/frontend/invalid_element_path/project.conf
    1 
    +# Project config for frontend build test
    2 
    +name: test
    3 
    +
    4 
    +elephant-path: elements

  • tests/frontend/push.py
    ... ... @@ -230,6 +230,8 @@ def test_artifact_expires(cli, datafiles, tmpdir):
    230 230 
         # Create an artifact share (remote artifact cache) in the tmpdir/artifactshare
    231 231 
         # Mock a file system with 12 MB free disk space
    232 232 
         with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'),
    233 
    +                               min_head_size=int(2e9),
    234 
    +                               max_head_size=int(2e9),
    233 235 
                                    total_space=int(10e9), free_space=(int(12e6) + int(2e9))) as share:
    234 236
    235 237 
             # Configure bst to push to the cache
    ... ... @@ -313,6 +315,8 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
    313 315 
         # Create an artifact share (remote cache) in tmpdir/artifactshare
    314 316 
         # Mock a file system with 12 MB free disk space
    315 317 
         with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare'),
    318 
    +                               min_head_size=int(2e9),
    319 
    +                               max_head_size=int(2e9),
    316 320 
                                    total_space=int(10e9), free_space=(int(12e6) + int(2e9))) as share:
    317 321
    318 322 
             # Configure bst to push to the cache

  • tests/frontend/show.py
    ... ... @@ -36,6 +36,19 @@ def test_show(cli, datafiles, target, format, expected):
    36 36 
                                  .format(expected, result.output))
    37 37
    38 38
    39 
    +@pytest.mark.datafiles(os.path.join(
    40 
    +    os.path.dirname(os.path.realpath(__file__)),
    41 
    +    "invalid_element_path",
    42 
    +))
    43 
    +def test_show_invalid_element_path(cli, datafiles):
    44 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    45 
    +    result = cli.run(project=project, silent=True, args=[
    46 
    +        'show',
    47 
    +        "foo.bst"])
    48 
    +
    49 
    +    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
    50 
    +
    51 
    +
    39 52 
     @pytest.mark.datafiles(DATA_DIR)
    40 53 
     @pytest.mark.parametrize("target,except_,expected", [
    41 54 
         ('target.bst', 'import-bin.bst', ['import-dev.bst', 'compose-all.bst', 'target.bst']),

  • tests/integration/project/elements/base/base-alpine.bst
    ... ... @@ -7,6 +7,11 @@ description: |
    7 7
    8 8 
     sources:
    9 9 
       - kind: tar
    10 
    -    url: alpine:integration-tests-base.v1.x86_64.tar.xz
    11 10 
         base-dir: ''
    12 
    -    ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
    11 
    +    (?):
    12 
    +    - arch == "x86_64":
    13 
    +        ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
    14 
    +        url: "alpine:integration-tests-base.v1.x86_64.tar.xz"
    15 
    +    - arch == "aarch64":
    16 
    +        ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a
    17 
    +        url: "alpine:integration-tests-base.v1.aarch64.tar.xz"

  • tests/integration/project/elements/sandbox-bwrap/break-shell.bst
    1 
    +kind: manual
    2 
    +depends:
    3 
    +  - base/base-alpine.bst
    4 
    +
    5 
    +public:
    6 
    +  bst:
    7 
    +    integration-commands:
    8 
    +    - |
    9 
    +      chmod a-x /bin/sh

  • tests/integration/project/elements/sandbox-bwrap/command-exit-42.bst
    1 
    +kind: manual
    2 
    +depends:
    3 
    +  - base/base-alpine.bst
    4 
    +
    5 
    +config:
    6 
    +  build-commands:
    7 
    +  - |
    8 
    +    exit 42

  • tests/integration/project/elements/sandbox-bwrap/non-executable-shell.bst
    1 
    +kind: manual
    2 
    +
    3 
    +depends:
    4 
    +  - sandbox-bwrap/break-shell.bst
    5 
    +
    6 
    +config:
    7 
    +  build-commands:
    8 
    +  - |
    9 
    +    exit 42

  • tests/integration/project/project.conf
    ... ... @@ -9,6 +9,12 @@ options:
    9 9 
         type: bool
    10 10 
         description: Whether to expect a linux platform
    11 11 
         default: True
    12 
    +  arch:
    13 
    +    type: arch
    14 
    +    description: Current architecture
    15 
    +    values:
    16 
    +      - x86_64
    17 
    +      - aarch64
    12 18 
     split-rules:
    13 19 
       test:
    14 20 
         - |

  • tests/integration/sandbox-bwrap.py
    1 1 
     import os
    2 2 
     import pytest
    3 3
    4 
    +from buildstream._exceptions import ErrorDomain
    5 
    +
    4 6 
     from tests.testutils import cli_integration as cli
    5 7 
     from tests.testutils.integration import assert_contains
    6 
    -from tests.testutils.site import HAVE_BWRAP
    8 
    +from tests.testutils.site import HAVE_BWRAP, HAVE_BWRAP_JSON_STATUS
    7 9
    8 10
    9 11 
     pytestmark = pytest.mark.integration
    ... ... @@ -29,3 +31,32 @@ def test_sandbox_bwrap_cleanup_build(cli, tmpdir, datafiles):
    29 31 
         # Here, BuildStream should not attempt any rmdir etc.
    30 32 
         result = cli.run(project=project, args=['build', element_name])
    31 33 
         assert result.exit_code == 0
    34 
    +
    35 
    +
    36 
    +@pytest.mark.skipif(not HAVE_BWRAP, reason='Only available with bubblewrap')
    37 
    +@pytest.mark.skipif(not HAVE_BWRAP_JSON_STATUS, reason='Only available with bubblewrap supporting --json-status-fd')
    38 
    +@pytest.mark.datafiles(DATA_DIR)
    39 
    +def test_sandbox_bwrap_distinguish_setup_error(cli, tmpdir, datafiles):
    40 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    41 
    +    element_name = 'sandbox-bwrap/non-executable-shell.bst'
    42 
    +
    43 
    +    result = cli.run(project=project, args=['build', element_name])
    44 
    +    result.assert_task_error(error_domain=ErrorDomain.SANDBOX, error_reason="bwrap-sandbox-fail")
    45 
    +
    46 
    +
    47 
    +@pytest.mark.integration
    48 
    +@pytest.mark.skipif(not HAVE_BWRAP, reason='Only available with bubblewrap')
    49 
    +@pytest.mark.datafiles(DATA_DIR)
    50 
    +def test_sandbox_bwrap_return_subprocess(cli, tmpdir, datafiles):
    51 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    52 
    +    element_name = 'sandbox-bwrap/command-exit-42.bst'
    53 
    +
    54 
    +    cli.configure({
    55 
    +        "logging": {
    56 
    +            "message-format": "%{element}|%{message}",
    57 
    +        },
    58 
    +    })
    59 
    +
    60 
    +    result = cli.run(project=project, args=['build', element_name])
    61 
    +    result.assert_task_error(error_domain=ErrorDomain.ELEMENT, error_reason=None)
    62 
    +    assert "sandbox-bwrap/command-exit-42.bst|Command 'exit 42' failed with exitcode 42" in result.stderr

  • tests/testutils/artifactshare.py
    ... ... @@ -29,7 +29,11 @@ from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution
    29 29 
     #
    30 30 
     class ArtifactShare():
    31 31
    32 
    -    def __init__(self, directory, *, total_space=None, free_space=None):
    32 
    +    def __init__(self, directory, *,
    33 
    +                 total_space=None,
    34 
    +                 free_space=None,
    35 
    +                 min_head_size=int(2e9),
    36 
    +                 max_head_size=int(10e9)):
    33 37
    34 38 
             # The working directory for the artifact share (in case it
    35 39 
             # needs to do something outside of its backend's storage folder).
    ... ... @@ -53,6 +57,9 @@ class ArtifactShare():
    53 57 
             self.total_space = total_space
    54 58 
             self.free_space = free_space
    55 59
    60 
    +        self.max_head_size = max_head_size
    61 
    +        self.min_head_size = min_head_size
    62 
    +
    56 63 
             q = Queue()
    57 64
    58 65 
             self.process = Process(target=self.run, args=(q,))
    ... ... @@ -76,7 +83,10 @@ class ArtifactShare():
    76 83 
                     self.free_space = self.total_space
    77 84 
                 os.statvfs = self._mock_statvfs
    78 85
    79 
    -        server = create_server(self.repodir, enable_push=True)
    86 
    +        server = create_server(self.repodir,
    87 
    +                               max_head_size=self.max_head_size,
    88 
    +                               min_head_size=self.min_head_size,
    89 
    +                               enable_push=True)
    80 90 
             port = server.add_insecure_port('localhost:0')
    81 91
    82 92 
             server.start()
    ... ... @@ -134,6 +144,15 @@ class ArtifactShare():
    134 144
    135 145 
             try:
    136 146 
                 tree = self.cas.resolve_ref(artifact_key)
    147 
    +            reachable = set()
    148 
    +            try:
    149 
    +                self.cas._reachable_refs_dir(reachable, tree, update_mtime=False)
    150 
    +            except FileNotFoundError:
    151 
    +                return False
    152 
    +            for digest in reachable:
    153 
    +                object_name = os.path.join(self.cas.casdir, 'objects', digest[:2], digest[2:])
    154 
    +                if not os.path.exists(object_name):
    155 
    +                    return False
    137 156 
                 return True
    138 157 
             except CASError:
    139 158 
                 return False
    ... ... @@ -165,8 +184,11 @@ class ArtifactShare():
    165 184 
     # Create an ArtifactShare for use in a test case
    166 185 
     #
    167 186 
     @contextmanager
    168 
    -def create_artifact_share(directory, *, total_space=None, free_space=None):
    169 
    -    share = ArtifactShare(directory, total_space=total_space, free_space=free_space)
    187 
    +def create_artifact_share(directory, *, total_space=None, free_space=None,
    188 
    +                          min_head_size=int(2e9),
    189 
    +                          max_head_size=int(10e9)):
    190 
    +    share = ArtifactShare(directory, total_space=total_space, free_space=free_space,
    191 
    +                          min_head_size=min_head_size, max_head_size=max_head_size)
    170 192 
         try:
    171 193 
             yield share
    172 194 
         finally:

  • tests/testutils/site.py
    ... ... @@ -4,7 +4,7 @@
    4 4 
     import os
    5 5 
     import sys
    6 6
    7 
    -from buildstream import utils, ProgramNotFoundError
    7 
    +from buildstream import _site, utils, ProgramNotFoundError
    8 8
    9 9 
     try:
    10 10 
         utils.get_host_tool('bzr')
    ... ... @@ -33,8 +33,10 @@ except (ImportError, ValueError):
    33 33 
     try:
    34 34 
         utils.get_host_tool('bwrap')
    35 35 
         HAVE_BWRAP = True
    36 
    +    HAVE_BWRAP_JSON_STATUS = _site.get_bwrap_version() >= (0, 3, 2)
    36 37 
     except ProgramNotFoundError:
    37 38 
         HAVE_BWRAP = False
    39 
    +    HAVE_BWRAP_JSON_STATUS = False
    38 40
    39 41 
     try:
    40 42 
         utils.get_host_tool('lzip')
    ... ... @@ -49,3 +51,5 @@ except ImportError:
    49 51 
         HAVE_ARPY = False
    50 52
    51 53 
     IS_LINUX = os.getenv('BST_FORCE_BACKEND', sys.platform).startswith('linux')
    54 
    +
    55 
    +_, _, _, _, MACHINE_ARCH = os.uname()

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]