GNOME.org
 

[Notes] [Git][BuildStream/buildstream][tpollard/566] 6 commits: doc/Makefile: Update comment about sphinx entrypoint

Title: GitLab

Tom Pollard pushed to branch tpollard/566 at BuildStream / buildstream

Commits:

12 changed files:

Changes:

  • .gitlab-ci.yml
    1 
    -image: buildstream/testsuite-debian:9-master-123-7ce6581b
    1 
    +image: buildstream/testsuite-debian:9-06bab030-b848172c
    2 2
    3 3 
     cache:
    4 4 
       key: "$CI_JOB_NAME-"
    ... ... @@ -91,23 +91,23 @@ source_dist:
    91 91 
         - ${COVERAGE_DIR}
    92 92
    93 93 
     tests-debian-9:
    94 
    -  image: buildstream/testsuite-debian:9-master-123-7ce6581b
    94 
    +  image: buildstream/testsuite-debian:9-06bab030-b848172c
    95 95 
       <<: *tests
    96 96
    97 97 
     tests-fedora-27:
    98 
    -  image: buildstream/testsuite-fedora:27-master-123-7ce6581b
    98 
    +  image: buildstream/testsuite-fedora:27-06bab030-b848172c
    99 99 
       <<: *tests
    100 100
    101 101 
     tests-fedora-28:
    102 
    -  image: buildstream/testsuite-fedora:28-master-123-7ce6581b
    102 
    +  image: buildstream/testsuite-fedora:28-06bab030-b848172c
    103 103 
       <<: *tests
    104 104
    105 105 
     tests-ubuntu-18.04:
    106 
    -  image: buildstream/testsuite-ubuntu:18.04-master-123-7ce6581b
    106 
    +  image: buildstream/testsuite-ubuntu:18.04-06bab030-b848172c
    107 107 
       <<: *tests
    108 108
    109 109 
     overnight-fedora-28-aarch64:
    110 
    -  image: buildstream/testsuite-fedora:aarch64-28-master-123-7ce6581b
    110 
    +  image: buildstream/testsuite-fedora:aarch64-28-06bab030-32a101f6
    111 111 
       tags:
    112 112 
         - aarch64
    113 113 
       <<: *tests
    ... ... @@ -120,7 +120,7 @@ overnight-fedora-28-aarch64:
    120 120 
     tests-unix:
    121 121 
       # Use fedora here, to a) run a test on fedora and b) ensure that we
    122 122 
       # can get rid of ostree - this is not possible with debian-8
    123 
    -  image: buildstream/testsuite-fedora:27-master-123-7ce6581b
    123 
    +  image: buildstream/testsuite-fedora:27-06bab030-b848172c
    124 124 
       <<: *tests
    125 125 
       variables:
    126 126 
         BST_FORCE_BACKEND: "unix"
    ... ... @@ -140,7 +140,7 @@ tests-unix:
    140 140
    141 141 
     tests-fedora-missing-deps:
    142 142 
       # Ensure that tests behave nicely while missing bwrap and ostree
    143 
    -  image: buildstream/testsuite-fedora:28-master-123-7ce6581b
    143 
    +  image: buildstream/testsuite-fedora:28-06bab030-b848172c
    144 144 
       <<: *tests
    145 145
    146 146 
       script:

  • buildstream/_artifactcache/artifactcache.py
    ... ... @@ -74,6 +74,7 @@ class ArtifactCache():
    74 74
    75 75 
             self._has_fetch_remotes = False
    76 76 
             self._has_push_remotes = False
    77 
    +        self._has_partial_push_remotes = False
    77 78
    78 79 
             os.makedirs(self.extractdir, exist_ok=True)
    79 80
    ... ... @@ -398,6 +399,8 @@ class ArtifactCache():
    398 399 
                     self._has_fetch_remotes = True
    399 400 
                     if remote_spec.push:
    400 401 
                         self._has_push_remotes = True
    402 
    +                    if remote_spec.partial_push:
    403 
    +                        self._has_partial_push_remotes = True
    401 404
    402 405 
                     remotes[remote_spec.url] = CASRemote(remote_spec)
    403 406
    ... ... @@ -596,6 +599,31 @@ class ArtifactCache():
    596 599 
                 remotes_for_project = self._remotes[element._get_project()]
    597 600 
                 return any(remote.spec.push for remote in remotes_for_project)
    598 601
    602 
    +    # has_partial_push_remotes():
    603 
    +    #
    604 
    +    # Check whether any remote repositories are available for pushing
    605 
    +    # non-complete artifacts
    606 
    +    #
    607 
    +    # Args:
    608 
    +    #     element (Element): The Element to check
    609 
    +    #
    610 
    +    # Returns:
    611 
    +    #   (bool): True if any remote repository is configured for optional
    612 
    +    #            partial pushes, False otherwise
    613 
    +    #
    614 
    +    def has_partial_push_remotes(self, *, element=None):
    615 
    +        # If there's no partial push remotes available, we can't partial push at all
    616 
    +        if not self._has_partial_push_remotes:
    617 
    +            return False
    618 
    +        elif element is None:
    619 
    +            # At least one remote is set to allow partial pushes
    620 
    +            return True
    621 
    +        else:
    622 
    +            # Check whether the specified element's project has push remotes configured
    623 
    +            # to not accept partial artifact pushes
    624 
    +            remotes_for_project = self._remotes[element._get_project()]
    625 
    +            return any(remote.spec.partial_push for remote in remotes_for_project)
    626 
    +
    599 627 
         # push():
    600 628 
         #
    601 629 
         # Push committed artifact to remote repository.
    ... ... @@ -603,6 +631,8 @@ class ArtifactCache():
    603 631 
         # Args:
    604 632 
         #     element (Element): The Element whose artifact is to be pushed
    605 633 
         #     keys (list): The cache keys to use
    634 
    +    #     partial(bool): If the artifact is cached in a partial state
    635 
    +    #     subdir(string): Optional subdir to not push
    606 636 
         #
    607 637 
         # Returns:
    608 638 
         #   (bool): True if any remote was updated, False if no pushes were required
    ... ... @@ -610,12 +640,25 @@ class ArtifactCache():
    610 640 
         # Raises:
    611 641 
         #   (ArtifactError): if there was an error
    612 642 
         #
    613 
    -    def push(self, element, keys):
    643 
    +    def push(self, element, keys, partial=False, subdir=None):
    614 644 
             refs = [self.get_artifact_fullname(element, key) for key in list(keys)]
    615 645
    616 646 
             project = element._get_project()
    617 647
    618 
    -        push_remotes = [r for r in self._remotes[project] if r.spec.push]
    648 
    +        push_remotes = []
    649 
    +        partial_remotes = []
    650 
    +
    651 
    +        # Create list of remotes to push to, given current element and partial push config
    652 
    +        if not partial:
    653 
    +            push_remotes = [r for r in self._remotes[project] if (r.spec.push and not r.spec.partial_push)]
    654 
    +
    655 
    +        if self._has_partial_push_remotes:
    656 
    +            # Create a specific list of the remotes expecting the artifact to be push in a partial
    657 
    +            # state. This list needs to be pushed in a partial state, without the optional subdir if
    658 
    +            # exists locally. No need to attempt pushing a partial artifact to a remote that is queued to
    659 
    +            # to also recieve a full artifact
    660 
    +            partial_remotes = [r for r in self._remotes[project] if (r.spec.partial_push and r.spec.push) and
    661 
    +                               r not in push_remotes]
    619 662
    620 663 
             pushed = False
    621 664
    ... ... @@ -632,6 +675,19 @@ class ArtifactCache():
    632 675 
                         remote.spec.url, element._get_brief_display_key()
    633 676 
                     ))
    634 677
    678 
    +        for remote in partial_remotes:
    679 
    +            remote.init()
    680 
    +            display_key = element._get_brief_display_key()
    681 
    +            element.status("Pushing partial artifact {} -> {}".format(display_key, remote.spec.url))
    682 
    +
    683 
    +            if self.cas.push(refs, remote, subdir=subdir):
    684 
    +                element.info("Pushed partial artifact {} -> {}".format(display_key, remote.spec.url))
    685 
    +                pushed = True
    686 
    +            else:
    687 
    +                element.info("Remote ({}) already has {} partial cached".format(
    688 
    +                    remote.spec.url, element._get_brief_display_key()
    689 
    +                ))
    690 
    +
    635 691 
             return pushed
    636 692
    637 693 
         # pull():
    ... ... @@ -659,14 +715,23 @@ class ArtifactCache():
    659 715 
                     element.status("Pulling artifact {} <- {}".format(display_key, remote.spec.url))
    660 716
    661 717 
                     if self.cas.pull(ref, remote, progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs):
    662 
    -                    element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
    663 718 
                         if subdir:
    664 
    -                        # Attempt to extract subdir into artifact extract dir if it already exists
    665 
    -                        # without containing the subdir. If the respective artifact extract dir does not
    666 
    -                        # exist a complete extraction will complete.
    667 
    -                        self.extract(element, key, subdir)
    668 
    -                    # no need to pull from additional remotes
    669 
    -                    return True
    719 
    +                        if not self.contains_subdir_artifact(element, key, subdir):
    720 
    +                            # The pull was expecting the specific subdit to be present, attempt
    721 
    +                            # to find it in other available remotes
    722 
    +                            element.info("Pulled partial artifact {} <- {}. Attempting to retrieve {} from remotes"
    723 
    +                                         .format(display_key, remote.spec.url, subdir))
    724 
    +                        else:
    725 
    +                            element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
    726 
    +                            # Attempt to extract subdir into artifact extract dir if it already exists
    727 
    +                            # without containing the subdir. If the respective artifact extract dir does not
    728 
    +                            # exist a complete extraction will complete.
    729 
    +                            self.extract(element, key, subdir)
    730 
    +                            # no need to pull from additional remotes
    731 
    +                            return True
    732 
    +                    else:
    733 
    +                        element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
    734 
    +                        return True
    670 735 
                     else:
    671 736 
                         element.info("Remote ({}) does not have {} cached".format(
    672 737 
                             remote.spec.url, element._get_brief_display_key()

  • buildstream/_artifactcache/cascache.py
    ... ... @@ -45,7 +45,8 @@ from .. import _yaml
    45 45 
     _MAX_PAYLOAD_BYTES = 1024 * 1024
    46 46
    47 47
    48 
    -class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key client_cert instance_name')):
    48 
    +class CASRemoteSpec(namedtuple('CASRemoteSpec',
    49 
    +                               'url push partial_push server_cert client_key client_cert instance_name')):
    49 50
    50 51 
         # _new_from_config_node
    51 52 
         #
    ... ... @@ -53,9 +54,13 @@ class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key
    53 54 
         #
    54 55 
         @staticmethod
    55 56 
         def _new_from_config_node(spec_node, basedir=None):
    56 
    -        _yaml.node_validate(spec_node, ['url', 'push', 'server-cert', 'client-key', 'client-cert', 'instance_name'])
    57 
    +        _yaml.node_validate(spec_node,
    58 
    +                            ['url', 'push', 'allow-partial-push', 'server-cert', 'client-key',
    59 
    +                             'client-cert', 'instance_name'])
    57 60 
             url = _yaml.node_get(spec_node, str, 'url')
    58 61 
             push = _yaml.node_get(spec_node, bool, 'push', default_value=False)
    62 
    +        partial_push = _yaml.node_get(spec_node, bool, 'allow-partial-push', default_value=False)
    63 
    +
    59 64 
             if not url:
    60 65 
                 provenance = _yaml.node_get_provenance(spec_node, 'url')
    61 66 
                 raise LoadError(LoadErrorReason.INVALID_DATA,
    ... ... @@ -85,10 +90,10 @@ class CASRemoteSpec(namedtuple('CASRemoteSpec', 'url push server_cert client_key
    85 90 
                 raise LoadError(LoadErrorReason.INVALID_DATA,
    86 91 
                                 "{}: 'client-cert' was specified without 'client-key'".format(provenance))
    87 92
    88 
    -        return CASRemoteSpec(url, push, server_cert, client_key, client_cert, instance_name)
    93 
    +        return CASRemoteSpec(url, push, partial_push, server_cert, client_key, client_cert, instance_name)
    89 94
    90 95
    91 
    -CASRemoteSpec.__new__.__defaults__ = (None, None, None, None)
    96 
    +CASRemoteSpec.__new__.__defaults__ = (False, None, None, None, None)
    92 97
    93 98
    94 99 
     class BlobNotFound(CASError):
    ... ... @@ -283,34 +288,40 @@ class CASCache():
    283 288 
         #   (bool): True if pull was successful, False if ref was not available
    284 289 
         #
    285 290 
         def pull(self, ref, remote, *, progress=None, subdir=None, excluded_subdirs=None):
    286 
    -        try:
    287 
    -            remote.init()
    288 291
    289 
    -            request = buildstream_pb2.GetReferenceRequest(instance_name=remote.spec.instance_name)
    290 
    -            request.key = ref
    291 
    -            response = remote.ref_storage.GetReference(request)
    292 
    +        while True:
    293 
    +            try:
    294 
    +                remote.init()
    292 295
    293 
    -            tree = remote_execution_pb2.Digest()
    294 
    -            tree.hash = response.digest.hash
    295 
    -            tree.size_bytes = response.digest.size_bytes
    296 
    +                request = buildstream_pb2.GetReferenceRequest(instance_name=remote.spec.instance_name)
    297 
    +                request.key = ref
    298 
    +                response = remote.ref_storage.GetReference(request)
    296 299
    297 
    -            # Check if the element artifact is present, if so just fetch the subdir.
    298 
    -            if subdir and os.path.exists(self.objpath(tree)):
    299 
    -                self._fetch_subdir(remote, tree, subdir)
    300 
    -            else:
    301 
    -                # Fetch artifact, excluded_subdirs determined in pullqueue
    302 
    -                self._fetch_directory(remote, tree, excluded_subdirs=excluded_subdirs)
    300 
    +                tree = remote_execution_pb2.Digest()
    301 
    +                tree.hash = response.digest.hash
    302 
    +                tree.size_bytes = response.digest.size_bytes
    303 303
    304 
    -            self.set_ref(ref, tree)
    304 
    +                # Check if the element artifact is present, if so just fetch the subdir.
    305 
    +                if subdir and os.path.exists(self.objpath(tree)):
    306 
    +                    self._fetch_subdir(remote, tree, subdir)
    307 
    +                else:
    308 
    +                    # Fetch artifact, excluded_subdirs determined in pullqueue
    309 
    +                    self._fetch_directory(remote, tree, excluded_subdirs=excluded_subdirs)
    305 310
    306 
    -            return True
    307 
    -        except grpc.RpcError as e:
    308 
    -            if e.code() != grpc.StatusCode.NOT_FOUND:
    309 
    -                raise CASError("Failed to pull ref {}: {}".format(ref, e)) from e
    310 
    -            else:
    311 
    -                return False
    312 
    -        except BlobNotFound as e:
    313 
    -            return False
    311 
    +                self.set_ref(ref, tree)
    312 
    +
    313 
    +                return True
    314 
    +            except grpc.RpcError as e:
    315 
    +                if e.code() != grpc.StatusCode.NOT_FOUND:
    316 
    +                    raise CASError("Failed to pull ref {}: {}".format(ref, e)) from e
    317 
    +                else:
    318 
    +                    return False
    319 
    +            except BlobNotFound as e:
    320 
    +                if not excluded_subdirs and subdir:
    321 
    +                    # Could not complete a full pull, attempt partial
    322 
    +                    excluded_subdirs, subdir = subdir, excluded_subdirs
    323 
    +                else:
    324 
    +                    return False
    314 325
    315 326 
         # pull_tree():
    316 327 
         #
    ... ... @@ -355,6 +366,7 @@ class CASCache():
    355 366 
         # Args:
    356 367 
         #     refs (list): The refs to push
    357 368 
         #     remote (CASRemote): The remote to push to
    369 
    +    #     subdir (string): Optional specific subdir to exempt from the push
    358 370 
         #
    359 371 
         # Returns:
    360 372 
         #   (bool): True if any remote was updated, False if no pushes were required
    ... ... @@ -362,7 +374,7 @@ class CASCache():
    362 374 
         # Raises:
    363 375 
         #   (CASError): if there was an error
    364 376 
         #
    365 
    -    def push(self, refs, remote):
    377 
    +    def push(self, refs, remote, subdir=None):
    366 378 
             skipped_remote = True
    367 379 
             try:
    368 380 
                 for ref in refs:
    ... ... @@ -384,7 +396,7 @@ class CASCache():
    384 396 
                             # Intentionally re-raise RpcError for outer except block.
    385 397 
                             raise
    386 398
    387 
    -                self._send_directory(remote, tree)
    399 
    +                self._send_directory(remote, tree, excluded_dir=subdir)
    388 400
    389 401 
                     request = buildstream_pb2.UpdateReferenceRequest(instance_name=remote.spec.instance_name)
    390 402 
                     request.keys.append(ref)
    ... ... @@ -866,10 +878,17 @@ class CASCache():
    866 878 
                     a += 1
    867 879 
                     b += 1
    868 880
    869 
    -    def _reachable_refs_dir(self, reachable, tree, update_mtime=False):
    881 
    +    def _reachable_refs_dir(self, reachable, tree, update_mtime=False, subdir=False):
    870 882 
             if tree.hash in reachable:
    871 883 
                 return
    872 884
    885 
    +        # If looping through subdir digests, skip processing if
    886 
    +        # ref path does not exist, allowing for partial objects
    887 
    +        if subdir and not os.path.exists(self.objpath(tree)):
    888 
    +            return
    889 
    +
    890 
    +        # Raises FileNotFound exception is path does not exist,
    891 
    +        # which should only be entered on the top level digest
    873 892 
             if update_mtime:
    874 893 
                 os.utime(self.objpath(tree))
    875 894
    ... ... @@ -886,9 +905,9 @@ class CASCache():
    886 905 
                 reachable.add(filenode.digest.hash)
    887 906
    888 907 
             for dirnode in directory.directories:
    889 
    -            self._reachable_refs_dir(reachable, dirnode.digest, update_mtime=update_mtime)
    908 
    +            self._reachable_refs_dir(reachable, dirnode.digest, update_mtime=update_mtime, subdir=True)
    890 909
    891 
    -    def _required_blobs(self, directory_digest):
    910 
    +    def _required_blobs(self, directory_digest, excluded_dir=None):
    892 911 
             # parse directory, and recursively add blobs
    893 912 
             d = remote_execution_pb2.Digest()
    894 913 
             d.hash = directory_digest.hash
    ... ... @@ -907,7 +926,8 @@ class CASCache():
    907 926 
                 yield d
    908 927
    909 928 
             for dirnode in directory.directories:
    910 
    -            yield from self._required_blobs(dirnode.digest)
    929 
    +            if dirnode.name != excluded_dir:
    930 
    +                yield from self._required_blobs(dirnode.digest)
    911 931
    912 932 
         def _fetch_blob(self, remote, digest, stream):
    913 933 
             resource_name_components = ['blobs', digest.hash, str(digest.size_bytes)]
    ... ... @@ -1029,6 +1049,7 @@ class CASCache():
    1029 1049 
                 objpath = self._ensure_blob(remote, dir_digest)
    1030 1050
    1031 1051 
                 directory = remote_execution_pb2.Directory()
    1052 
    +
    1032 1053 
                 with open(objpath, 'rb') as f:
    1033 1054 
                     directory.ParseFromString(f.read())
    1034 1055
    ... ... @@ -1104,9 +1125,8 @@ class CASCache():
    1104 1125
    1105 1126 
             assert response.committed_size == digest.size_bytes
    1106 1127
    1107 
    -    def _send_directory(self, remote, digest, u_uid=uuid.uuid4()):
    1108 
    -        required_blobs = self._required_blobs(digest)
    1109 
    -
    1128 
    +    def _send_directory(self, remote, digest, u_uid=uuid.uuid4(), excluded_dir=None):
    1129 
    +        required_blobs = self._required_blobs(digest, excluded_dir=excluded_dir)
    1110 1130 
             missing_blobs = dict()
    1111 1131 
             # Limit size of FindMissingBlobs request
    1112 1132 
             for required_blobs_group in _grouper(required_blobs, 512):

  • buildstream/_ostree.py
    ... ... @@ -34,7 +34,7 @@ from ._exceptions import BstError, ErrorDomain
    34 34
    35 35 
     # pylint: disable=wrong-import-position,wrong-import-order
    36 36 
     gi.require_version('OSTree', '1.0')
    37 
    -from gi.repository import GLib, Gio, OSTree  # nopep8
    37 
    +from gi.repository import GLib, Gio, OSTree  # noqa
    38 38
    39 39
    40 40 
     # For users of this file, they must expect (except) it.

  • buildstream/_signals.py
    ... ... @@ -38,7 +38,7 @@ def terminator_handler(signal_, frame):
    38 38 
             terminator_ = terminator_stack.pop()
    39 39 
             try:
    40 40 
                 terminator_()
    41 
    -        except:                                                 # pylint: disable=bare-except
    41 
    +        except:                               # noqa pylint: disable=bare-except
    42 42 
                 # Ensure we print something if there's an exception raised when
    43 43 
                 # processing the handlers. Note that the default exception
    44 44 
                 # handler won't be called because we os._exit next, so we must

  • buildstream/element.py
    ... ... @@ -1568,7 +1568,7 @@ class Element(Plugin):
    1568 1568 
                     utils._force_rmtree(rootdir)
    1569 1569
    1570 1570 
                 with _signals.terminator(cleanup_rootdir), \
    1571 
    -                self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox:  # nopep8
    1571 
    +                self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox:  # noqa
    1572 1572
    1573 1573 
                     # By default, the dynamic public data is the same as the static public data.
    1574 1574 
                     # The plugin's assemble() method may modify this, though.
    ... ... @@ -1800,13 +1800,19 @@ class Element(Plugin):
    1800 1800 
         #   (bool): True if this element does not need a push job to be created
    1801 1801 
         #
    1802 1802 
         def _skip_push(self):
    1803 
    +
    1803 1804 
             if not self.__artifacts.has_push_remotes(element=self):
    1804 1805 
                 # No push remotes for this element's project
    1805 1806 
                 return True
    1806 1807
    1807 1808 
             # Do not push elements that aren't cached, or that are cached with a dangling buildtree
    1808 
    -        # artifact unless element type is expected to have an an empty buildtree directory
    1809 
    -        if not self._cached_buildtree():
    1809 
    +        # artifact unless element type is expected to have an an empty buildtree directory. Check
    1810 
    +        # that this default behaviour is not overriden via a remote configured to allow pushing
    1811 
    +        # artifacts without their corresponding buildtree.
    1812 
    +        if not self._cached():
    1813 
    +            return True
    1814 
    +
    1815 
    +        if not self._cached_buildtree() and not self.__artifacts.has_partial_push_remotes(element=self):
    1810 1816 
                 return True
    1811 1817
    1812 1818 
             # Do not push tainted artifact
    ... ... @@ -1817,7 +1823,8 @@ class Element(Plugin):
    1817 1823
    1818 1824 
         # _push():
    1819 1825 
         #
    1820 
    -    # Push locally cached artifact to remote artifact repository.
    1826 
    +    # Push locally cached artifact to remote artifact repository. An attempt
    1827 
    +    # will be made to push partial artifacts given current config
    1821 1828 
         #
    1822 1829 
         # Returns:
    1823 1830 
         #   (bool): True if the remote was updated, False if it already existed
    ... ... @@ -1830,8 +1837,19 @@ class Element(Plugin):
    1830 1837 
                 self.warn("Not pushing tainted artifact.")
    1831 1838 
                 return False
    1832 1839
    1833 
    -        # Push all keys used for local commit
    1834 
    -        pushed = self.__artifacts.push(self, self.__get_cache_keys_for_commit())
    1840 
    +        # Push all keys used for local commit, this could be full or partial,
    1841 
    +        # given previous _skip_push() logic. If buildtree isn't cached, then
    1842 
    +        # set partial push
    1843 
    +
    1844 
    +        partial = False
    1845 
    +        subdir = 'buildtree'
    1846 
    +        if not self._cached_buildtree():
    1847 
    +            partial = True
    1848 
    +
    1849 
    +        pushed = self.__artifacts.push(self, self.__get_cache_keys_for_commit(), partial=partial, subdir=subdir)
    1850 
    +
    1851 
    +        # Artifact might be cached in the server partially with the top level ref existing.
    1852 
    +        # Check if we need to attempt a push of a locally cached buildtree given current config
    1835 1853 
             if not pushed:
    1836 1854 
                 return False
    1837 1855

  • dev-requirements.txt
    1 1 
     coverage == 4.4.0
    2 
    -pep8
    3 2 
     pylint
    4 3 
     pytest >= 3.9
    4 
    +pytest-codestyle >= 1.4.0
    5 5 
     pytest-cov >= 2.5.0
    6 6 
     pytest-datafiles >= 2.0
    7 7 
     pytest-env
    8 
    -pytest-pep8
    9 8 
     pytest-pylint
    10 9 
     pytest-xdist
    11 10 
     pytest-timeout

  • doc/Makefile
    ... ... @@ -6,10 +6,10 @@
    6 6 
     # we dont use the standard `sphinx-build` and `sphinx-apidoc` entry points.
    7 7 
     #
    8 8 
     # The following technique works as long as sphinx is installed for python3,
    9 
    -# regardless of the entry point which might be in /usr/bin or PATH, but
    10 
    -# will stop working in sphinx >= 2.0. Hopefully by then, the mentioned
    11 
    -# bug will be fixed and we can use a standard python3 specific script to
    12 
    -# invoke sphnix.
    9 
    +# regardless of the entry point which might be in /usr/bin or PATH.
    10 
    +#
    11 
    +# Since Sphinx 2.0 is planned to be Python 3-only, this workaround should not
    12 
    +# be needed once Spinx 2.0 is released, and we upgrade to it
    13 13 
     #
    14 14 
     SPHINXOPTS    =
    15 15 
     SPHINXBUILD   = python3 -m sphinx

  • setup.cfg
    ... ... @@ -11,20 +11,11 @@ parentdir_prefix = BuildStream-
    11 11 
     test=pytest
    12 12
    13 13 
     [tool:pytest]
    14 
    -addopts = --verbose --basetemp ./tmp --pep8 --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc --durations=20
    14 
    +addopts = --verbose --basetemp ./tmp --codestyle --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc --durations=20
    15 15 
     norecursedirs = tests/integration/project integration-cache tmp __pycache__ .eggs
    16 16 
     python_files = tests/*/*.py
    17 
    -pep8maxlinelength = 119
    18 
    -pep8ignore =
    19 
    -    * E129
    20 
    -    * E125
    21 
    -    doc/source/conf.py ALL
    22 
    -    tmp/* ALL
    23 
    -    */lib/python3* ALL
    24 
    -    */bin/* ALL
    25 
    -    buildstream/_fuse/fuse.py ALL
    26 
    -    .eggs/* ALL
    27 
    -    *_pb2.py ALL
    28 
    -    *_pb2_grpc.py ALL
    17 
    +codestyle_max_line_length = 119
    18 
    +codestyle_ignore = E129 E125 W504 W605
    19 
    +codestyle_exclude = doc/source/conf.py buildstream/_fuse/fuse.py buildstream/_protos/**/*py
    29 20 
     env =
    30 21 
         D:BST_TEST_SUITE=True

  • tests/integration/pushbuildtrees.py
    1 
    +import os
    2 
    +import shutil
    3 
    +import pytest
    4 
    +
    5 
    +from tests.testutils import cli_integration as cli, create_artifact_share
    6 
    +from tests.testutils.integration import assert_contains
    7 
    +from tests.testutils.site import HAVE_BWRAP, IS_LINUX
    8 
    +from buildstream._exceptions import ErrorDomain, LoadErrorReason
    9 
    +
    10 
    +
    11 
    +DATA_DIR = os.path.join(
    12 
    +    os.path.dirname(os.path.realpath(__file__)),
    13 
    +    "project"
    14 
    +)
    15 
    +
    16 
    +
    17 
    +# Remove artifact cache & set cli.config value of pull-buildtrees
    18 
    +# to false, which is the default user context. The cache has to be
    19 
    +# cleared as just forcefully removing the refpath leaves dangling objects.
    20 
    +def default_state(cli, tmpdir, share):
    21 
    +    shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    22 
    +    cli.configure({
    23 
    +        'artifacts': {'url': share.repo, 'push': False},
    24 
    +        'artifactdir': os.path.join(str(tmpdir), 'artifacts'),
    25 
    +        'cache': {'pull-buildtrees': False},
    26 
    +    })
    27 
    +
    28 
    +
    29 
    +# Tests to capture the integration of the optionl push of buildtrees.
    30 
    +# The behaviour should encompass pushing artifacts that are already cached
    31 
    +# without a buildtree as well as artifacts that are cached with their buildtree.
    32 
    +# This option is handled via 'allow-partial-push' on a per artifact remote config
    33 
    +# node basis. Multiple remote config nodes can point to the same url and as such can
    34 
    +# have different 'allow-partial-push' options, tests need to cover this using project
    35 
    +# confs.
    36 
    +@pytest.mark.integration
    37 
    +@pytest.mark.datafiles(DATA_DIR)
    38 
    +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    39 
    +def test_pushbuildtrees(cli, tmpdir, datafiles, integration_cache):
    40 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    41 
    +    element_name = 'autotools/amhello.bst'
    42 
    +
    43 
    +    # Create artifact shares for pull & push testing
    44 
    +    with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
    45 
    +        create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\
    46 
    +        create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3,\
    47 
    +        create_artifact_share(os.path.join(str(tmpdir), 'share4')) as share4:
    48 
    +
    49 
    +        cli.configure({
    50 
    +            'artifacts': {'url': share1.repo, 'push': True},
    51 
    +            'artifactdir': os.path.join(str(tmpdir), 'artifacts')
    52 
    +        })
    53 
    +
    54 
    +        cli.configure({'artifacts': [{'url': share1.repo, 'push': True},
    55 
    +                                     {'url': share2.repo, 'push': True, 'allow-partial-push': True}]})
    56 
    +
    57 
    +        # Build autotools element, checked pushed, delete local.
    58 
    +        # As share 2 has push & allow-partial-push set a true, it
    59 
    +        # should have pushed the artifacts, without the cached buildtrees,
    60 
    +        # to it.
    61 
    +        result = cli.run(project=project, args=['build', element_name])
    62 
    +        assert result.exit_code == 0
    63 
    +        assert cli.get_element_state(project, element_name) == 'cached'
    64 
    +        elementdigest = share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    65 
    +        buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
    66 
    +                                    elementdigest.hash, 'buildtree')
    67 
    +        assert os.path.isdir(buildtreedir)
    68 
    +        assert element_name in result.get_partial_pushed_elements()
    69 
    +        assert element_name in result.get_pushed_elements()
    70 
    +        assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    71 
    +        assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    72 
    +        default_state(cli, tmpdir, share1)
    73 
    +
    74 
    +        # Check that after explictly pulling an artifact without it's buildtree,
    75 
    +        # we can push it to another remote that is configured to accept the partial
    76 
    +        # artifact
    77 
    +        result = cli.run(project=project, args=['pull', element_name])
    78 
    +        assert element_name in result.get_pulled_elements()
    79 
    +        cli.configure({'artifacts': {'url': share3.repo, 'push': True, 'allow-partial-push': True}})
    80 
    +        assert cli.get_element_state(project, element_name) == 'cached'
    81 
    +        assert not os.path.isdir(buildtreedir)
    82 
    +        result = cli.run(project=project, args=['push', element_name])
    83 
    +        assert result.exit_code == 0
    84 
    +        assert element_name in result.get_partial_pushed_elements()
    85 
    +        assert element_name not in result.get_pushed_elements()
    86 
    +        assert share3.has_artifact('test', element_name, cli.get_element_key(project, element_name))
    87 
    +        default_state(cli, tmpdir, share3)
    88 
    +
    89 
    +        # Delete the local cache and pull the partial artifact from share 3,
    90 
    +        # this should not include the buildtree when extracted locally, even when
    91 
    +        # pull-buildtrees is given as a cli parameter as no available remotes will
    92 
    +        # contain the buildtree
    93 
    +        assert not os.path.isdir(buildtreedir)
    94 
    +        assert cli.get_element_state(project, element_name) != 'cached'
    95 
    +        result = cli.run(project=project, args=['--pull-buildtrees', 'pull', element_name])
    96 
    +        assert element_name in result.get_partial_pulled_elements()
    97 
    +        assert not os.path.isdir(buildtreedir)
    98 
    +        default_state(cli, tmpdir, share3)
    99 
    +
    100 
    +        # Delete the local cache and attempt to pull a 'full' artifact, including its
    101 
    +        # buildtree. As with before share3 being the first listed remote will not have
    102 
    +        # the buildtree available and should spawn a partial pull. Having share1 as the
    103 
    +        # second available remote should allow the buildtree to be pulled thus 'completing'
    104 
    +        # the artifact
    105 
    +        cli.configure({'artifacts': [{'url': share3.repo, 'push': True, 'allow-partial-push': True},
    106 
    +                                     {'url': share1.repo, 'push': True}]})
    107 
    +        assert cli.get_element_state(project, element_name) != 'cached'
    108 
    +        result = cli.run(project=project, args=['--pull-buildtrees', 'pull', element_name])
    109 
    +        assert element_name in result.get_partial_pulled_elements()
    110 
    +        assert element_name in result.get_pulled_elements()
    111 
    +        assert "Attempting to retrieve buildtree from remotes" in result.stderr
    112 
    +        assert os.path.isdir(buildtreedir)
    113 
    +        assert cli.get_element_state(project, element_name) == 'cached'

  • tests/testutils/http_server.py
    ... ... @@ -29,7 +29,7 @@ class RequestHandler(SimpleHTTPRequestHandler):
    29 29 
                     expected_password, directory = self.server.users[user]
    30 30 
                     if password == expected_password:
    31 31 
                         return directory
    32 
    -            except:
    32 
    +            except:                           # noqa
    33 33 
                     raise Unauthorized('unauthorized')
    34 34 
                 return None
    35 35

  • tests/testutils/runcli.py
    ... ... @@ -191,6 +191,13 @@ class Result():
    191 191
    192 192 
             return list(pushed)
    193 193
    194 
    +    def get_partial_pushed_elements(self):
    195 
    +        pushed = re.findall(r'\[\s*push:(\S+)\s*\]\s*INFO\s*Pushed partial artifact', self.stderr)
    196 
    +        if pushed is None:
    197 
    +            return []
    198 
    +
    199 
    +        return list(pushed)
    200 
    +
    194 201 
         def get_pulled_elements(self):
    195 202 
             pulled = re.findall(r'\[\s*pull:(\S+)\s*\]\s*INFO\s*Pulled artifact', self.stderr)
    196 203 
             if pulled is None:
    ... ... @@ -198,6 +205,13 @@ class Result():
    198 205
    199 206 
             return list(pulled)
    200 207
    208 
    +    def get_partial_pulled_elements(self):
    209 
    +        pulled = re.findall(r'\[\s*pull:(\S+)\s*\]\s*INFO\s*Pulled partial artifact', self.stderr)
    210 
    +        if pulled is None:
    211 
    +            return []
    212 
    +
    213 
    +        return list(pulled)
    214 
    +
    201 215
    202 216 
     class Cli():
    203 217

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]