[Notes] [Git][BuildStream/buildstream][master] 8 commits: _casbaseddirectory.py: Fix order in _recalculate_recursing_up()



Title: GitLab

Jürg Billeter pushed to branch master at BuildStream / buildstream

Commits:

5 changed files:

Changes:

  • buildstream/_artifactcache.py
    ... ... @@ -588,13 +588,16 @@ class ArtifactCache():
    588 588
         #
    
    589 589
         # Args:
    
    590 590
         #     element (Element): The Element commit an artifact for
    
    591
    -    #     content (str): The element's content directory
    
    591
    +    #     content (Directory): The element's content directory
    
    592 592
         #     keys (list): The cache keys to use
    
    593 593
         #
    
    594 594
         def commit(self, element, content, keys):
    
    595 595
             refs = [element.get_artifact_name(key) for key in keys]
    
    596 596
     
    
    597
    -        self.cas.commit(refs, content)
    
    597
    +        tree = content._get_digest()
    
    598
    +
    
    599
    +        for ref in refs:
    
    600
    +            self.cas.set_ref(ref, tree)
    
    598 601
     
    
    599 602
         # diff():
    
    600 603
         #
    

  • buildstream/element.py
    ... ... @@ -103,6 +103,7 @@ from .types import _KeyStrength, CoreWarnings
    103 103
     
    
    104 104
     from .storage.directory import Directory
    
    105 105
     from .storage._filebaseddirectory import FileBasedDirectory
    
    106
    +from .storage._casbaseddirectory import CasBasedDirectory
    
    106 107
     from .storage.directory import VirtualDirectoryError
    
    107 108
     
    
    108 109
     
    
    ... ... @@ -1670,106 +1671,109 @@ class Element(Plugin):
    1670 1671
                         cleanup_rootdir()
    
    1671 1672
     
    
    1672 1673
         def _cache_artifact(self, rootdir, sandbox, collect):
    
    1673
    -        if collect is not None:
    
    1674
    -            try:
    
    1675
    -                sandbox_vroot = sandbox.get_virtual_directory()
    
    1676
    -                collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
    
    1677
    -            except VirtualDirectoryError:
    
    1678
    -                # No collect directory existed
    
    1679
    -                collectvdir = None
    
    1674
    +        with self.timed_activity("Caching artifact"):
    
    1675
    +            if collect is not None:
    
    1676
    +                try:
    
    1677
    +                    sandbox_vroot = sandbox.get_virtual_directory()
    
    1678
    +                    collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
    
    1679
    +                except VirtualDirectoryError:
    
    1680
    +                    # No collect directory existed
    
    1681
    +                    collectvdir = None
    
    1680 1682
     
    
    1681
    -        context = self._get_context()
    
    1683
    +            context = self._get_context()
    
    1682 1684
     
    
    1683
    -        # Create artifact directory structure
    
    1684
    -        assembledir = os.path.join(rootdir, 'artifact')
    
    1685
    -        filesdir = os.path.join(assembledir, 'files')
    
    1686
    -        logsdir = os.path.join(assembledir, 'logs')
    
    1687
    -        metadir = os.path.join(assembledir, 'meta')
    
    1688
    -        buildtreedir = os.path.join(assembledir, 'buildtree')
    
    1689
    -        os.mkdir(assembledir)
    
    1690
    -        if collect is not None and collectvdir is not None:
    
    1691
    -            os.mkdir(filesdir)
    
    1692
    -        os.mkdir(logsdir)
    
    1693
    -        os.mkdir(metadir)
    
    1694
    -        os.mkdir(buildtreedir)
    
    1695
    -
    
    1696
    -        # Hard link files from collect dir to files directory
    
    1697
    -        if collect is not None and collectvdir is not None:
    
    1698
    -            collectvdir.export_files(filesdir, can_link=True)
    
    1699
    -
    
    1700
    -        cache_buildtrees = context.cache_buildtrees
    
    1701
    -        build_success = self.__build_result[0]
    
    1702
    -
    
    1703
    -        # cache_buildtrees defaults to 'always', as such the
    
    1704
    -        # default behaviour is to attempt to cache them. If only
    
    1705
    -        # caching failed artifact buildtrees, then query the build
    
    1706
    -        # result. Element types without a build-root dir will be cached
    
    1707
    -        # with an empty buildtreedir regardless of this configuration.
    
    1708
    -
    
    1709
    -        if cache_buildtrees == 'always' or (cache_buildtrees == 'failure' and not build_success):
    
    1710
    -            try:
    
    1685
    +            assemblevdir = CasBasedDirectory(cas_cache=context.artifactcache.cas, ref=None)
    
    1686
    +            logsvdir = assemblevdir.descend("logs", create=True)
    
    1687
    +            metavdir = assemblevdir.descend("meta", create=True)
    
    1688
    +            buildtreevdir = assemblevdir.descend("buildtree", create=True)
    
    1689
    +
    
    1690
    +            # Create artifact directory structure
    
    1691
    +            assembledir = os.path.join(rootdir, 'artifact')
    
    1692
    +            logsdir = os.path.join(assembledir, 'logs')
    
    1693
    +            metadir = os.path.join(assembledir, 'meta')
    
    1694
    +            os.mkdir(assembledir)
    
    1695
    +            os.mkdir(logsdir)
    
    1696
    +            os.mkdir(metadir)
    
    1697
    +
    
    1698
    +            if collect is not None and collectvdir is not None:
    
    1699
    +                filesvdir = assemblevdir.descend("files", create=True)
    
    1700
    +                filesvdir.import_files(collectvdir)
    
    1701
    +
    
    1702
    +            cache_buildtrees = context.cache_buildtrees
    
    1703
    +            build_success = self.__build_result[0]
    
    1704
    +
    
    1705
    +            # cache_buildtrees defaults to 'always', as such the
    
    1706
    +            # default behaviour is to attempt to cache them. If only
    
    1707
    +            # caching failed artifact buildtrees, then query the build
    
    1708
    +            # result. Element types without a build-root dir will be cached
    
    1709
    +            # with an empty buildtreedir regardless of this configuration.
    
    1710
    +
    
    1711
    +            if cache_buildtrees == 'always' or (cache_buildtrees == 'failure' and not build_success):
    
    1711 1712
                     sandbox_vroot = sandbox.get_virtual_directory()
    
    1712
    -                sandbox_build_dir = sandbox_vroot.descend(
    
    1713
    -                    self.get_variable('build-root').lstrip(os.sep).split(os.sep))
    
    1714
    -                # Hard link files from build-root dir to buildtreedir directory
    
    1715
    -                sandbox_build_dir.export_files(buildtreedir)
    
    1716
    -            except VirtualDirectoryError:
    
    1717
    -                # Directory could not be found. Pre-virtual
    
    1718
    -                # directory behaviour was to continue silently
    
    1719
    -                # if the directory could not be found.
    
    1720
    -                pass
    
    1713
    +                try:
    
    1714
    +                    sandbox_build_dir = sandbox_vroot.descend(
    
    1715
    +                        self.get_variable('build-root').lstrip(os.sep).split(os.sep))
    
    1716
    +                    buildtreevdir.import_files(sandbox_build_dir)
    
    1717
    +                except VirtualDirectoryError:
    
    1718
    +                    # Directory could not be found. Pre-virtual
    
    1719
    +                    # directory behaviour was to continue silently
    
    1720
    +                    # if the directory could not be found.
    
    1721
    +                    pass
    
    1722
    +
    
    1723
    +            # Write some logs out to normal directories: logsdir and metadir
    
    1724
    +            # Copy build log
    
    1725
    +            log_filename = context.get_log_filename()
    
    1726
    +            self._build_log_path = os.path.join(logsdir, 'build.log')
    
    1727
    +            if log_filename:
    
    1728
    +                shutil.copyfile(log_filename, self._build_log_path)
    
    1729
    +
    
    1730
    +            # Store public data
    
    1731
    +            _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
    
    1732
    +
    
    1733
    +            # Store result
    
    1734
    +            build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
    
    1735
    +            if self.__build_result[2] is not None:
    
    1736
    +                build_result_dict["detail"] = self.__build_result[2]
    
    1737
    +            _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
    
    1738
    +
    
    1739
    +            # ensure we have cache keys
    
    1740
    +            self._assemble_done()
    
    1741
    +
    
    1742
    +            # Store keys.yaml
    
    1743
    +            _yaml.dump(_yaml.node_sanitize({
    
    1744
    +                'strong': self._get_cache_key(),
    
    1745
    +                'weak': self._get_cache_key(_KeyStrength.WEAK),
    
    1746
    +            }), os.path.join(metadir, 'keys.yaml'))
    
    1747
    +
    
    1748
    +            # Store dependencies.yaml
    
    1749
    +            _yaml.dump(_yaml.node_sanitize({
    
    1750
    +                e.name: e._get_cache_key() for e in self.dependencies(Scope.BUILD)
    
    1751
    +            }), os.path.join(metadir, 'dependencies.yaml'))
    
    1752
    +
    
    1753
    +            # Store workspaced.yaml
    
    1754
    +            _yaml.dump(_yaml.node_sanitize({
    
    1755
    +                'workspaced': bool(self._get_workspace())
    
    1756
    +            }), os.path.join(metadir, 'workspaced.yaml'))
    
    1757
    +
    
    1758
    +            # Store workspaced-dependencies.yaml
    
    1759
    +            _yaml.dump(_yaml.node_sanitize({
    
    1760
    +                'workspaced-dependencies': [
    
    1761
    +                    e.name for e in self.dependencies(Scope.BUILD)
    
    1762
    +                    if e._get_workspace()
    
    1763
    +                ]
    
    1764
    +            }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
    
    1721 1765
     
    
    1722
    -        # Copy build log
    
    1723
    -        log_filename = context.get_log_filename()
    
    1724
    -        self._build_log_path = os.path.join(logsdir, 'build.log')
    
    1725
    -        if log_filename:
    
    1726
    -            shutil.copyfile(log_filename, self._build_log_path)
    
    1727
    -
    
    1728
    -        # Store public data
    
    1729
    -        _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
    
    1730
    -
    
    1731
    -        # Store result
    
    1732
    -        build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
    
    1733
    -        if self.__build_result[2] is not None:
    
    1734
    -            build_result_dict["detail"] = self.__build_result[2]
    
    1735
    -        _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
    
    1736
    -
    
    1737
    -        # ensure we have cache keys
    
    1738
    -        self._assemble_done()
    
    1739
    -
    
    1740
    -        # Store keys.yaml
    
    1741
    -        _yaml.dump(_yaml.node_sanitize({
    
    1742
    -            'strong': self._get_cache_key(),
    
    1743
    -            'weak': self._get_cache_key(_KeyStrength.WEAK),
    
    1744
    -        }), os.path.join(metadir, 'keys.yaml'))
    
    1745
    -
    
    1746
    -        # Store dependencies.yaml
    
    1747
    -        _yaml.dump(_yaml.node_sanitize({
    
    1748
    -            e.name: e._get_cache_key() for e in self.dependencies(Scope.BUILD)
    
    1749
    -        }), os.path.join(metadir, 'dependencies.yaml'))
    
    1750
    -
    
    1751
    -        # Store workspaced.yaml
    
    1752
    -        _yaml.dump(_yaml.node_sanitize({
    
    1753
    -            'workspaced': bool(self._get_workspace())
    
    1754
    -        }), os.path.join(metadir, 'workspaced.yaml'))
    
    1755
    -
    
    1756
    -        # Store workspaced-dependencies.yaml
    
    1757
    -        _yaml.dump(_yaml.node_sanitize({
    
    1758
    -            'workspaced-dependencies': [
    
    1759
    -                e.name for e in self.dependencies(Scope.BUILD)
    
    1760
    -                if e._get_workspace()
    
    1761
    -            ]
    
    1762
    -        }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
    
    1766
    +            metavdir.import_files(metadir)
    
    1767
    +            logsvdir.import_files(logsdir)
    
    1763 1768
     
    
    1764
    -        with self.timed_activity("Caching artifact"):
    
    1765
    -            artifact_size = utils._get_dir_size(assembledir)
    
    1766
    -            self.__artifacts.commit(self, assembledir, self.__get_cache_keys_for_commit())
    
    1767
    -
    
    1768
    -        if collect is not None and collectvdir is None:
    
    1769
    -            raise ElementError(
    
    1770
    -                "Directory '{}' was not found inside the sandbox, "
    
    1771
    -                "unable to collect artifact contents"
    
    1772
    -                .format(collect))
    
    1769
    +            artifact_size = assemblevdir.get_size()
    
    1770
    +            self.__artifacts.commit(self, assemblevdir, self.__get_cache_keys_for_commit())
    
    1771
    +
    
    1772
    +            if collect is not None and collectvdir is None:
    
    1773
    +                raise ElementError(
    
    1774
    +                    "Directory '{}' was not found inside the sandbox, "
    
    1775
    +                    "unable to collect artifact contents"
    
    1776
    +                    .format(collect))
    
    1773 1777
     
    
    1774 1778
             return artifact_size
    
    1775 1779
     
    

  • buildstream/storage/_casbaseddirectory.py
    ... ... @@ -136,10 +136,10 @@ class CasBasedDirectory(Directory):
    136 136
             the parent).
    
    137 137
     
    
    138 138
             """
    
    139
    -        self.ref = self.cas_cache.add_object(buffer=self.pb2_directory.SerializeToString())
    
    140 139
             if caller:
    
    141 140
                 old_dir = self._find_pb2_entry(caller.filename)
    
    142 141
                 self.cas_cache.add_object(digest=old_dir.digest, buffer=caller.pb2_directory.SerializeToString())
    
    142
    +        self.ref = self.cas_cache.add_object(buffer=self.pb2_directory.SerializeToString())
    
    143 143
             if self.parent:
    
    144 144
                 self.parent._recalculate_recursing_up(self)
    
    145 145
     
    
    ... ... @@ -277,14 +277,6 @@ class CasBasedDirectory(Directory):
    277 277
                                                              directory_list))
    
    278 278
             return None
    
    279 279
     
    
    280
    -    def find_root(self):
    
    281
    -        """ Finds the root of this directory tree by following 'parent' until there is
    
    282
    -        no parent. """
    
    283
    -        if self.parent:
    
    284
    -            return self.parent.find_root()
    
    285
    -        else:
    
    286
    -            return self
    
    287
    -
    
    288 280
         def _check_replacement(self, name, path_prefix, fileListResult):
    
    289 281
             """ Checks whether 'name' exists, and if so, whether we can overwrite it.
    
    290 282
             If we can, add the name to 'overwritten_files' and delete the existing entry.
    
    ... ... @@ -451,7 +443,7 @@ class CasBasedDirectory(Directory):
    451 443
                     files = external_pathspec.list_relative_paths()
    
    452 444
     
    
    453 445
             if isinstance(external_pathspec, FileBasedDirectory):
    
    454
    -            source_directory = external_pathspec.get_underlying_directory()
    
    446
    +            source_directory = external_pathspec._get_underlying_directory()
    
    455 447
                 result = self._import_files_from_directory(source_directory, files=files)
    
    456 448
             elif isinstance(external_pathspec, str):
    
    457 449
                 source_directory = external_pathspec
    
    ... ... @@ -635,6 +627,18 @@ class CasBasedDirectory(Directory):
    635 627
             self._recalculate_recursing_up()
    
    636 628
             self._recalculate_recursing_down()
    
    637 629
     
    
    630
    +    def get_size(self):
    
    631
    +        total = len(self.pb2_directory.SerializeToString())
    
    632
    +        for i in self.index.values():
    
    633
    +            if isinstance(i.buildstream_object, CasBasedDirectory):
    
    634
    +                total += i.buildstream_object.get_size()
    
    635
    +            elif isinstance(i.pb_object, remote_execution_pb2.FileNode):
    
    636
    +                src_name = self.cas_cache.objpath(i.pb_object.digest)
    
    637
    +                filesize = os.stat(src_name).st_size
    
    638
    +                total += filesize
    
    639
    +            # Symlink nodes are encoded as part of the directory serialization.
    
    640
    +        return total
    
    641
    +
    
    638 642
         def _get_identifier(self):
    
    639 643
             path = ""
    
    640 644
             if self.parent:
    
    ... ... @@ -653,3 +657,15 @@ class CasBasedDirectory(Directory):
    653 657
             throw an exception. """
    
    654 658
             raise VirtualDirectoryError("_get_underlying_directory was called on a CAS-backed directory," +
    
    655 659
                                         " which has no underlying directory.")
    
    660
    +
    
    661
    +    # _get_digest():
    
    662
    +    #
    
    663
    +    # Return the Digest for this directory.
    
    664
    +    #
    
    665
    +    # Returns:
    
    666
    +    #   (Digest): The Digest protobuf object for the Directory protobuf
    
    667
    +    #
    
    668
    +    def _get_digest(self):
    
    669
    +        if not self.ref:
    
    670
    +            self.ref = self.cas_cache.add_object(buffer=self.pb2_directory.SerializeToString())
    
    671
    +        return self.ref

  • buildstream/storage/_filebaseddirectory.py
    ... ... @@ -30,6 +30,7 @@ See also: :ref:`sandboxing`.
    30 30
     import os
    
    31 31
     import time
    
    32 32
     from .directory import Directory, VirtualDirectoryError
    
    33
    +from .. import utils
    
    33 34
     from ..utils import link_files, copy_files, list_relative_paths, _get_link_mtime, _magic_timestamp
    
    34 35
     from ..utils import _set_deterministic_user, _set_deterministic_mtime
    
    35 36
     
    
    ... ... @@ -201,6 +202,9 @@ class FileBasedDirectory(Directory):
    201 202
     
    
    202 203
             return list_relative_paths(self.external_directory)
    
    203 204
     
    
    205
    +    def get_size(self):
    
    206
    +        return utils._get_dir_size(self.external_directory)
    
    207
    +
    
    204 208
         def __str__(self):
    
    205 209
             # This returns the whole path (since we don't know where the directory started)
    
    206 210
             # which exposes the sandbox directory; we will have to assume for the time being
    

  • buildstream/storage/directory.py
    ... ... @@ -177,3 +177,9 @@ class Directory():
    177 177
     
    
    178 178
             """
    
    179 179
             raise NotImplementedError()
    
    180
    +
    
    181
    +    def get_size(self):
    
    182
    +        """ Get an approximation of the storage space in bytes used by this directory
    
    183
    +        and all files and subdirectories in it. Storage space varies by implementation
    
    184
    +        and effective space used may be lower than this number due to deduplication. """
    
    185
    +        raise NotImplementedError()



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]