[Notes] [Git][BuildStream/buildstream][raoul/802-refactor-artifactcache]

Raoul Hidalgo Charman pushed to branch raoul/802-refactor-artifactcache at BuildStream / buildstream

Commits:

9182b8f4

by Raoul Hidalgo Charman at 2019-01-02T11:34:46Z

artifactcache: implemented pull_tree similar to pull

New yield_directory_digests function implemented in CASRemote, which given a
tree digest can be iterated over, to give blob digests.

3 changed files:

buildstream/_artifactcache.py
buildstream/_cas/cascache.py
buildstream/_cas/casremote.py

Changes:

buildstream/_artifactcache.py

@@ -694,15 +694,26 @@ class ArtifactCache():
+     #
      # Args:
      #     project (Project): The current project
 -    #     digest (Digest): The digest of the tree
 +    #     tree_digest (Digest): The digest of the tree
+     #
 -    def pull_tree(self, project, digest):
 +    def pull_tree(self, project, tree_digest):
          for remote in self._remotes[project]:
 -            digest = self.cas.pull_tree(remote, digest)
 +            try:
 +                for blob_digest in remote.yield_tree_digests(tree_digest):
 +                    if self.cas.check_blob(blob_digest):
 +                        continue
 +                    remote.request_blob(blob_digest)
 +                    for blob_file in remote.get_blobs():
 +                        self.cas.add_object(path=blob_file.name, link_directly=True)
 -            if digest:
 -                # no need to pull from additional remotes
 -                return digest
 +                # Get the last batch
 +                for blob_file in remote.get_blobs(request_batch=True):
 +                    self.cas.add_object(path=blob_file.name, link_directly=True)
++
 +            except BlobNotFound:
 +                continue
 +            else:
 +                return tree_digest
          return None

buildstream/_cas/cascache.py

@@ -183,29 +183,6 @@ class CASCache():
          return modified, removed, added
 -    # pull_tree():
 -    #
 -    # Pull a single Tree rather than a ref.
 -    # Does not update local refs.
 -    #
 -    # Args:
 -    #     remote (CASRemote): The remote to pull from
 -    #     digest (Digest): The digest of the tree
 -    #
 -    def pull_tree(self, remote, digest):
 -        try:
 -            remote.init()
+-
 -            digest = self._fetch_tree(remote, digest)
+-
 -            return digest
+-
 -        except grpc.RpcError as e:
 -            if e.code() != grpc.StatusCode.NOT_FOUND:
 -                raise
+-
 -        return None
+-
      # link_ref():
+     #
      # Add an alias for an existing ref.
@@ -771,29 +748,6 @@ class CASCache():
          return objpath
 -    def _fetch_tree(self, remote, digest):
 -        # download but do not store the Tree object
 -        with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
 -            remote._fetch_blob(digest, out)
+-
 -            tree = remote_execution_pb2.Tree()
+-
 -            with open(out.name, 'rb') as f:
 -                tree.ParseFromString(f.read())
+-
 -            tree.children.extend([tree.root])
 -            for directory in tree.children:
 -                for filenode in directory.files:
 -                    self._ensure_blob(remote, filenode.digest)
+-
 -                # place directory blob only in final location when we've downloaded
 -                # all referenced blobs to avoid dangling references in the repository
 -                dirbuffer = directory.SerializeToString()
 -                dirdigest = self.add_object(buffer=dirbuffer)
 -                assert dirdigest.size_bytes == len(dirbuffer)
+-
 -        return dirdigest
+-
      def _send_directory(self, remote, digest, u_uid=uuid.uuid4()):
          required_blobs = self._required_blobs(digest)

buildstream/_cas/casremote.py

@@ -281,19 +281,29 @@ class CASRemote():
              else:
                  return None
 +    def get_tree_blob(self, tree_digest):
 +        self.init()
 +        f = tempfile.NamedTemporaryFile(dir=self.tmpdir)
 +        self._fetch_blob(tree_digest, f)
++
 +        tree = remote_execution_pb2.Tree()
 +        with open(f.name, 'rb') as tmp:
 +            tree.ParseFromString(tmp.read())
++
 +        return tree
++
      # yield_directory_digests():
+     #
      # Iterate over blobs digests starting from a root digest
+     #
      # Args:
 -    #     root_digest (str): The root_digest to get a tree of
 +    #     root_digest (digest): The root_digest to get a tree of
      #     progress (callable): The progress callback, if any
      #     subdir (str): The optional specific subdir to pull
      #     excluded_subdirs (list): The optional list of subdirs to not pull
+     #
      # Returns:
 -    #   (iter): True if pull was successful, False if ref was not available
 -    #
 +    #     (iter digests): recursively iterates over digests contained in root directory
      def yield_directory_digests(self, root_digest, *, progress=None, subdir=None, excluded_subdirs=[]):
          self.init()
@@ -301,6 +311,37 @@ class CASRemote():
          # Fetch artifact, excluded_subdirs determined in pullqueue
          yield from self._yield_directory_digests(root_digest, excluded_subdirs=excluded_subdirs)
 +    # yield_tree_digests():
 +    #
 +    # Fetches a tree file from digests and then iterates over child digests
 +    #
 +    # Args:
 +    #     tree_digest (digest): tree digest
 +    #
 +    # Returns:
 +    #     (iter digests): iterates over digests in tree message
 +    def yield_tree_digests(self, tree_digest):
 +        self.init()
++
 +        # get tree file
 +        f = tempfile.NamedTemporaryFile(dir=self.tmpdir)
 +        self._fetch_blob(tree_digest, f)
 +        tree = remote_execution_pb2.Tree()
 +        tree.ParseFromString(f.read())
++
 +        tree.children.extend([tree.root])
 +        for directory in tree.children:
 +            for filenode in directory.files:
 +                yield filenode.digest
++
 +            # add the directory to downloaded tmp files to be added
 +            f2 = tempfile.NamedTemporaryFile(dir=self.tmpdir)
 +            f2.write(directory.SerializeToString())
 +            self.__tmp_downloads.append(f2)
++
 +        # Add the tree directory to downloads right at the end
 +        self.__tmp_downloads.append(f)
++
      # request_blob():
+     #
      # Request blob, triggering download depending via bytestream or cas

[Notes] [Git][BuildStream/buildstream][raoul/802-refactor-artifactcache] artifactcache: implemented pull_tree similar to pull

Raoul Hidalgo Charman pushed to branch raoul/802-refactor-artifactcache at BuildStream / buildstream

Commits:

3 changed files:

Changes: