[Notes] [Git][BuildStream/buildstream][jmac/cas_to_cas_oct

Jim MacArthur pushed to branch jmac/cas_to_cas_oct_v2 at BuildStream / buildstream

Commits:

505c6ddc

by Jim MacArthur at 2018-10-29T14:25:44Z

Add a main() to virtual_directory_test.py to allow manual testing

0f429c9b

by Jim MacArthur at 2018-10-30T09:39:42Z

CasBasedDirectory: Remove 6 functions and rename files_in_subdir -> _files_in_subdir

f2068505

by Jim MacArthur at 2018-10-30T09:59:02Z

_casbaseddirectory.py: _resolve_symlink_or_directory -> _force_resolve

1a0669a2

by Jim MacArthur at 2018-10-30T10:00:26Z

casbaseddirectory: Replace one instance of _force_resolve with descend

c74845da
by Jim MacArthur at 2018-10-30T10:25:51Z
```
Remove create_directory
```
3cced840
by Jim MacArthur at 2018-10-30T11:05:19Z
```
virtual_directory_test: PEP8
```
7bb8f20f
by Jim MacArthur at 2018-10-30T11:17:21Z
```
Remove some prints and whitespace
```

3c704967

by Jim MacArthur at 2018-10-30T11:17:41Z

Rename _add_new_link and remove duplicated code

545219dc
by Jim MacArthur at 2018-10-30T11:48:44Z
```
Remove some prints and improve comments
```

c9cfcf02

by Jim MacArthur at 2018-10-30T12:19:23Z

_casbaseddirectory: Restructure resolve to make it a bit more logical

285b6f33
by Jim MacArthur at 2018-10-30T12:21:55Z
```
Remove _symlink_target_is_directory
```
68e10546
by Jim MacArthur at 2018-10-30T12:34:18Z
```
Restructure of .import()
```

2 changed files:

buildstream/storage/_casbaseddirectory.py
tests/storage/virtual_directory_import.py

Changes:

buildstream/storage/_casbaseddirectory.py

@@ -87,9 +87,7 @@ class CasBasedDirectory(Directory):
          if ref:
              with open(self.cas_cache.objpath(ref), 'rb') as f:
                  self.pb2_directory.ParseFromString(f.read())
 -                print("Opening ref {} and parsed into directory containing: {} {} {}.".format(ref.hash, [d.name for d in self.pb2_directory.directories],
 -                                                                                        [d.name for d in self.pb2_directory.symlinks],
 -                                                                                        [d.name for d in self.pb2_directory.files]))
++
          self.ref = ref
          self.index = OrderedDict()
          self.parent = parent
@@ -141,41 +139,6 @@ class CasBasedDirectory(Directory):
          # We don't need to do anything more than that; files were already added ealier, and symlinks are
          # part of the directory structure.
 -    def _add_new_blank_directory(self, name) -> Directory:
 -        bst_dir = CasBasedDirectory(self.context, parent=self, filename=name)
 -        new_pb2_dirnode = self.pb2_directory.directories.add()
 -        new_pb2_dirnode.name = name
 -        # Calculate the hash for an empty directory
 -        if name in self.index:
 -            raise VirtualDirectoryError("Creating directory {} would overwrite an existing item in {}"
 -                                        .format(name, str(self)))
 -        new_pb2_directory = remote_execution_pb2.Directory()
 -        self.cas_cache.add_object(digest=new_pb2_dirnode.digest, buffer=new_pb2_directory.SerializeToString())
 -        self.index[name] = IndexEntry(new_pb2_dirnode, buildstream_object=bst_dir)
 -        return bst_dir
+-
 -    def create_directory(self, name: str) -> Directory:
 -        """Creates a directory if it does not already exist. This does not
 -        cause an error if something exists; it will remove files and
 -        symlinks to files which have the same name in this
 -        directory. Symlinks to directories with the name 'name' are
 -        unaltered; it's assumed that the target of that symlink will
 -        be used.
+-
 -        """
 -        existing_item = self._find_pb2_entry(name)
 -        if isinstance(existing_item, remote_execution_pb2.FileNode):
 -            # Directory imported over file with same name
 -            self.delete_entry(name)
 -        elif isinstance(existing_item, remote_execution_pb2.SymlinkNode):
 -            # Directory imported over symlink with same source name
 -            if self.symlink_target_is_directory(existing_item):
 -                return self._resolve_symlink_or_directory(name) # That's fine; any files in the source directory should end up at the target of the symlink.
 -            else:
 -                self.delete_entry(name) # Symlinks to files get replaced
 -        return self.descend(name, create=True) # Creates the directory if it doesn't already exist.
+-
+-
      def _find_pb2_entry(self, name):
          if name in self.index:
              return self.index[name].pb_object
@@ -217,16 +180,8 @@ class CasBasedDirectory(Directory):
          filenode.is_executable = is_executable
          self.index[filename] = IndexEntry(filenode, modified=(filename in self.index))
 -    def _add_new_link(self, basename, filename):
 -        existing_link = self._find_pb2_entry(filename)
 -        if existing_link:
 -            symlinknode = existing_link
 -        else:
 -            symlinknode = self.pb2_directory.symlinks.add()
 -        symlinknode.name = filename
 -        # A symlink node has no digest.
 -        symlinknode.target = os.readlink(os.path.join(basename, filename))
 -        self.index[filename] = IndexEntry(symlinknode, modified=(existing_link is not None))
 +    def _copy_link_from_filesystem(self, basename, filename):
 +        self._add_new_link_direct(filename, os.readlink(os.path.join(basename, filename)))
      def _add_new_link_direct(self, name, target):
          existing_link = self._find_pb2_entry(name)
@@ -240,7 +195,6 @@ class CasBasedDirectory(Directory):
          symlinknode.target = target
          self.index[name] = IndexEntry(symlinknode, modified=(existing_link is not None))
+-
      def delete_entry(self, name):
          for collection in [self.pb2_directory.files, self.pb2_directory.symlinks, self.pb2_directory.directories]:
              for thing in collection:
@@ -297,9 +251,6 @@ class CasBasedDirectory(Directory):
          else:
              if create:
                  newdir = self._add_directory(subdirectory_spec[0])
 -                print("Created new directory called {} and descending into it".format(subdirectory_spec[0]))
 -                #if subdirectory_spec[0] == "broken":
 -                #    assert False
                  return newdir.descend(subdirectory_spec[1:], create)
              else:
                  error = "No entry called '{}' found in {}. There are directories called {}."
@@ -316,7 +267,7 @@ class CasBasedDirectory(Directory):
          else:
              return self
 -    def _resolve_symlink_or_directory(self, name):
 +    def _force_resolve(self, name):
          """Used only by _import_files_from_directory. Tries to resolve a
          directory name or symlink name. 'name' must be an entry in this
          directory. It must be a single symlink or directory name, not a path
@@ -338,28 +289,31 @@ class CasBasedDirectory(Directory):
          if isinstance(self.index[name].buildstream_object, Directory):
              return True
          target = self._resolve(name)
 -        print("Is {} followable? Resolved to {}".format(name, target))
 -        return isinstance(target, CasBasedDirectory) or target is None
 +        return isinstance(target, CasBasedDirectory) or target is None  #  TODO: But why return True if it's None (broken link/circular loop)? Surely that is against the docstring.
 -    def _resolve_symlink(self, node, force_create=True):
 -        """Same as _resolve_symlink_or_directory but takes a SymlinkNode.
 -        """
 -        return self._resolve(node.name, force_create=True)
+-
      def _resolve(self, name, absolute_symlinks_resolve=True, force_create=False, first_seen_object = None):
          """ Resolves any name to an object. If the name points to a symlink in
          this directory, it returns the thing it points to,
 -        recursively. Returns a CasBasedDirectory, FileNode or
 -        None.
 +        recursively.
++
 +        Returns a CasBasedDirectory, FileNode or
 +        None. None indicates any of these cases:
 +        * 'name' does not exist in this directory
 +        * 'name' is a broken symlink,
 +        * 'name' points to an infinite symlink loop.
 +        * 'name' points to an absolute symlink and absolute_symlinks_resolve is False.
          If force_create is on, will attempt to create directories to make symlinks and directories resolve.
          If force_create is off, this will never alter this directory.
          """
 -        # First check if it's a normal object and return that
 +        # TODO: first_seen_object isn't sufficient. We could get into a loop after following one link and not detect it.
 +        # TODO: 'None' is overloaded, maybe we should use exceptions and leave 'None' for actual nonexistent things.
          if name not in self.index:
              return None
++
 +        # First check if it's a normal object and return that
          index_entry = self.index[name]
          if isinstance(index_entry.buildstream_object, Directory):
              return index_entry.buildstream_object
@@ -375,7 +329,6 @@ class CasBasedDirectory(Directory):
                  ### Infinite symlink loop detected ###
                  return None
 -        print("Resolving '{}': This is a symlink node in the current directory.".format(name))
          symlink = index_entry.pb_object
          components = symlink.target.split(CasBasedDirectory._pb2_path_sep)
@@ -386,12 +339,12 @@ class CasBasedDirectory(Directory):
                  # Discard the first empty element
                  components.pop(0)
              else:
 -                print("  _resolve: Absolute symlink, which we won't resolve.")
 +                # Unresolvable absolute symlink
                  return None
          else:
              start_directory = self
++
          directory = start_directory
 -        print("Resolve {}: starting from {}".format(symlink.target, start_directory))
          while True:
              if not components:
                  # We ran out of path elements and ended up in a directory
@@ -407,30 +360,37 @@ class CasBasedDirectory(Directory):
                  # returns the root.
              else:
                  if c in directory.index:
 +                    # Recursive resolve and continue
                      f = directory._resolve(c, absolute_symlinks_resolve, first_seen_object=first_seen_object)
 -                    # Ultimately f must now be a file or directory
++
                      if isinstance(f, CasBasedDirectory):
                          directory = f
+-
 +                    elif isinstance(f, remote_execution_pb2.FileNode):
 +                        # F is a file
 +                        if components:
 +                            # We have components still to resolve, but one of the path components
 +                            # is a file.
 +                            if force_create:
 +                                self.delete_entry(c)
 +                                directory = directory.descend(c, create=True)
 +                            else:
 +                                return f # TODO: Why return f? We've got components left and hit a file; this should be an error.
 +                                #raise VirtualDirectoryError("Reached a file called {} while trying to resolve a symlink; cannot proceed".format(c))
 +                        else:
 +                            # It's a file, but there's no components left, so just return that.
 +                            return f
                      else:
 -                        # This is a file or None (i.e. broken symlink)
 -                        if f is None and force_create:
 -                            directory = directory.descend(c, create=True)
 -                        elif components and force_create:
 -                            # Oh dear. We have components left to resolve, but the one we're trying to resolve points to a file.
 -                            print("Trying to resolve {}, but found {} was a file.".format(symlink.target, c))
 -                            self.delete_entry(c)
 +                        # f is none, which covers many cases
 +                        if force_create:
                              directory = directory.descend(c, create=True)
 -                            #raise VirtualDirectoryError("Reached a file called {} while trying to resolve a symlink; cannot proceed".format(c))
                          else:
 -                            return f
 +                            return None
                  else:
                      if force_create:
                          directory = directory.descend(c, create=True)
                      else:
                          return None
+-
 -        # Shouldn't get here.
 +        # You can only exit the while loop with a return, so you shouldn't be here.
      def _check_replacement(self, name, path_prefix, fileListResult):
@@ -471,7 +431,7 @@ class CasBasedDirectory(Directory):
          as a directory tree is descended. """
          if directory_name in self.index:
              if self._is_followable(directory_name):
 -                subdir = self._resolve_symlink_or_directory(directory_name)
 +                subdir = self._force_resolve(directory_name)
              else:
                  print("Overwriting unfollowable thing {}".format(directory_name))
                  self.delete_entry(directory_name)
@@ -504,7 +464,7 @@ class CasBasedDirectory(Directory):
                  result.combine(subdir_result)
              elif os.path.islink(import_file):
                  if self._check_replacement(entry, path_prefix, result):
 -                    self._add_new_link(source_directory, entry)
 +                    self._copy_link_from_filesystem(source_directory, entry)
                      result.files_written.append(relative_pathname)
              elif os.path.isdir(import_file):
                  # A plain directory which already exists isn't a problem; just ignore it.
@@ -517,47 +477,7 @@ class CasBasedDirectory(Directory):
          return result
 -    def _save(self, name):
 -        """ Saves this directory into the content cache as a named ref. This function is not
 -        currently in use, but may be useful later. """
 -        self._recalculate_recursing_up()
 -        self._recalculate_recursing_down()
 -        (rel_refpath, refname) = os.path.split(name)
 -        refdir = os.path.join(self.cas_directory, 'refs', 'heads', rel_refpath)
 -        refname = os.path.join(refdir, refname)
+-
 -        if not os.path.exists(refdir):
 -            os.makedirs(refdir)
 -        with open(refname, "wb") as f:
 -            f.write(self.ref.SerializeToString())
+-
 -    def find_updated_files(self, modified_directory, prefix=""):
 -        """Find the list of written and overwritten files that would result
 -        from importing 'modified_directory' into this one.  This does
 -        not change either directory. The reason this exists is for
 -        direct imports of cas directories into other ones, which can
 -        be done by simply replacing a hash, but we still need the file
 -        lists.
+-
 -        """
 -        result = FileListResult()
 -        for entry in modified_directory.pb2_directory.directories:
 -            existing_dir = self._find_pb2_entry(entry.name)
 -            if existing_dir:
 -                updates_files = existing_dir.find_updated_files(modified_directory.descend(entry.name),
 -                                                                os.path.join(prefix, entry.name))
 -                result.combine(updated_files)
 -            else:
 -                for f in source_directory.descend(entry.name).list_relative_paths():
 -                    result.files_written.append(os.path.join(prefix, f))
 -                    # None of these can overwrite anything, since the original files don't exist
 -        for entry in modified_directory.pb2_directory.files + modified_directory.pb2_directory.symlinks:
 -            if self._find_pb2_entry(entry.name):
 -                result.files_overwritten.apppend(os.path.join(prefix, entry.name))
 -            result.file_written.apppend(os.path.join(prefix, entry.name))
 -        return result
+-
 -    def files_in_subdir(sorted_files, dirname):
 +    def _files_in_subdir(sorted_files, dirname):
          """Filters sorted_files and returns only the ones which have
             'dirname' as a prefix, with that prefix removed.
@@ -566,9 +486,6 @@ class CasBasedDirectory(Directory):
              dirname += os.path.sep
          return [f[len(dirname):] for f in sorted_files if f.startswith(dirname)]
 -    def symlink_target_is_directory(self, symlink_node):
 -        x = self._resolve_symlink(symlink_node, force_create=False)
 -        return isinstance(x, CasBasedDirectory)
      def _partial_import_cas_into_cas(self, source_directory, files, path_prefix="", file_list_required=True):
          """ Import only the files and symlinks listed in 'files' from source_directory to this one.
@@ -589,7 +506,7 @@ class CasBasedDirectory(Directory):
                  dirname = components[0]
                  if dirname not in processed_directories:
                      # Now strip off the first directory name and import files recursively.
 -                    subcomponents = CasBasedDirectory.files_in_subdir(files, dirname)
 +                    subcomponents = CasBasedDirectory._files_in_subdir(files, dirname)
                      # We will fail at this point if there is a file or symlink to file called 'dirname'.
                      if dirname in self.index:
                          x = self._resolve(dirname, force_create=True)
@@ -600,8 +517,7 @@ class CasBasedDirectory(Directory):
                          else:
                              dest_subdir = x
                      else:
 -                        self.create_directory(dirname)
 -                        dest_subdir = self._resolve_symlink_or_directory(dirname)
 +                        dest_subdir = self.descend(dirname, create=True)
                      src_subdir = source_directory.descend(dirname)
                      import_result = dest_subdir._partial_import_cas_into_cas(src_subdir, subcomponents,
                                                                               path_prefix=fullname, file_list_required=file_list_required)
@@ -622,7 +538,7 @@ class CasBasedDirectory(Directory):
                          # There's either a symlink (valid or not) or existing directory with this name, so do nothing.
                          pass
                  else:
 -                    self.create_directory(f)
 +                    self.descend(f, create=True)
              else:
                  # We're importing a file or symlink - replace anything with the same name.
                  print("Import of file/symlink {} into this directory. Removing anything existing...".format(f))
@@ -639,49 +555,11 @@ class CasBasedDirectory(Directory):
                          self._add_new_link_direct(name=f, target=item.target)
          return result
 -    def transfer_node_contents(destination, source):
 -        """Transfers all fields from the source PB2 node into the
 -        destination. Destination and source must be of the same type and must
 -        be a FileNode, SymlinkNode or DirectoryNode.
 -        """
 -        assert(type(destination) == type(source))
 -        destination.name = source.name
 -        if isinstance(destination, remote_execution_pb2.FileNode):
 -            destination.digest.hash = source.digest.hash
 -            destination.digest.size_bytes = source.digest.size_bytes
 -            destination.is_executable = source.is_executable
 -        elif isinstance(destination, remote_execution_pb2.SymlinkNode):
 -            destination.target = source.target
 -        elif isinstance(destination, remote_execution_pb2.DirectoryNode):
 -            destination.digest.hash = source.digest.hash
 -            destination.digest.size_bytes = source.digest.size_bytes
 -        else:
 -            raise VirtualDirectoryError("Incompatible type '{}' used as destination for transfer_node_contents"
 -                                        .format(destination.type))
+-
 -    def _add_directory_from_node(self, source_node, source_casdir, can_hardlink=False):
 -        # Duplicate the given node and add it to our index with a CasBasedDirectory object.
 -        # No existing entry with the source node's name can exist.
 -        # source_casdir is only needed if can_hardlink is True.
 -        assert(self._find_pb2_entry(source_node.name) is None)
+-
 -        if can_hardlink:
 -            new_dir_node = self.pb2_directory.directories.add()
 -            CasBasedDirectory.transfer_node_contents(new_dir_node, source_node)
 -            self.index[source_node.name] = IndexEntry(source_node, buildstream_object=source_casdir, modified=True)
 -        else:
 -            new_dir_node = self.pb2_directory.directories.add()
 -            CasBasedDirectory.transfer_node_contents(new_dir_node, source_node)
 -            buildStreamDirectory = CasBasedDirectory(self.context, ref=source_node.digest,
 -                                                     parent=self, filename=source_node.name)
 -            self.index[source_node.name] = IndexEntry(source_node, buildstream_object=buildStreamDirectory, modified=True)
+-
      def _import_cas_into_cas(self, source_directory, files=None):
          """ A full import is significantly quicker than a partial import, because we can just
          replace one directory with another's hash, without doing any recursion.
          """
 -        if files is None:
 -            files = source_directory.list_relative_paths()
++
          # You must pass a list into _partial_import (not a generator)
          return self._partial_import_cas_into_cas(source_directory, list(files))
@@ -771,27 +649,27 @@ class CasBasedDirectory(Directory):
          can_link (bool): Ignored, since hard links do not have any meaning within CAS.
          """
 -        print("Directory before import: {}".format(self.show_files_recursive()))
+-
 -        if isinstance(external_pathspec, CasBasedDirectory):
 -            print("-"*80 + "Performing direct CAS-to-CAS import")
 -            result = self._import_cas_into_cas(external_pathspec, files=files)
 -            print("Result of cas-to-cas import: {}".format(self.show_files_recursive()))
 -        else:
 -            print("-"*80 + "Performing initial import")
 -            if isinstance(external_pathspec, FileBasedDirectory):
 -                source_directory = external_pathspec.get_underlying_directory()
 -            else:
 -                source_directory = external_pathspec
 -            if files is None:
 +        if files is None:
 +            if isinstance(external_pathspec, str):
                  files = list_relative_paths(external_pathspec)
 +            else:
 +                assert isinstance(external_pathspec, Directory)
 +                files = external_pathspec.list_relative_paths()
++
 +        if isinstance(external_pathspec, FileBasedDirectory):
 +            source_directory = external_pathspec.get_underlying_directory()
 +            result = self._import_files_from_directory(source_directory, files=files)
 +        elif isinstance(external_pathspec, str):
 +            source_directory = external_pathspec
              result = self._import_files_from_directory(source_directory, files=files)
 +        else:
 +            assert isinstance(external_pathspec, CasBasedDirectory)
 +            result = self._import_cas_into_cas(external_pathspec, files=files)
          # TODO: No notice is taken of report_written, update_utimes or can_link.
          # Current behaviour is to fully populate the report, which is inefficient,
          # but still correct.
+-
          # We need to recalculate and store the hashes of all directories both
          # up and down the tree; we have changed our directory by importing files
          # which changes our hash and all our parents' hashes of us. The trees
@@ -919,12 +797,6 @@ class CasBasedDirectory(Directory):
                  filelist.append(k)
          return filelist
 -    def _contains_only_directories(self):
 -        for (k, v) in self.index.items():
 -            if not isinstance(v.buildstream_object, CasBasedDirectory):
 -                return False
 -        return True
+-
      def list_relative_paths(self, relpath=""):
          """Provide a list of all relative paths.

tests/storage/virtual_directory_import.py

@@ -8,8 +8,11 @@ from tests.testutils import cli
  from buildstream.storage import CasBasedDirectory
  from buildstream.storage import FileBasedDirectory
 +from buildstream._artifactcache import ArtifactCache
 +from buildstream._artifactcache.cascache import CASCache
  from buildstream import utils
++
  class FakeContext():
      def __init__(self):
          self.config_cache_quota = "65536"
@@ -29,8 +32,8 @@ root_filesets = [
      [('a/b/d', 'D', '')],
      [('a/b/c', 'S', '/a/b/d')],
      [('a/b/d', 'S', '/a/b/c')],
 -    [('a/b/d', 'D', ''), ('a/b/c', 'S', '/a/b/d')],
 -    [('a/b/c', 'D', ''), ('a/b/d', 'S', '/a/b/c')],
 +    [('a/b/d', 'D', ''), ('a/b/c', 'S', '/a/b/d')],
 +    [('a/b/c', 'D', ''), ('a/b/d', 'S', '/a/b/c')],
      [('a/b', 'F', 'This is textfile 1\n')],
      [('a/b/c', 'F', 'This is textfile 1\n')],
      [('a/b/c', 'D', '')]
@@ -59,7 +62,7 @@ def generate_import_roots(rootno, directory):
  def generate_random_root(rootno, directory):
 -    random.seed(RANDOM_SEED+rootno)
 +    random.seed(RANDOM_SEED + rootno)
      rootname = "root{}".format(rootno)
      rootdir = os.path.join(directory, "content", rootname)
      things = []
@@ -156,14 +159,15 @@ def directory_not_empty(path):
  def _import_test(tmpdir, original, overlay, generator_function, verify_contents=False):
      fake_context = FakeContext()
      fake_context.artifactdir = tmpdir
 +    print("Creating CAS Cache with artifact dir {}".format(tmpdir))
 +    fake_context.artifactcache = CASCache(fake_context)
      # Create some fake content
      generator_function(original, tmpdir)
      if original != overlay:
          generator_function(overlay, tmpdir)
+-
++
      d = create_new_casdir(original, fake_context, tmpdir)
 -    #duplicate_cas = CasBasedDirectory(fake_context, ref=copy.copy(d.ref))
      duplicate_cas = create_new_casdir(original, fake_context, tmpdir)
      assert duplicate_cas.ref.hash == d.ref.hash
@@ -175,7 +179,7 @@ def _import_test(tmpdir, original, overlay, generator_function, verify_contents=
      roundtrip_dir = os.path.join(tmpdir, "roundtrip")
      d2.export_files(roundtrip_dir)
      d.export_files(export_dir)
+-
++
      if verify_contents:
          for item in root_filesets[overlay - 1]:
              (path, typename, content) = item
@@ -195,8 +199,10 @@ def _import_test(tmpdir, original, overlay, generator_function, verify_contents=
                      assert os.path.islink(realpath)
                      assert os.readlink(realpath) == content
              elif typename == 'D':
 -                # We can't do any more tests than this because it depends on things present in the original. Blank directories
 -                # here will be ignored and the original left in place.
 +                # We can't do any more tests than this because it
 +                # depends on things present in the original. Blank
 +                # directories here will be ignored and the original
 +                # left in place.
                  assert os.path.lexists(realpath)
      # Now do the same thing with filebaseddirectories and check the contents match
@@ -211,23 +217,22 @@ def _import_test(tmpdir, original, overlay, generator_function, verify_contents=
      assert duplicate_cas.ref.hash == d.ref.hash
 -    #d3 = create_new_casdir(original, fake_context, tmpdir)
 -    #d4 = create_new_filedir(overlay, tmpdir)
 -    #d3.import_files(d2)
 -    #assert d.ref.hash == d3.ref.hash
 -@pytest.mark.parametrize("original,overlay", combinations(range(1,len(root_filesets)+1)))
 +@pytest.mark.parametrize("original,overlay", combinations(range(1, len(root_filesets) + 1)))
  def test_fixed_cas_import(cli, tmpdir, original, overlay):
      _import_test(tmpdir, original, overlay, generate_import_roots, verify_contents=True)
 -@pytest.mark.parametrize("original,overlay", combinations(range(1,11)))
++
 +@pytest.mark.parametrize("original,overlay", combinations(range(1, 11)))
  def test_random_cas_import_fast(cli, tmpdir, original, overlay):
      _import_test(tmpdir, original, overlay, generate_random_root, verify_contents=False)
+-
++
  def _listing_test(tmpdir, root, generator_function):
      fake_context = FakeContext()
      fake_context.artifactdir = tmpdir
 +    print("Creating CAS Cache with artifact dir {}".format(tmpdir))
 +    fake_context.artifactcache = CASCache(fake_context)
      # Create some fake content
      generator_function(root, tmpdir)
@@ -242,12 +247,37 @@ def _listing_test(tmpdir, root, generator_function):
      print("filelist for root {} via CasBasedDirectory:".format(root))
      print("{}".format(filelist2))
      assert filelist == filelist2
+-
 -@pytest.mark.parametrize("root", range(1,11))
++
 +@pytest.mark.parametrize("root", range(1, 11))
  def test_random_directory_listing(cli, tmpdir, root):
      _listing_test(tmpdir, root, generate_random_root)
+-
++
++
  @pytest.mark.parametrize("root", [1, 2, 3, 4, 5])
  def test_fixed_directory_listing(cli, tmpdir, root):
      _listing_test(tmpdir, root, generate_import_roots)
++
++
 +def main():
 +    for i in range(1, 6):
 +        with tempfile.TemporaryDirectory(prefix="/home/jimmacarthur/.cache/buildstream/cas") as tmpdirname:
 +            test_fixed_directory_listing(None, tmpdirname, i)
++
 +    for i in range(1, 11):
 +        with tempfile.TemporaryDirectory(prefix="/home/jimmacarthur/.cache/buildstream/cas") as tmpdirname:
 +            test_random_directory_listing(None, tmpdirname, i)
++
 +    for i in range(1, 21):
 +        for j in range(1, 21):
 +            with tempfile.TemporaryDirectory(prefix="/home/jimmacarthur/.cache/buildstream/cas") as tmpdirname:
 +                test_random_cas_import_fast(None, tmpdirname, i, j)
++
 +    for i in range(1, len(root_filesets) + 1):
 +        for j in range(1, len(root_filesets) + 1):
 +            with tempfile.TemporaryDirectory(prefix="/home/jimmacarthur/.cache/buildstream/cas") as tmpdirname:
 +                test_fixed_cas_import(None, tmpdirname, i, j)
++
++
 +if __name__ == "__main__":
 +    main()

[Notes] [Git][BuildStream/buildstream][jmac/cas_to_cas_oct_v2] 12 commits: Add a main() to virtual_directory_test.py to allow manual testing

Jim MacArthur pushed to branch jmac/cas_to_cas_oct_v2 at BuildStream / buildstream

Commits:

2 changed files:

Changes: