[Notes] [Git][BuildStream/buildstream][jmac/cas_to_cas

Jim MacArthur pushed to branch jmac/cas_to_cas_oct at BuildStream / buildstream

Commits:

010565db

by Jim MacArthur at 2018-10-19T17:28:45Z

Add a tool to show differences in two CAS directories

957539a0
by Jim MacArthur at 2018-10-23T15:16:18Z
```
Correct deleting and overwriting cases
```
e1b047ef
by Jim MacArthur at 2018-10-23T16:56:12Z
```
Fix 'remove_item'->delete_entry
```
a8c76395
by Jim MacArthur at 2018-10-23T16:57:16Z
```
casbaseddirectory: Various fixes.
```

74650bb1

by Jim MacArthur at 2018-10-23T16:57:37Z

Virtual directory test: Expand random testing to 6 roots

241f8cc0

by Jim MacArthur at 2018-10-23T16:57:54Z

CASBasedDirectory: Do not sort the input file list!

4a9924b5
by Jim MacArthur at 2018-10-23T16:58:07Z
```
CASBasedDirectory: Import '.'
```

2 changed files:

buildstream/storage/_casbaseddirectory.py
tests/storage/virtual_directory_import.py

Changes:

buildstream/storage/_casbaseddirectory.py

@@ -40,6 +40,8 @@ from ._filebaseddirectory import FileBasedDirectory
  from ..utils import FileListResult, safe_copy, list_relative_paths, _relative_symlink_target
  from .._artifactcache.cascache import CASCache
 +import copy # Temporary
 +import operator
  class IndexEntry():
      """ Used in our index of names to objects to store the 'modified' flag
@@ -84,7 +86,9 @@ class CasBasedDirectory(Directory):
          if ref:
              with open(self.cas_cache.objpath(ref), 'rb') as f:
                  self.pb2_directory.ParseFromString(f.read())
+-
 +                print("Opening ref {} and parsed into directory containing: {} {} {}.".format(ref.hash, [d.name for d in self.pb2_directory.directories],
 +                                                                                        [d.name for d in self.pb2_directory.symlinks],
 +                                                                                        [d.name for d in self.pb2_directory.files]))
          self.ref = ref
          self.index = OrderedDict()
          self.parent = parent
@@ -161,13 +165,13 @@ class CasBasedDirectory(Directory):
          existing_item = self._find_pb2_entry(name)
          if isinstance(existing_item, remote_execution_pb2.FileNode):
              # Directory imported over file with same name
 -            self.remove_item(name)
 +            self.delete_entry(name)
          elif isinstance(existing_item, remote_execution_pb2.SymlinkNode):
              # Directory imported over symlink with same source name
              if self.symlink_target_is_directory(existing_item):
                  return self._resolve_symlink_or_directory(name) # That's fine; any files in the source directory should end up at the target of the symlink.
              else:
 -                self.remove_item(name) # Symlinks to files get replaced
 +                self.delete_entry(name) # Symlinks to files get replaced
          return self.descend(name, create=True) # Creates the directory if it doesn't already exist.
@@ -223,11 +227,27 @@ class CasBasedDirectory(Directory):
          symlinknode.target = os.readlink(os.path.join(basename, filename))
          self.index[filename] = IndexEntry(symlinknode, modified=(existing_link is not None))
 +    def _add_new_link_direct(self, name, target):
 +        existing_link = self._find_pb2_entry(name)
 +        if existing_link:
 +            symlinknode = existing_link
 +        else:
 +            symlinknode = self.pb2_directory.symlinks.add()
 +        assert(isinstance(symlinknode, remote_execution_pb2.SymlinkNode))
 +        symlinknode.name = name
 +        # A symlink node has no digest.
 +        symlinknode.target = target
 +        self.index[name] = IndexEntry(symlinknode, modified=(existing_link is not None))
++
++
      def delete_entry(self, name):
          for collection in [self.pb2_directory.files, self.pb2_directory.symlinks, self.pb2_directory.directories]:
 -            if name in collection:
 -                collection.remove(name)
 +            for thing in collection:
 +                if thing.name == name:
 +                    print("Removing {} from PB2".format(name))
 +                    collection.remove(thing)
          if name in self.index:
 +            print("Removing {} from index".format(name))
              del self.index[name]
      def descend(self, subdirectory_spec, create=False):
@@ -268,9 +288,12 @@ class CasBasedDirectory(Directory):
                  return entry.descend(subdirectory_spec[1:], create)
              else:
                  # May be a symlink
 +                target = self._resolve(subdirectory_spec[0])
 +                if isinstance(target, CasBasedDirectory):
 +                    return target
                  error = "Cannot descend into {}, which is a '{}' in the directory {}"
                  raise VirtualDirectoryError(error.format(subdirectory_spec[0],
 -                                                         type(entry).__name__,
 +                                                         type(self.index[subdirectory_spec[0]].pb_object).__name__,
                                                           self))
          else:
              if create:
@@ -308,6 +331,7 @@ class CasBasedDirectory(Directory):
              return self.index[name].buildstream_object
          # OK then, it's a symlink
          symlink = self._find_pb2_entry(name)
 +        assert isinstance(symlink, remote_execution_pb2.SymlinkNode)
          absolute = symlink.target.startswith(CasBasedDirectory._pb2_absolute_path_prefix)
          if absolute:
              root = self.find_root()
@@ -324,6 +348,16 @@ class CasBasedDirectory(Directory):
                  directory = directory.descend(c, create=True)
          return directory
 +    def _is_followable(self, name):
 +        """ Returns true if this is a directory or symlink to a valid directory. """
 +        if name not in self.index:
 +            return False
 +        if isinstance(self.index[name].buildstream_object, Directory):
 +            return True
 +        target = self._resolve(name)
 +        print("Is {} followable? Resolved to {}".format(name, target))
 +        return isinstance(target, CasBasedDirectory) or target is None
++
      def _resolve_symlink(self, node):
          """Same as _resolve_symlink_or_directory but takes a SymlinkNode.
          """
@@ -348,8 +382,13 @@ class CasBasedDirectory(Directory):
      def _resolve(self, name, absolute_symlinks_resolve=True):
 -        """ Resolves any name to an object. If the name points to a symlink in this
 -        directory, it returns the thing it points to, recursively. Returns a CasBasedDirectory, FileNode or None. Never creates a directory or otherwise alters the directory. """
 +        """ Resolves any name to an object. If the name points to a symlink in
 +        this directory, it returns the thing it points to,
 +        recursively. Returns a CasBasedDirectory, FileNode or
 +        None. Never creates a directory or otherwise alters the
 +        directory.
++
 +        """
          # First check if it's a normal object and return that
          if name not in self.index:
@@ -408,7 +447,7 @@ class CasBasedDirectory(Directory):
                          else:
                              return f
                  else:
 -                    print("  resolving {}: nonexistent!".format(c))
 +                    print("  resolving {}: Broken symlink".format(c))
                      return None
          # Shouldn't get here.
@@ -426,17 +465,21 @@ class CasBasedDirectory(Directory):
              return True
          if (isinstance(existing_entry,
                         (remote_execution_pb2.FileNode, remote_execution_pb2.SymlinkNode))):
 +            self.delete_entry(name)
 +            print("Processing overwrite of file/symlink {}: Added to overwritten list and deleted".format(name))
              fileListResult.overwritten.append(relative_pathname)
              return True
          elif isinstance(existing_entry, remote_execution_pb2.DirectoryNode):
              # If 'name' maps to a DirectoryNode, then there must be an entry in index
              # pointing to another Directory.
              if self.index[name].buildstream_object.is_empty():
 +                print("Processing overwrite of directory: Removing original")
                  self.delete_entry(name)
                  fileListResult.overwritten.append(relative_pathname)
                  return True
              else:
                  # We can't overwrite a non-empty directory, so we just ignore it.
 +                print("Processing overwrite of non-empty directory: Ignoring overwrite")
                  fileListResult.ignored.append(relative_pathname)
                  return False
          assert False, ("Entry '{}' is not a recognised file/link/directory and not None; it is {}"
@@ -447,7 +490,13 @@ class CasBasedDirectory(Directory):
          """ _import_directory_recursively and _import_files_from_directory will be called alternately
          as a directory tree is descended. """
          if directory_name in self.index:
 -            subdir = self._resolve_symlink_or_directory(directory_name)
 +            if self._is_followable(directory_name):
 +                subdir = self._resolve_symlink_or_directory(directory_name)
 +            else:
 +                print("Overwriting unfollowable thing {}".format(directory_name))
 +                self.delete_entry(directory_name)
 +                subdir = self._add_directory(directory_name)
 +                # TODO: Add this to the list of overwritten things.
          else:
              subdir = self._add_directory(directory_name)
          new_path_prefix = os.path.join(path_prefix, directory_name)
@@ -459,7 +508,10 @@ class CasBasedDirectory(Directory):
      def _import_files_from_directory(self, source_directory, files, path_prefix=""):
          """ Imports files from a traditional directory """
          result = FileListResult()
 -        for entry in sorted(files):
 +        for entry in files:
 +            print("Importing {} from file system".format(entry))
 +            print("...Order of elements was {}".format(", ".join(self.index.keys())))
++
              split_path = entry.split(os.path.sep)
              # The actual file on the FS we're importing
              import_file = os.path.join(source_directory, entry)
@@ -484,6 +536,8 @@ class CasBasedDirectory(Directory):
                  if self._check_replacement(entry, path_prefix, result):
                      self._add_new_file(source_directory, entry)
                      result.files_written.append(relative_pathname)
 +            print("...Order of elements is now {}".format(", ".join(self.index.keys())))
++
          return result
@@ -540,6 +594,17 @@ class CasBasedDirectory(Directory):
          x = self._resolve_symlink(symlink_node)
          return isinstance(x, CasBasedDirectory)
 +    def _verify_unique(self):
 +        # Verifies that there are no duplicate names in this directory or subdirectories.
 +        names = []
 +        for entrylist in [self.pb2_directory.files, self.pb2_directory.directories, self.pb2_directory.symlinks]:
 +            for e in entrylist:
 +                if e.name in names:
 +                    raise VirtualDirectoryError("Duplicate entry for name {} found".format(e.name))
 +                names.append(e.name)
 +        for d in self.pb2_directory.directories:
 +            self.index[d.name].buildstream_object._verify_unique()
++
      def _partial_import_cas_into_cas(self, source_directory, files, path_prefix="", file_list_required=True):
          """ Import only the files and symlinks listed in 'files' from source_directory to this one.
          Args:
@@ -548,11 +613,11 @@ class CasBasedDirectory(Directory):
             path_prefix (str): Prefix used to add entries to the file list result.
             file_list_required: Whether to update the file list while processing.
          """
 -        print("Beginning partial import of {} into {}".format(source_directory, self))
 +        print("Beginning partial import of {} into {}. Files are: >{}<".format(source_directory, self, ", ".join(files)))
          result = FileListResult()
          processed_directories = set()
          for f in files:
 -            if f == ".": continue
 +            #if f == ".": continue
              fullname = os.path.join(path_prefix, f)
              components = f.split(os.path.sep)
              if len(components)>1:
@@ -562,6 +627,12 @@ class CasBasedDirectory(Directory):
                  if dirname not in processed_directories:
                      # Now strip off the first directory name and import files recursively.
                      subcomponents = CasBasedDirectory.files_in_subdir(files, dirname)
 +                    # We will fail at this point if there is a file or symlink to file called 'dirname'.
 +                    if dirname in self.index:
 +                        x = self._resolve(dirname)
 +                        if isinstance(x, remote_execution_pb2.FileNode):
 +                            self.delete_entry(dirname)
 +                            result.overwritten.append(f)
                      self.create_directory(dirname)
                      print("Creating destination in {}: {}".format(self, dirname))
                      dest_subdir = self._resolve_symlink_or_directory(dirname)
@@ -576,17 +647,24 @@ class CasBasedDirectory(Directory):
                  self.create_directory(f)
              else:
                  # We're importing a file or symlink - replace anything with the same name.
 -                self._check_replacement(f, path_prefix, result)
 -                item = source_directory.index[f].pb_object
 -                if isinstance(item, remote_execution_pb2.FileNode):
 -                    filenode = self.pb2_directory.files.add(digest=item.digest, name=f,
 -                                                            is_executable=item.is_executable)
 -                    self.index[f] = IndexEntry(filenode, modified=(fullname in result.overwritten))
 -                else:
 -                    assert(isinstance(item, remote_execution_pb2.SymlinkNode))
 -                    symlinknode = self.pb2_directory.symlinks.add(name=f, target=item.target)
 -                    # A symlink node has no digest.
 -                    self.index[f] = IndexEntry(symlinknode, modified=(fullname in result.overwritten))
 +                print("Import of file/symlink {} into this directory. Removing anything existing...".format(f))
 +                print("   ... ordering of nodes in this dir was: {}".format(self.index.keys()))
 +                print("   ... symlinks were {}".format([x.name for x in self.pb2_directory.symlinks]))
 +                importable = self._check_replacement(f, path_prefix, result)
 +                if importable:
 +                    print("   ... after replacement of '{}', symlinks are now {}".format(f, [x.name for x in self.pb2_directory.symlinks]))
 +                    item = source_directory.index[f].pb_object
 +                    if isinstance(item, remote_execution_pb2.FileNode):
 +                        print("   ... importing file")
 +                        filenode = self.pb2_directory.files.add(digest=item.digest, name=f,
 +                                                                is_executable=item.is_executable)
 +                        self.index[f] = IndexEntry(filenode, modified=(fullname in result.overwritten))
 +                    else:
 +                        print("   ... importing symlink")
 +                        assert(isinstance(item, remote_execution_pb2.SymlinkNode))
 +                        self._add_new_link_direct(name=f, target=item.target)
 +                        print("   ... symlinks are now {}".format([x.name for x in self.pb2_directory.symlinks]))
 +                    print("   ... ordering of nodes in this dir is now: {}".format(self.index.keys()))
          return result
      def transfer_node_contents(destination, source):
@@ -632,11 +710,69 @@ class CasBasedDirectory(Directory):
          """
          if files is None:
              #return self._full_import_cas_into_cas(source_directory, can_hardlink=True)
 -            files = source_directory.list_relative_paths()
 +            files = list(source_directory.list_relative_paths())
              print("Extracted all files from source directory '{}': {}".format(source_directory, files))
 -        return self._partial_import_cas_into_cas(source_directory, files)
+-
 +        return self._partial_import_cas_into_cas(source_directory, list(files))
++
 +    def _describe(self, thing):
 +        # Describes protocol buffer objects
 +        if isinstance(thing, remote_execution_pb2.DirectoryNode):
 +            return "directory called {}".format(thing.name)
 +        elif isinstance(thing, remote_execution_pb2.SymlinkNode):
 +            return "symlink called {} pointing to {}".format(thing.name, thing.target)
 +        elif isinstance(thing, remote_execution_pb2.FileNode):
 +            return "file called {}".format(thing.name)
 +        else:
 +            return "strange thing"
++
++
 +    def showdiff(self, other):
 +        print("Diffing {} and {}:".format(self, other))
++
 +        def compare_list(l1, l2):
 +            item2 = None
 +            index = 0
 +            print("Comparing lists: {} vs {}".format([d.name for d in l1], [d.name for d in l2]))
 +            for item1 in l1:
 +                if index>=len(l2):
 +                    print("l2 is short: no item to correspond to '{}' in l1.".format(item1.name))
 +                    return False
 +                item2 = l2[index]
 +                if item1.name != item2.name:
 +                    print("Items do not match: {}, a {} in l1, vs {}, a {} in l2".format(item1.name, self._describe(item1), item2.name, self._describe(item2)))
 +                    return False
 +                index += 1
 +            if index != len(l2):
 +                print("l2 is long: Has extra items {}".format(l2[index:]))
 +                return False
 +            return True
 +        def compare_pb2_directories(d1, d2):
 +            result = (compare_list(d1.directories, d2.directories)
 +                    and compare_list(d1.symlinks, d2.symlinks)
 +                    and compare_list(d1.files, d2.files))
 +            return result
++
 +        if not compare_pb2_directories(self.pb2_directory, other.pb2_directory):
 +            return False
++
 +        for d in self.pb2_directory.directories:
 +            self.index[d.name].buildstream_object.showdiff(other.index[d.name].buildstream_object)
 +        print("No differences found in {}".format(self))
++
 +    def show_files_recursive(self):
 +        elems = []
 +        for (k,v) in self.index.items():
 +            if type(v.pb_object) == remote_execution_pb2.DirectoryNode:
 +                elems.append("{}=[{}]".format(k, v.buildstream_object.show_files_recursive()))
 +            elif type(v.pb_object) == remote_execution_pb2.SymlinkNode:
 +                elems.append("{}(s)".format(k))
 +            elif type(v.pb_object) == remote_execution_pb2.FileNode:
 +                elems.append("{}(f)".format(k))
 +            else:
 +                elems.append("{}(?)".format(k))
 +        return " ".join(elems)
++
      def import_files(self, external_pathspec, *, files=None,
                       report_written=True, update_utimes=False,
                       can_link=False):
@@ -659,12 +795,30 @@ class CasBasedDirectory(Directory):
          can_link (bool): Ignored, since hard links do not have any meaning within CAS.
          """
 +        print("Directory before import: {}".format(self.show_files_recursive()))
++
 +        # Sync self
 +        self._recalculate_recursing_down()
 +        if self.parent:
 +            self.parent._recalculate_recursing_up(self)
++
 +        # Duplicate the current directory
++
++
 +        print("Original CAS before CAS-based import: {}".format(self.show_files_recursive()))
 +        print("Original CAS hash: {}".format(self.ref.hash))
          duplicate_cas = None
 +        self._verify_unique()
          if isinstance(external_pathspec, CasBasedDirectory):
 +            duplicate_cas = CasBasedDirectory(self.context, ref=copy.copy(self.ref))
 +            duplicate_cas._verify_unique()
 +            print("-"*80 + "Performing direct CAS-to-CAS import")
 +            print("Duplicated CAS before file-based import: {}".format(duplicate_cas.show_files_recursive()))
 +            print("Duplicate CAS hash: {}".format(duplicate_cas.ref.hash))
              result = self._import_cas_into_cas(external_pathspec, files=files)
+-
 -            # Duplicate the current directory and do an import that way.
 -            duplicate_cas = CasBasedDirectory(self.context, ref=self.ref)
 +            self._verify_unique()
 +            print("Result of cas-to-cas import: {}".format(self.show_files_recursive()))
 +            print("-"*80 + "Performing round-trip import via file system")
              with tempfile.TemporaryDirectory(prefix="roundtrip") as tmpdir:
                  external_pathspec.export_files(tmpdir)
                  if files is None:
@@ -672,8 +826,12 @@ class CasBasedDirectory(Directory):
                  duplicate_cas._import_files_from_directory(tmpdir, files=files)
                  duplicate_cas._recalculate_recursing_down()
                  if duplicate_cas.parent:
 -                    duplicate_cas.parent._recalculate_recursing_up(self)
 +                    duplicate_cas.parent._recalculate_recursing_up(duplicate_cas)
 +                print("Result of direct import: {}".format(duplicate_cas.show_files_recursive()))
++
++
          else:
 +            print("-"*80 + "Performing initial import")
              if isinstance(external_pathspec, FileBasedDirectory):
                  source_directory = external_pathspec.get_underlying_directory()
              else:
@@ -697,6 +855,7 @@ class CasBasedDirectory(Directory):
              self.parent._recalculate_recursing_up(self)
          if duplicate_cas:
              if duplicate_cas.ref.hash != self.ref.hash:
 +                self.showdiff(duplicate_cas)
                  raise VirtualDirectoryError("Mismatch between file-imported result {} and cas-to-cas imported result {}.".format(duplicate_cas.ref.hash,self.ref.hash))
          return result
@@ -757,6 +916,7 @@ class CasBasedDirectory(Directory):
          for entry in self.pb2_directory.symlinks:
              src_name = os.path.join(to_directory, entry.name)
              target_name = entry.target
 +            print("Exporting symlink named {}".format(src_name))
              try:
                  os.symlink(target_name, src_name)
              except FileExistsError as e:
@@ -857,6 +1017,7 @@ class CasBasedDirectory(Directory):
          for (k, v) in sorted(directory_list):
              print("Yielding from subdirectory name {}".format(k))
              yield from v.buildstream_object.list_relative_paths(relpath=os.path.join(relpath, k))
 +        print("List_relative_paths on {} complete".format(relpath))
      def recalculate_hash(self):
          """ Recalcuates the hash for this directory and store the results in

tests/storage/virtual_directory_import.py

@@ -50,31 +50,33 @@ def generate_import_roots(directory):
                  os.symlink(content, os.path.join(rootdir, path))
 -def generate_random_root(directory):
 +def generate_random_roots(directory):
      random.seed(RANDOM_SEED)
 -    rootname = "root6"
 -    rootdir = os.path.join(directory, "content", rootname)
 -    things = []
 -    locations = ['.']
 -    for i in range(0, 100):
 -        location = random.choice(locations)
 -        thingname = "node{}".format(i)
 -        thing = random.choice(['dir', 'link', 'file'])
 -        target = os.path.join(rootdir, location, thingname)
 -        if thing == 'dir':
 -            os.makedirs(target)
 -            locations.append(os.path.join(location, thingname))
 -        elif thing == 'file':
 -            with open(target, "wt") as f:
 -                f.write("This is node {}\n".format(i))
 -        elif thing == 'link':
 -            # TODO: Make some relative symlinks
 -            if random.randint(1, 3) == 1 or len(things) == 0:
 -                os.symlink("/broken", target)
 -            else:
 -                os.symlink(random.choice(things), target)
 -        things.append(os.path.join(location, thingname))
 -        print("Generated {}/{} ".format(rootdir, things[-1]))
 +    for rootno in range(6,13):
 +        rootname = "root{}".format(rootno)
 +        rootdir = os.path.join(directory, "content", rootname)
 +        things = []
 +        locations = ['.']
 +        os.makedirs(rootdir)
 +        for i in range(0, 100):
 +            location = random.choice(locations)
 +            thingname = "node{}".format(i)
 +            thing = random.choice(['dir', 'link', 'file'])
 +            target = os.path.join(rootdir, location, thingname)
 +            if thing == 'dir':
 +                os.makedirs(target)
 +                locations.append(os.path.join(location, thingname))
 +            elif thing == 'file':
 +                with open(target, "wt") as f:
 +                    f.write("This is node {}\n".format(i))
 +            elif thing == 'link':
 +                # TODO: Make some relative symlinks
 +                if random.randint(1, 3) == 1 or len(things) == 0:
 +                    os.symlink("/broken", target)
 +                else:
 +                    os.symlink(random.choice(things), target)
 +            things.append(os.path.join(location, thingname))
 +            print("Generated {}/{} ".format(rootdir, things[-1]))
  def file_contents(path):
@@ -141,39 +143,40 @@ def directory_not_empty(path):
      return os.listdir(path)
 -@pytest.mark.parametrize("original,overlay", combinations([1, 2, 3, 4, 5]))
 +@pytest.mark.parametrize("original,overlay", combinations([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]))
  def test_cas_import(cli, tmpdir, original, overlay):
      fake_context = FakeContext()
      fake_context.artifactdir = tmpdir
      # Create some fake content
      generate_import_roots(tmpdir)
 -    generate_random_root(tmpdir)
 +    generate_random_roots(tmpdir)
      d = create_new_casdir(original, fake_context, tmpdir)
      d2 = create_new_casdir(overlay, fake_context, tmpdir)
      print("Importing dir {} into {}".format(overlay, original))
      d.import_files(d2)
      d.export_files(os.path.join(tmpdir, "output"))
 -    for item in root_filesets[overlay - 1]:
 -        (path, typename, content) = item
 -        realpath = resolve_symlinks(path, os.path.join(tmpdir, "output"))
 -        if typename == 'F':
 -            if os.path.isdir(realpath) and directory_not_empty(realpath):
 -                # The file should not have overwritten the directory in this case.
 -                pass
 -            else:
 -                assert os.path.isfile(realpath), "{} did not exist in the combined virtual directory".format(path)
 -                assert file_contents_are(realpath, content)
 -        elif typename == 'S':
 -            if os.path.isdir(realpath) and directory_not_empty(realpath):
 -                # The symlink should not have overwritten the directory in this case.
 -                pass
 -            else:
 -                assert os.path.islink(realpath)
 -                assert os.readlink(realpath) == content
 -        elif typename == 'D':
 -            # Note that isdir accepts symlinks to dirs, so a symlink to a dir is acceptable.
 -            assert os.path.isdir(realpath)
 +    if overlay < 6:
 +        for item in root_filesets[overlay - 1]:
 +            (path, typename, content) = item
 +            realpath = resolve_symlinks(path, os.path.join(tmpdir, "output"))
 +            if typename == 'F':
 +                if os.path.isdir(realpath) and directory_not_empty(realpath):
 +                    # The file should not have overwritten the directory in this case.
 +                    pass
 +                else:
 +                    assert os.path.isfile(realpath), "{} did not exist in the combined virtual directory".format(path)
 +                    assert file_contents_are(realpath, content)
 +            elif typename == 'S':
 +                if os.path.isdir(realpath) and directory_not_empty(realpath):
 +                    # The symlink should not have overwritten the directory in this case.
 +                    pass
 +                else:
 +                    assert os.path.islink(realpath)
 +                    assert os.readlink(realpath) == content
 +            elif typename == 'D':
 +                # Note that isdir accepts symlinks to dirs, so a symlink to a dir is acceptable.
 +                assert os.path.isdir(realpath)
      # Now do the same thing with filebaseddirectories and check the contents match
      d3 = create_new_casdir(original, fake_context, tmpdir)
@@ -188,7 +191,7 @@ def test_directory_listing(cli, tmpdir, root):
      fake_context.artifactdir = tmpdir
      # Create some fake content
      generate_import_roots(tmpdir)
 -    generate_random_root(tmpdir)
 +    generate_random_roots(tmpdir)
      d = create_new_filedir(root, tmpdir)
      filelist = list(d.list_relative_paths())

[Notes] [Git][BuildStream/buildstream][jmac/cas_to_cas_oct] 7 commits: Add a tool to show differences in two CAS directories

Jim MacArthur pushed to branch jmac/cas_to_cas_oct at BuildStream / buildstream

Commits:

2 changed files:

Changes: