Jim MacArthur pushed to branch jmac/cas_to_cas_oct at BuildStream / buildstream
Commits:
-
b3ebec7c
by Jim MacArthur at 2018-10-25T14:12:05Z
-
459f809b
by Jim MacArthur at 2018-10-25T15:48:07Z
-
b28b6c9e
by Jim MacArthur at 2018-10-25T15:48:23Z
-
052da4e9
by Jim MacArthur at 2018-10-25T15:48:50Z
2 changed files:
Changes:
| ... | ... | @@ -387,7 +387,7 @@ class CasBasedDirectory(Directory): |
| 387 | 387 |
return directory
|
| 388 | 388 |
|
| 389 | 389 |
|
| 390 |
- def _resolve(self, name, absolute_symlinks_resolve=True, force_create=False):
|
|
| 390 |
+ def _resolve(self, name, absolute_symlinks_resolve=True, force_create=False, first_seen_object = None):
|
|
| 391 | 391 |
""" Resolves any name to an object. If the name points to a symlink in
|
| 392 | 392 |
this directory, it returns the thing it points to,
|
| 393 | 393 |
recursively. Returns a CasBasedDirectory, FileNode or
|
| ... | ... | @@ -406,6 +406,14 @@ class CasBasedDirectory(Directory): |
| 406 | 406 |
return index_entry.pb_object
|
| 407 | 407 |
|
| 408 | 408 |
assert isinstance(index_entry.pb_object, remote_execution_pb2.SymlinkNode)
|
| 409 |
+ |
|
| 410 |
+ if first_seen_object is None:
|
|
| 411 |
+ first_seen_object = index_entry.pb_object
|
|
| 412 |
+ else:
|
|
| 413 |
+ if index_entry.pb_object == first_seen_object:
|
|
| 414 |
+ ### Infinite symlink loop detected ###
|
|
| 415 |
+ return None
|
|
| 416 |
+
|
|
| 409 | 417 |
print("Resolving '{}': This is a symlink node in the current directory.".format(name))
|
| 410 | 418 |
symlink = index_entry.pb_object
|
| 411 | 419 |
components = symlink.target.split(CasBasedDirectory._pb2_path_sep)
|
| ... | ... | @@ -439,7 +447,7 @@ class CasBasedDirectory(Directory): |
| 439 | 447 |
directory = directory.parent
|
| 440 | 448 |
else:
|
| 441 | 449 |
if c in directory.index:
|
| 442 |
- f = directory._resolve(c, absolute_symlinks_resolve)
|
|
| 450 |
+ f = directory._resolve(c, absolute_symlinks_resolve, first_seen_object=first_seen_object)
|
|
| 443 | 451 |
# Ultimately f must now be a file or directory
|
| 444 | 452 |
if isinstance(f, CasBasedDirectory):
|
| 445 | 453 |
directory = f
|
| ... | ... | @@ -453,7 +461,10 @@ class CasBasedDirectory(Directory): |
| 453 | 461 |
directory = directory.descend(c, create=True)
|
| 454 | 462 |
elif components:
|
| 455 | 463 |
# Oh dear. We have components left to resolve, but the one we're trying to resolve points to a file.
|
| 456 |
- raise VirtualDirectoryError("Reached a file called {} while trying to resolve a symlink; cannot proceed".format(c))
|
|
| 464 |
+ print("Trying to resolve {}, but found {} was a file.".format(symlink.target, c))
|
|
| 465 |
+ self.delete_entry(c)
|
|
| 466 |
+ directory = directory.descend(c, create=True)
|
|
| 467 |
+ #raise VirtualDirectoryError("Reached a file called {} while trying to resolve a symlink; cannot proceed".format(c))
|
|
| 457 | 468 |
else:
|
| 458 | 469 |
return f
|
| 459 | 470 |
else:
|
| ... | ... | @@ -842,35 +853,34 @@ class CasBasedDirectory(Directory): |
| 842 | 853 |
if self.parent:
|
| 843 | 854 |
self.parent._recalculate_recursing_up(self)
|
| 844 | 855 |
|
| 845 |
- # Duplicate the current directory
|
|
| 846 |
- |
|
| 856 |
+ duplicate_test = False
|
|
| 847 | 857 |
|
| 848 | 858 |
print("Original CAS before CAS-based import: {}".format(self.show_files_recursive()))
|
| 849 | 859 |
print("Original CAS hash: {}".format(self.ref.hash))
|
| 850 | 860 |
duplicate_cas = None
|
| 851 | 861 |
self._verify_unique()
|
| 852 | 862 |
if isinstance(external_pathspec, CasBasedDirectory):
|
| 853 |
- duplicate_cas = CasBasedDirectory(self.context, ref=copy.copy(self.ref))
|
|
| 854 |
- duplicate_cas._verify_unique()
|
|
| 863 |
+ if duplicate_test:
|
|
| 864 |
+ duplicate_cas = CasBasedDirectory(self.context, ref=copy.copy(self.ref))
|
|
| 865 |
+ duplicate_cas._verify_unique()
|
|
| 866 |
+ print("Duplicated CAS before file-based import: {}".format(duplicate_cas.show_files_recursive()))
|
|
| 867 |
+ print("Duplicate CAS hash: {}".format(duplicate_cas.ref.hash))
|
|
| 855 | 868 |
print("-"*80 + "Performing direct CAS-to-CAS import")
|
| 856 |
- print("Duplicated CAS before file-based import: {}".format(duplicate_cas.show_files_recursive()))
|
|
| 857 |
- print("Duplicate CAS hash: {}".format(duplicate_cas.ref.hash))
|
|
| 858 | 869 |
result = self._import_cas_into_cas(external_pathspec, files=files)
|
| 859 | 870 |
self._verify_unique()
|
| 860 | 871 |
print("Result of cas-to-cas import: {}".format(self.show_files_recursive()))
|
| 861 | 872 |
print("-"*80 + "Performing round-trip import via file system")
|
| 862 |
- with tempfile.TemporaryDirectory(prefix="roundtrip") as tmpdir:
|
|
| 863 |
- external_pathspec.export_files(tmpdir)
|
|
| 864 |
- if files is None:
|
|
| 865 |
- files = list(list_relative_paths(tmpdir))
|
|
| 866 |
- print("Importing from filesystem: filelist is: {}".format(files))
|
|
| 867 |
- duplicate_cas._import_files_from_directory(tmpdir, files=files)
|
|
| 868 |
- duplicate_cas._recalculate_recursing_down()
|
|
| 869 |
- if duplicate_cas.parent:
|
|
| 870 |
- duplicate_cas.parent._recalculate_recursing_up(duplicate_cas)
|
|
| 871 |
- print("Result of direct import: {}".format(duplicate_cas.show_files_recursive()))
|
|
| 872 |
-
|
|
| 873 |
- |
|
| 873 |
+ if duplicate_test:
|
|
| 874 |
+ with tempfile.TemporaryDirectory(prefix="roundtrip") as tmpdir:
|
|
| 875 |
+ external_pathspec.export_files(tmpdir)
|
|
| 876 |
+ if files is None:
|
|
| 877 |
+ files = list(list_relative_paths(tmpdir))
|
|
| 878 |
+ print("Importing from filesystem: filelist is: {}".format(files))
|
|
| 879 |
+ duplicate_cas._import_files_from_directory(tmpdir, files=files)
|
|
| 880 |
+ duplicate_cas._recalculate_recursing_down()
|
|
| 881 |
+ if duplicate_cas.parent:
|
|
| 882 |
+ duplicate_cas.parent._recalculate_recursing_up(duplicate_cas)
|
|
| 883 |
+ print("Result of direct import: {}".format(duplicate_cas.show_files_recursive()))
|
|
| 874 | 884 |
else:
|
| 875 | 885 |
print("-"*80 + "Performing initial import")
|
| 876 | 886 |
if isinstance(external_pathspec, FileBasedDirectory):
|
| ... | ... | @@ -20,11 +20,17 @@ class FakeContext(): |
| 20 | 20 |
# 'F' (file), 'S' (symlink) or 'D' (directory) with content being the contents
|
| 21 | 21 |
# for a file or the destination for a symlink.
|
| 22 | 22 |
root_filesets = [
|
| 23 |
+ # Arbitrary test sets
|
|
| 23 | 24 |
[('a/b/c/textfile1', 'F', 'This is textfile 1\n')],
|
| 24 | 25 |
[('a/b/c/textfile1', 'F', 'This is the replacement textfile 1\n')],
|
| 25 | 26 |
[('a/b/d', 'D', '')],
|
| 26 | 27 |
[('a/b/c', 'S', '/a/b/d')],
|
| 27 |
- [('a/b/d', 'D', ''), ('a/b/c', 'S', '/a/b/d')]
|
|
| 28 |
+ [('a/b/d', 'S', '/a/b/c')],
|
|
| 29 |
+ [('a/b/d', 'D', ''), ('a/b/c', 'S', '/a/b/d')],
|
|
| 30 |
+ [('a/b/c', 'D', ''), ('a/b/d', 'S', '/a/b/c')],
|
|
| 31 |
+ [('a/b', 'F', 'This is textfile 1\n')],
|
|
| 32 |
+ [('a/b/c', 'F', 'This is textfile 1\n')],
|
|
| 33 |
+ [('a/b/c', 'D', '')]
|
|
| 28 | 34 |
]
|
| 29 | 35 |
|
| 30 | 36 |
empty_hash_ref = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
| ... | ... | @@ -178,8 +184,9 @@ def _import_test(tmpdir, original, overlay, generator_function, verify_contents= |
| 178 | 184 |
assert os.path.islink(realpath)
|
| 179 | 185 |
assert os.readlink(realpath) == content
|
| 180 | 186 |
elif typename == 'D':
|
| 181 |
- # Note that isdir accepts symlinks to dirs, so a symlink to a dir is acceptable.
|
|
| 182 |
- assert os.path.isdir(realpath)
|
|
| 187 |
+ # We can't do any more tests than this because it depends on things present in the original. Blank directories
|
|
| 188 |
+ # here will be ignored and the original left in place.
|
|
| 189 |
+ assert os.path.lexists(realpath)
|
|
| 183 | 190 |
|
| 184 | 191 |
# Now do the same thing with filebaseddirectories and check the contents match
|
| 185 | 192 |
d3 = create_new_casdir(original, fake_context, tmpdir)
|
| ... | ... | @@ -187,14 +194,15 @@ def _import_test(tmpdir, original, overlay, generator_function, verify_contents= |
| 187 | 194 |
d3.import_files(d2)
|
| 188 | 195 |
assert d.ref.hash == d3.ref.hash
|
| 189 | 196 |
|
| 190 |
-@pytest.mark.parametrize("original,overlay", combinations(range(1,6)))
|
|
| 197 |
+@pytest.mark.parametrize("original,overlay", combinations(range(1,len(root_filesets)+1)))
|
|
| 191 | 198 |
def test_fixed_cas_import(cli, tmpdir, original, overlay):
|
| 192 | 199 |
_import_test(tmpdir, original, overlay, generate_import_roots, verify_contents=True)
|
| 193 | 200 |
|
| 194 | 201 |
@pytest.mark.parametrize("original,overlay", combinations(range(1,11)))
|
| 195 |
-def test_random_cas_import(cli, tmpdir, original, overlay):
|
|
| 202 |
+def test_random_cas_import_fast(cli, tmpdir, original, overlay):
|
|
| 196 | 203 |
_import_test(tmpdir, original, overlay, generate_random_root, verify_contents=False)
|
| 197 | 204 |
|
| 205 |
+
|
|
| 198 | 206 |
def _listing_test(tmpdir, root, generator_function):
|
| 199 | 207 |
fake_context = FakeContext()
|
| 200 | 208 |
fake_context.artifactdir = tmpdir
|