[Notes] [Git][BuildStream/buildstream][jmac/cas_to_cas_oct] 2 commits: CAS-to-CAS: Now passing all 20x20 tests

[Jim MacArthur <gitlab mg gitlab com>]

Title: GitLab

Jim MacArthur pushed to branch jmac/cas_to_cas_oct at BuildStream / buildstream

Commits:

• 68393482
  by Jim MacArthur at 2018-10-24T17:28:39Z

  CAS-to-CAS: Now passing all 20x20 tests
  

• b320812d
  by Jim MacArthur at 2018-10-24T18:01:36Z

  Separation of fixed/random tests in virtual_directory_import
  

2 changed files:

• buildstream/storage/_casbaseddirectory.py
• tests/storage/virtual_directory_import.py

Changes:

buildstream/storage/_casbaseddirectory.py

---

...	...	@@ -298,6 +298,9 @@ class CasBasedDirectory(Directory):
298	298	else:
299	299	if create:
300	300	newdir = self._add_directory(subdirectory_spec[0])
	301	+ print("Created new directory called {} and descending into it".format(subdirectory_spec[0]))
	302	+ #if subdirectory_spec[0] == "broken":
	303	+ # assert False
301	304	return newdir.descend(subdirectory_spec[1:], create)
302	305	else:
303	306	error = "No entry called '{}' found in {}. There are directories called {}."
...	...	@@ -358,7 +361,7 @@ class CasBasedDirectory(Directory):
358	361	print("Is {} followable? Resolved to {}".format(name, target))
359	362	return isinstance(target, CasBasedDirectory) or target is None
360	363
361		- def _resolve_symlink(self, node):
	364	+ def _resolve_symlink(self, node, force_create=True):
362	365	"""Same as _resolve_symlink_or_directory but takes a SymlinkNode.
363	366	"""
364	367
...	...	@@ -377,7 +380,10 @@ class CasBasedDirectory(Directory):
377	380	elif c == "..":
378	381	directory = directory.parent
379	382	else:
380		- directory = directory.descend(c, create=True)
	383	+ if c in directory.index or force_create:
	384	+ directory = directory.descend(c, create=True)
	385	+ else:
	386	+ return None
381	387	return directory
382	388
383	389
...	...	@@ -400,6 +406,7 @@ class CasBasedDirectory(Directory):
400	406	return index_entry.pb_object
401	407
402	408	assert isinstance(index_entry.pb_object, remote_execution_pb2.SymlinkNode)
	409	+ print("Resolving '{}': This is a symlink node in the current directory.".format(name))
403	410	symlink = index_entry.pb_object
404	411	components = symlink.target.split(CasBasedDirectory._pb2_path_sep)
405	412
...	...	@@ -443,7 +450,7 @@ class CasBasedDirectory(Directory):
443	450	print(" resolving {}: file/broken link".format(c))
444	451	if f is None and force_create:
445	452	print("Creating target of broken link {}".format(c))
446		- return directory.descend(c, create=True)
	453	+ directory = directory.descend(c, create=True)
447	454	elif components:
448	455	# Oh dear. We have components left to resolve, but the one we're trying to resolve points to a file.
449	456	raise VirtualDirectoryError("Reached a file called {} while trying to resolve a symlink; cannot proceed".format(c))
...	...	@@ -453,7 +460,7 @@ class CasBasedDirectory(Directory):
453	460	print(" resolving {}: Non-existent file; must be from a broken symlink.".format(c))
454	461	if force_create:
455	462	print("Creating target of broken link {} (2)".format(c))
456		- return directory.descend(c, create=True)
	463	+ directory = directory.descend(c, create=True)
457	464	else:
458	465	return None
459	466
...	...	@@ -528,6 +535,8 @@ class CasBasedDirectory(Directory):
528	535	directory_name = split_path[0]
529	536	# Hand this off to the importer for that subdir. This will only do one file -
530	537	# a better way would be to hand off all the files in this subdir at once.
	538	+ # failed here because directory_name didn't point to a directory...
	539	+ print("Attempting to import into {} from {}".format(directory_name, source_directory))
531	540	subdir_result = self._import_directory_recursively(directory_name, source_directory,
532	541	split_path[1:], path_prefix)
533	542	result.combine(subdir_result)
...	...	@@ -598,7 +607,7 @@ class CasBasedDirectory(Directory):
598	607	return [f[len(dirname):] for f in sorted_files if f.startswith(dirname)]
599	608
600	609	def symlink_target_is_directory(self, symlink_node):
601		- x = self._resolve_symlink(symlink_node)
	610	+ x = self._resolve_symlink(symlink_node, force_create=False)
602	611	return isinstance(x, CasBasedDirectory)
603	612
604	613	def _verify_unique(self):
...	...	@@ -636,14 +645,20 @@ class CasBasedDirectory(Directory):
636	645	subcomponents = CasBasedDirectory.files_in_subdir(files, dirname)
637	646	# We will fail at this point if there is a file or symlink to file called 'dirname'.
638	647	if dirname in self.index:
639		- x = self._resolve(dirname)
	648	+ x = self._resolve(dirname, force_create=True)
640	649	if isinstance(x, remote_execution_pb2.FileNode):
641	650	self.delete_entry(dirname)
642	651	result.overwritten.append(f)
643		- self.create_directory(dirname)
644		- print("Creating destination in {}: {}".format(self, dirname))
645		- dest_subdir = self._resolve_symlink_or_directory(dirname)
	652	+ dest_subdir = self.descend(dirname, create=True)
	653	+ else:
	654	+ dest_subdir = x
	655	+ else:
	656	+ print("Importing {}: {} does not exist in {}, so it is created as a directory".format(f, dirname, self))
	657	+
	658	+ self.create_directory(dirname)
	659	+ dest_subdir = self._resolve_symlink_or_directory(dirname)
646	660	src_subdir = source_directory.descend(dirname)
	661	+ print("Now recursing into {} to continue adding {}".format(src_subdir, f))
647	662	import_result = dest_subdir._partial_import_cas_into_cas(src_subdir, subcomponents,
648	663	path_prefix=fullname, file_list_required=file_list_required)
649	664	result.combine(import_result)
...	...	@@ -651,7 +666,19 @@ class CasBasedDirectory(Directory):
651	666	elif isinstance(source_directory.index[f].buildstream_object, CasBasedDirectory):
652	667	# The thing in the input file list is a directory on its own. In which case, replace any existing file, or symlink to file
653	668	# with the new, blank directory - if it's neither of those things, or doesn't exist, then just create the dir.
654		- self.create_directory(f)
	669	+ if f in self.index:
	670	+ x = self._resolve(f)
	671	+ if x is None:
	672	+ # If we're importing a blank directory, and the target has a broken symlink, then do nothing.
	673	+ pass
	674	+ elif isinstance(x, remote_execution_pb2.FileNode):
	675	+ # Files with the same name, or symlinks to files, get removed.
	676	+ pass
	677	+ else:
	678	+ # There's either a symlink (valid or not) or existing directory with this name, so do nothing.
	679	+ pass
	680	+ else:
	681	+ self.create_directory(f)
655	682	else:
656	683	# We're importing a file or symlink - replace anything with the same name.
657	684	print("Import of file/symlink {} into this directory. Removing anything existing...".format(f))
...	...	@@ -736,18 +763,22 @@ class CasBasedDirectory(Directory):
736	763	def showdiff(self, other):
737	764	print("Diffing {} and {}:".format(self, other))
738	765
739		- def compare_list(l1, l2):
	766	+ def compare_list(l1, l2, name):
740	767	item2 = None
741	768	index = 0
742		- print("Comparing lists: {} vs {}".format([d.name for d in l1], [d.name for d in l2]))
	769	+ print("Comparing {} lists: {} vs {}".format(name, [d.name for d in l1], [d.name for d in l2]))
743	770	for item1 in l1:
744	771	if index>=len(l2):
745	772	print("l2 is short: no item to correspond to '{}' in l1.".format(item1.name))
746	773	return False
747	774	item2 = l2[index]
748	775	if item1.name != item2.name:
749		- print("Items do not match: {}, a {} in l1, vs {}, a {} in l2".format(item1.name, self._describe(item1), item2.name, self._describe(item2)))
	776	+ print("Items do not match in {} list: {}, a {} in l1, vs {}, a {} in l2".format(name, item1.name, self._describe(item1), item2.name, self._describe(item2)))
750	777	return False
	778	+ if isinstance(item1, remote_execution_pb2.FileNode):
	779	+ if item1.is_executable != item2.is_executable:
	780	+ print("Executable flags do not match on file {}.".format(item1.name))
	781	+ return False
751	782	index += 1
752	783	if index != len(l2):
753	784	print("l2 is long: Has extra items {}".format(l2[index:]))
...	...	@@ -755,17 +786,19 @@ class CasBasedDirectory(Directory):
755	786	return True
756	787
757	788	def compare_pb2_directories(d1, d2):
758		- result = (compare_list(d1.directories, d2.directories)
759		- and compare_list(d1.symlinks, d2.symlinks)
760		- and compare_list(d1.files, d2.files))
	789	+ result = (compare_list(d1.directories, d2.directories, "directory")
	790	+ and compare_list(d1.symlinks, d2.symlinks, "symlink")
	791	+ and compare_list(d1.files, d2.files, "file"))
761	792	return result
762	793
763	794	if not compare_pb2_directories(self.pb2_directory, other.pb2_directory):
764	795	return False
765	796
766	797	for d in self.pb2_directory.directories:
767		- self.index[d.name].buildstream_object.showdiff(other.index[d.name].buildstream_object)
	798	+ if not self.index[d.name].buildstream_object.showdiff(other.index[d.name].buildstream_object):
	799	+ return False
768	800	print("No differences found in {}".format(self))
	801	+ return True
769	802
770	803	def show_files_recursive(self):
771	804	elems = []
...	...	@@ -829,7 +862,8 @@ class CasBasedDirectory(Directory):
829	862	with tempfile.TemporaryDirectory(prefix="roundtrip") as tmpdir:
830	863	external_pathspec.export_files(tmpdir)
831	864	if files is None:
832		- files = list_relative_paths(tmpdir)
	865	+ files = list(list_relative_paths(tmpdir))
	866	+ print("Importing from filesystem: filelist is: {}".format(files))
833	867	duplicate_cas._import_files_from_directory(tmpdir, files=files)
834	868	duplicate_cas._recalculate_recursing_down()
835	869	if duplicate_cas.parent:

tests/storage/virtual_directory_import.py

---

...	...	@@ -24,59 +24,61 @@ root_filesets = [
24	24	[('a/b/c/textfile1', 'F', 'This is the replacement textfile 1\n')],
25	25	[('a/b/d', 'D', '')],
26	26	[('a/b/c', 'S', '/a/b/d')],
27		- [('a/b/d', 'D', ''), ('a/b/c', 'S', '/a/b/d')],
	27	+ [('a/b/d', 'D', ''), ('a/b/c', 'S', '/a/b/d')]
28	28	]
29	29
30	30	empty_hash_ref = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
31	31	RANDOM_SEED = 69105
32	32
33	33
34		-def generate_import_roots(directory):
35		- for fileset in range(1, len(root_filesets) + 1):
36		- rootname = "root{}".format(fileset)
37		- rootdir = os.path.join(directory, "content", rootname)
38		-
39		- for (path, typesymbol, content) in root_filesets[fileset - 1]:
40		- if typesymbol == 'F':
41		- (dirnames, filename) = os.path.split(path)
42		- os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
43		- with open(os.path.join(rootdir, dirnames, filename), "wt") as f:
44		- f.write(content)
45		- elif typesymbol == 'D':
46		- os.makedirs(os.path.join(rootdir, path), exist_ok=True)
47		- elif typesymbol == 'S':
48		- (dirnames, filename) = os.path.split(path)
49		- os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
50		- os.symlink(content, os.path.join(rootdir, path))
51		-
52		-
53		-def generate_random_roots(directory):
54		- random.seed(RANDOM_SEED)
55		- for rootno in range(6,13):
56		- rootname = "root{}".format(rootno)
57		- rootdir = os.path.join(directory, "content", rootname)
58		- things = []
59		- locations = ['.']
60		- os.makedirs(rootdir)
61		- for i in range(0, 100):
62		- location = random.choice(locations)
63		- thingname = "node{}".format(i)
64		- thing = random.choice(['dir', 'link', 'file'])
65		- target = os.path.join(rootdir, location, thingname)
66		- if thing == 'dir':
67		- os.makedirs(target)
68		- locations.append(os.path.join(location, thingname))
69		- elif thing == 'file':
70		- with open(target, "wt") as f:
71		- f.write("This is node {}\n".format(i))
72		- elif thing == 'link':
73		- # TODO: Make some relative symlinks
74		- if random.randint(1, 3) == 1 or len(things) == 0:
75		- os.symlink("/broken", target)
76		- else:
77		- os.symlink(random.choice(things), target)
78		- things.append(os.path.join(location, thingname))
79		- print("Generated {}/{} ".format(rootdir, things[-1]))
	34	+def generate_import_roots(rootno, directory):
	35	+ rootname = "root{}".format(rootno)
	36	+ rootdir = os.path.join(directory, "content", rootname)
	37	+
	38	+ for (path, typesymbol, content) in root_filesets[rootno - 1]:
	39	+ if typesymbol == 'F':
	40	+ (dirnames, filename) = os.path.split(path)
	41	+ os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
	42	+ with open(os.path.join(rootdir, dirnames, filename), "wt") as f:
	43	+ f.write(content)
	44	+ elif typesymbol == 'D':
	45	+ os.makedirs(os.path.join(rootdir, path), exist_ok=True)
	46	+ elif typesymbol == 'S':
	47	+ (dirnames, filename) = os.path.split(path)
	48	+ os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
	49	+ os.symlink(content, os.path.join(rootdir, path))
	50	+
	51	+
	52	+def generate_random_root(rootno, directory):
	53	+ random.seed(RANDOM_SEED+rootno)
	54	+ rootname = "root{}".format(rootno)
	55	+ rootdir = os.path.join(directory, "content", rootname)
	56	+ things = []
	57	+ locations = ['.']
	58	+ os.makedirs(rootdir)
	59	+ for i in range(0, 100):
	60	+ location = random.choice(locations)
	61	+ thingname = "node{}".format(i)
	62	+ thing = random.choice(['dir', 'link', 'file'])
	63	+ target = os.path.join(rootdir, location, thingname)
	64	+ description = thing
	65	+ if thing == 'dir':
	66	+ os.makedirs(target)
	67	+ locations.append(os.path.join(location, thingname))
	68	+ elif thing == 'file':
	69	+ with open(target, "wt") as f:
	70	+ f.write("This is node {}\n".format(i))
	71	+ elif thing == 'link':
	72	+ # TODO: Make some relative symlinks
	73	+ if random.randint(1, 3) == 1 or len(things) == 0:
	74	+ os.symlink("/broken", target)
	75	+ description = "symlink pointing to /broken"
	76	+ else:
	77	+ symlink_destination = random.choice(things)
	78	+ os.symlink(symlink_destination, target)
	79	+ description = "symlink pointing to {}".format(symlink_destination)
	80	+ things.append(os.path.join(location, thingname))
	81	+ print("Generated {}/{}, a {}".format(rootdir, things[-1], description))
80	82
81	83
82	84	def file_contents(path):
...	...	@@ -143,20 +145,21 @@ def directory_not_empty(path):
143	145	return os.listdir(path)
144	146
145	147
146		-@pytest.mark.parametrize("original,overlay", combinations([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]))
147		-def test_cas_import(cli, tmpdir, original, overlay):
	148	+def _import_test(tmpdir, original, overlay, generator_function, verify_contents=False):
148	149	fake_context = FakeContext()
149	150	fake_context.artifactdir = tmpdir
150	151	# Create some fake content
151		- generate_import_roots(tmpdir)
152		- generate_random_roots(tmpdir)
	152	+ generator_function(original, tmpdir)
	153	+ if original != overlay:
	154	+ generator_function(overlay, tmpdir)
	155	+
153	156	d = create_new_casdir(original, fake_context, tmpdir)
154	157	d2 = create_new_casdir(overlay, fake_context, tmpdir)
155	158	print("Importing dir {} into {}".format(overlay, original))
156	159	d.import_files(d2)
157	160	d.export_files(os.path.join(tmpdir, "output"))
158	161
159		- if overlay < 6:
	162	+ if verify_contents:
160	163	for item in root_filesets[overlay - 1]:
161	164	(path, typename, content) = item
162	165	realpath = resolve_symlinks(path, os.path.join(tmpdir, "output"))
...	...	@@ -184,14 +187,19 @@ def test_cas_import(cli, tmpdir, original, overlay):
184	187	d3.import_files(d2)
185	188	assert d.ref.hash == d3.ref.hash
186	189
	190	+@pytest.mark.parametrize("original,overlay", combinations(range(1,6)))
	191	+def test_fixed_cas_import(cli, tmpdir, original, overlay):
	192	+ _import_test(tmpdir, original, overlay, generate_import_roots, verify_contents=True)
	193	+
	194	+@pytest.mark.parametrize("original,overlay", combinations(range(1,11)))
	195	+def test_random_cas_import(cli, tmpdir, original, overlay):
	196	+ _import_test(tmpdir, original, overlay, generate_random_root, verify_contents=False)
187	197
188		-@pytest.mark.parametrize("root", [1, 2, 3, 4, 5, 6])
189		-def test_directory_listing(cli, tmpdir, root):
	198	+def _listing_test(tmpdir, root, generator_function):
190	199	fake_context = FakeContext()
191	200	fake_context.artifactdir = tmpdir
192	201	# Create some fake content
193		- generate_import_roots(tmpdir)
194		- generate_random_roots(tmpdir)
	202	+ generator_function(root, tmpdir)
195	203
196	204	d = create_new_filedir(root, tmpdir)
197	205	filelist = list(d.list_relative_paths())
...	...	@@ -204,3 +212,12 @@ def test_directory_listing(cli, tmpdir, root):
204	212	print("filelist for root {} via CasBasedDirectory:".format(root))
205	213	print("{}".format(filelist2))
206	214	assert filelist == filelist2
	215	+
	216	+
	217	+@pytest.mark.parametrize("root", range(1,11))
	218	+def test_random_directory_listing(cli, tmpdir, root):
	219	+ _listing_test(tmpdir, root, generate_random_root)
	220	+
	221	+@pytest.mark.parametrize("root", [1, 2, 3, 4, 5])
	222	+def test_fixed_directory_listing(cli, tmpdir, root):
	223	+ _listing_test(tmpdir, root, generate_import_roots)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.