[Notes] [Git][BuildStream/buildstream][jmac/cas_to_cas_v2] 5 commits: plugin.py: Redirect DEBUG from `__del__` to `sys.stderr`

[Jim MacArthur <gitlab mg gitlab com>]

Title: GitLab

Jim MacArthur pushed to branch jmac/cas_to_cas_v2 at BuildStream / buildstream

Commits:

• 7738f6df
  by Daniel Silverstone at 2018-11-09T11:25:44Z

  plugin.py: Redirect DEBUG from `__del__` to `sys.stderr`
  
  Previously, debug messages from `Plugin.__del__()` went to stdout
  which was a problem for the test suite in extremely rare and hard to
  predict circumstances.  This corrects that by sending the message to
  stderr where all the rest of the messages tend to go.
  
  Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>
  

• 62f59eaa
  by Valentin David at 2018-11-09T13:30:06Z

  Merge branch 'danielsilverstone-ct/plugin-destroy-debug-to-stderr' into 'master'
  
  plugin.py: Redirect DEBUG from `__del__` to `sys.stderr`
  
  See merge request BuildStream/buildstream!944

• ddc0c8c2
  by Jim MacArthur at 2018-11-09T14:32:52Z

  _casbaseddirectory.py: Enable direct CAS-to-CAS import.
  
  Fixes issue #574.
  

• cb4695a0
  by Jim MacArthur at 2018-11-09T14:32:52Z

  _casbaseddirectory: Fix order of list_modified_paths
  
  This will now return paths in the same order as list_relative_paths.
  

• 95efb1f3
  by Jim MacArthur at 2018-11-09T14:32:52Z

  virtual_directory_import.py: New test
  

3 changed files:

• buildstream/plugin.py
• buildstream/storage/_casbaseddirectory.py
• + tests/storage/virtual_directory_import.py

Changes:

buildstream/plugin.py

---

...	...	@@ -111,6 +111,7 @@ Class Reference
111	111
112	112	import os
113	113	import subprocess
	114	+import sys
114	115	from contextlib import contextmanager
115	116	from weakref import WeakValueDictionary
116	117
...	...	@@ -190,7 +191,7 @@ class Plugin():
190	191	# Dont send anything through the Message() pipeline at destruction time,
191	192	# any subsequent lookup of plugin by unique id would raise KeyError.
192	193	if self.__context.log_debug:
193		- print("DEBUG: Destroyed: {}".format(self))
	194	+ sys.stderr.write("DEBUG: Destroyed: {}\n".format(self))
194	195
195	196	def __str__(self):
196	197	return "{kind} {typetag} at {provenance}".format(

buildstream/storage/_casbaseddirectory.py

---

...	...	@@ -30,7 +30,6 @@ See also: :ref:`sandboxing`.
30	30	from collections import OrderedDict
31	31
32	32	import os
33		-import tempfile
34	33	import stat
35	34
36	35	from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
...	...	@@ -51,6 +50,184 @@ class IndexEntry():
51	50	self.modified = modified
52	51
53	52
	53	+class ResolutionException(VirtualDirectoryError):
	54	+ """ Superclass of all exceptions that can be raised by
	55	+ CasBasedDirectory._resolve. Should not be used outside this module. """
	56	+ pass
	57	+
	58	+
	59	+class InfiniteSymlinkException(ResolutionException):
	60	+ """ Raised when an infinite symlink loop is found. """
	61	+ pass
	62	+
	63	+
	64	+class AbsoluteSymlinkException(ResolutionException):
	65	+ """Raised if we try to follow an absolute symlink (i.e. one whose
	66	+ target starts with the path separator) and we have disallowed
	67	+ following such symlinks.
	68	+ """
	69	+ pass
	70	+
	71	+
	72	+class UnexpectedFileException(ResolutionException):
	73	+ """Raised if we were found a file where a directory or symlink was
	74	+ expected, for example we try to resolve a symlink pointing to
	75	+ /a/b/c but /a/b is a file.
	76	+ """
	77	+ def __init__(self, message=""):
	78	+ """Allow constructor with no arguments, since this can be raised in
	79	+ places where there isn't sufficient information to write the
	80	+ message.
	81	+ """
	82	+ super().__init__(message)
	83	+ pass
	84	+
	85	+
	86	+class _Resolver():
	87	+ """A class for resolving symlinks inside CAS-based directories. As
	88	+ well as providing a namespace for some functions, this also
	89	+ contains two flags which are constant throughout one resolution
	90	+ operation and the 'seen_objects' list used to detect infinite
	91	+ symlink loops.
	92	+
	93	+ """
	94	+
	95	+ def __init__(self, absolute_symlinks_resolve=True, force_create=False):
	96	+ self.absolute_symlinks_resolve = absolute_symlinks_resolve
	97	+ self.force_create = force_create
	98	+ self.seen_objects = []
	99	+
	100	+ def resolve(self, name, directory):
	101	+ """Resolves any name to an object. If the name points to a symlink in
	102	+ the directory, it returns the thing it points to,
	103	+ recursively.
	104	+
	105	+ Returns a CasBasedDirectory, FileNode or None. None indicates
	106	+ either that 'target' does not exist in this directory, or is a
	107	+ symlink chain which points to a nonexistent name (broken
	108	+ symlink).
	109	+
	110	+ Raises:
	111	+
	112	+ - InfiniteSymlinkException if 'name' points to an infinite
	113	+ symlink loop.
	114	+ - AbsoluteSymlinkException if 'name' points to an absolute
	115	+ symlink and absolute_symlinks_resolve is False.
	116	+ - UnexpectedFileException if at any point during resolution we
	117	+ find a file which we expected to be a directory or symlink.
	118	+
	119	+ If force_create is set, this will attempt to create
	120	+ directories to make symlinks and directories resolve. Files
	121	+ present in symlink target paths will also be removed and
	122	+ replaced with directories. If force_create is off, this will
	123	+ never alter 'directory'.
	124	+
	125	+ """
	126	+
	127	+ # First check for nonexistent things or 'normal' objects and return them
	128	+ if name not in directory.index:
	129	+ return None
	130	+ index_entry = directory.index[name]
	131	+ if isinstance(index_entry.buildstream_object, Directory):
	132	+ return index_entry.buildstream_object
	133	+ elif isinstance(index_entry.pb_object, remote_execution_pb2.FileNode):
	134	+ return index_entry.pb_object
	135	+
	136	+ # Now we must be dealing with a symlink.
	137	+ assert isinstance(index_entry.pb_object, remote_execution_pb2.SymlinkNode)
	138	+
	139	+ symlink_object = index_entry.pb_object
	140	+ if symlink_object in self.seen_objects:
	141	+ # Infinite symlink loop detected
	142	+ message = ("Infinite symlink loop found during resolution. " +
	143	+ "First repeated element is {}".format(name))
	144	+ raise InfiniteSymlinkException(message=message)
	145	+
	146	+ self.seen_objects.append(symlink_object)
	147	+
	148	+ components = symlink_object.target.split(CasBasedDirectory._pb2_path_sep)
	149	+ absolute = symlink_object.target.startswith(CasBasedDirectory._pb2_absolute_path_prefix)
	150	+
	151	+ if absolute:
	152	+ if self.absolute_symlinks_resolve:
	153	+ directory = directory.find_root()
	154	+ # Discard the first empty element
	155	+ components.pop(0)
	156	+ else:
	157	+ # Unresolvable absolute symlink
	158	+ message = "{} is an absolute symlink, which was disallowed during resolution".format(name)
	159	+ raise AbsoluteSymlinkException(message=message)
	160	+
	161	+ resolution = directory
	162	+ while components and isinstance(resolution, CasBasedDirectory):
	163	+ c = components.pop(0)
	164	+ directory = resolution
	165	+
	166	+ try:
	167	+ resolution = self._resolve_path_component(c, directory, components)
	168	+ except UnexpectedFileException as original:
	169	+ errormsg = ("Reached a file called {} while trying to resolve a symlink; " +
	170	+ "cannot proceed. The remaining path components are {}.")
	171	+ raise UnexpectedFileException(errormsg.format(c, components)) from original
	172	+
	173	+ return resolution
	174	+
	175	+ def _resolve_path_component(self, c, directory, components_remaining):
	176	+ if c == ".":
	177	+ resolution = directory
	178	+ elif c == "..":
	179	+ if directory.parent is not None:
	180	+ resolution = directory.parent
	181	+ else:
	182	+ # If directory.parent *is* None, this is an attempt to
	183	+ # access '..' from the root, which is valid under
	184	+ # POSIX; it just returns the root.
	185	+ resolution = directory
	186	+ elif c in directory.index:
	187	+ try:
	188	+ resolution = self._resolve_through_files(c, directory, components_remaining)
	189	+ except UnexpectedFileException as original:
	190	+ errormsg = ("Reached a file called {} while trying to resolve a symlink; " +
	191	+ "cannot proceed. The remaining path components are {}.")
	192	+ raise UnexpectedFileException(errormsg.format(c, components_remaining)) from original
	193	+ else:
	194	+ # c is not in our index
	195	+ if self.force_create:
	196	+ resolution = directory.descend(c, create=True)
	197	+ else:
	198	+ resolution = None
	199	+ return resolution
	200	+
	201	+ def _resolve_through_files(self, c, directory, require_traversable):
	202	+ """A wrapper to resolve() which deals with files being found
	203	+ in the middle of paths, for example trying to resolve a symlink
	204	+ which points to /usr/lib64/libfoo when 'lib64' is a file.
	205	+
	206	+ require_traversable: If this is True, never return a file
	207	+ node. Instead, if force_create is set, destroy the file node,
	208	+ then create and return a normal directory in its place. If
	209	+ force_create is off, throws ResolutionException.
	210	+
	211	+ """
	212	+ resolved_thing = self.resolve(c, directory)
	213	+
	214	+ if isinstance(resolved_thing, remote_execution_pb2.FileNode):
	215	+ if require_traversable:
	216	+ # We have components still to resolve, but one of the path components
	217	+ # is a file.
	218	+ if self.force_create:
	219	+ directory.delete_entry(c)
	220	+ resolved_thing = directory.descend(c, create=True)
	221	+ else:
	222	+ # This is a signal that we hit a file, but don't
	223	+ # have the data to give a proper message, so the
	224	+ # caller should reraise this with a proper
	225	+ # description.
	226	+ raise UnexpectedFileException()
	227	+
	228	+ return resolved_thing
	229	+
	230	+
54	231	# CasBasedDirectory intentionally doesn't call its superclass constuctor,
55	232	# which is meant to be unimplemented.
56	233	# pylint: disable=super-init-not-called
...	...	@@ -168,29 +345,34 @@ class CasBasedDirectory(Directory):
168	345	self.index[name] = IndexEntry(dirnode, buildstream_object=newdir)
169	346	return newdir
170	347
171		- def _add_new_file(self, basename, filename):
	348	+ def _add_file(self, basename, filename, modified=False):
172	349	filenode = self.pb2_directory.files.add()
173	350	filenode.name = filename
174	351	self.cas_cache.add_object(digest=filenode.digest, path=os.path.join(basename, filename))
175	352	is_executable = os.access(os.path.join(basename, filename), os.X_OK)
176	353	filenode.is_executable = is_executable
177		- self.index[filename] = IndexEntry(filenode, modified=(filename in self.index))
	354	+ self.index[filename] = IndexEntry(filenode, modified=modified or filename in self.index)
178	355
179		- def _add_new_link(self, basename, filename):
180		- existing_link = self._find_pb2_entry(filename)
	356	+ def _copy_link_from_filesystem(self, basename, filename):
	357	+ self._add_new_link_direct(filename, os.readlink(os.path.join(basename, filename)))
	358	+
	359	+ def _add_new_link_direct(self, name, target):
	360	+ existing_link = self._find_pb2_entry(name)
181	361	if existing_link:
182	362	symlinknode = existing_link
183	363	else:
184	364	symlinknode = self.pb2_directory.symlinks.add()
185		- symlinknode.name = filename
	365	+ assert isinstance(symlinknode, remote_execution_pb2.SymlinkNode)
	366	+ symlinknode.name = name
186	367	# A symlink node has no digest.
187		- symlinknode.target = os.readlink(os.path.join(basename, filename))
188		- self.index[filename] = IndexEntry(symlinknode, modified=(existing_link is not None))
	368	+ symlinknode.target = target
	369	+ self.index[name] = IndexEntry(symlinknode, modified=(existing_link is not None))
189	370
190	371	def delete_entry(self, name):
191	372	for collection in [self.pb2_directory.files, self.pb2_directory.symlinks, self.pb2_directory.directories]:
192		- if name in collection:
193		- collection.remove(name)
	373	+ for thing in collection:
	374	+ if thing.name == name:
	375	+ collection.remove(thing)
194	376	if name in self.index:
195	377	del self.index[name]
196	378
...	...	@@ -231,9 +413,13 @@ class CasBasedDirectory(Directory):
231	413	if isinstance(entry, CasBasedDirectory):
232	414	return entry.descend(subdirectory_spec[1:], create)
233	415	else:
	416	+ # May be a symlink
	417	+ target = self._resolve(subdirectory_spec[0], force_create=create)
	418	+ if isinstance(target, CasBasedDirectory):
	419	+ return target
234	420	error = "Cannot descend into {}, which is a '{}' in the directory {}"
235	421	raise VirtualDirectoryError(error.format(subdirectory_spec[0],
236		- type(entry).__name__,
	422	+ type(self.index[subdirectory_spec[0]].pb_object).__name__,
237	423	self))
238	424	else:
239	425	if create:
...	...	@@ -254,36 +440,9 @@ class CasBasedDirectory(Directory):
254	440	else:
255	441	return self
256	442
257		- def _resolve_symlink_or_directory(self, name):
258		- """Used only by _import_files_from_directory. Tries to resolve a
259		- directory name or symlink name. 'name' must be an entry in this
260		- directory. It must be a single symlink or directory name, not a path
261		- separated by path separators. If it's an existing directory name, it
262		- just returns the Directory object for that. If it's a symlink, it will
263		- attempt to find the target of the symlink and return that as a
264		- Directory object.
265		-
266		- If a symlink target doesn't exist, it will attempt to create it
267		- as a directory as long as it's within this directory tree.
268		- """
269		-
270		- if isinstance(self.index[name].buildstream_object, Directory):
271		- return self.index[name].buildstream_object
272		- # OK then, it's a symlink
273		- symlink = self._find_pb2_entry(name)
274		- absolute = symlink.target.startswith(CasBasedDirectory._pb2_absolute_path_prefix)
275		- if absolute:
276		- root = self.find_root()
277		- else:
278		- root = self
279		- directory = root
280		- components = symlink.target.split(CasBasedDirectory._pb2_path_sep)
281		- for c in components:
282		- if c == "..":
283		- directory = directory.parent
284		- else:
285		- directory = directory.descend(c, create=True)
286		- return directory
	443	+ def _resolve(self, name, absolute_symlinks_resolve=True, force_create=False):
	444	+ resolver = _Resolver(absolute_symlinks_resolve, force_create)
	445	+ return resolver.resolve(name, self)
287	446
288	447	def _check_replacement(self, name, path_prefix, fileListResult):
289	448	""" Checks whether 'name' exists, and if so, whether we can overwrite it.
...	...	@@ -297,6 +456,7 @@ class CasBasedDirectory(Directory):
297	456	return True
298	457	if (isinstance(existing_entry,
299	458	(remote_execution_pb2.FileNode, remote_execution_pb2.SymlinkNode))):
	459	+ self.delete_entry(name)
300	460	fileListResult.overwritten.append(relative_pathname)
301	461	return True
302	462	elif isinstance(existing_entry, remote_execution_pb2.DirectoryNode):
...	...	@@ -314,23 +474,44 @@ class CasBasedDirectory(Directory):
314	474	.format(name, type(existing_entry)))
315	475	return False # In case asserts are disabled
316	476
317		- def _import_directory_recursively(self, directory_name, source_directory, remaining_path, path_prefix):
318		- """ _import_directory_recursively and _import_files_from_directory will be called alternately
319		- as a directory tree is descended. """
320		- if directory_name in self.index:
321		- subdir = self._resolve_symlink_or_directory(directory_name)
322		- else:
323		- subdir = self._add_directory(directory_name)
324		- new_path_prefix = os.path.join(path_prefix, directory_name)
325		- subdir_result = subdir._import_files_from_directory(os.path.join(source_directory, directory_name),
326		- [os.path.sep.join(remaining_path)],
327		- path_prefix=new_path_prefix)
328		- return subdir_result
	477	+ def _replace_anything_with_dir(self, name, path_prefix, overwritten_files_list):
	478	+ self.delete_entry(name)
	479	+ subdir = self._add_directory(name)
	480	+ overwritten_files_list.append(os.path.join(path_prefix, name))
	481	+ return subdir
329	482
330	483	def _import_files_from_directory(self, source_directory, files, path_prefix=""):
331		- """ Imports files from a traditional directory """
	484	+ """ Imports files from a traditional directory. """
	485	+
	486	+ def _ensure_followable(name, path_prefix):
	487	+ """ Makes sure 'name' is a directory or symlink to a directory which can be descended into. """
	488	+ if isinstance(self.index[name].buildstream_object, Directory):
	489	+ return self.descend(name)
	490	+ try:
	491	+ target = self._resolve(name, force_create=True)
	492	+ except InfiniteSymlinkException:
	493	+ return self._replace_anything_with_dir(name, path_prefix, result.overwritten)
	494	+ if isinstance(target, CasBasedDirectory):
	495	+ return target
	496	+ elif isinstance(target, remote_execution_pb2.FileNode):
	497	+ return self._replace_anything_with_dir(name, path_prefix, result.overwritten)
	498	+ return target
	499	+
	500	+ def _import_directory_recursively(directory_name, source_directory, remaining_path, path_prefix):
	501	+ """ _import_directory_recursively and _import_files_from_directory will be called alternately
	502	+ as a directory tree is descended. """
	503	+ if directory_name in self.index:
	504	+ subdir = _ensure_followable(directory_name, path_prefix)
	505	+ else:
	506	+ subdir = self._add_directory(directory_name)
	507	+ new_path_prefix = os.path.join(path_prefix, directory_name)
	508	+ subdir_result = subdir._import_files_from_directory(os.path.join(source_directory, directory_name),
	509	+ [os.path.sep.join(remaining_path)],
	510	+ path_prefix=new_path_prefix)
	511	+ return subdir_result
	512	+
332	513	result = FileListResult()
333		- for entry in sorted(files):
	514	+ for entry in files:
334	515	split_path = entry.split(os.path.sep)
335	516	# The actual file on the FS we're importing
336	517	import_file = os.path.join(source_directory, entry)
...	...	@@ -338,14 +519,18 @@ class CasBasedDirectory(Directory):
338	519	relative_pathname = os.path.join(path_prefix, entry)
339	520	if len(split_path) > 1:
340	521	directory_name = split_path[0]
341		- # Hand this off to the importer for that subdir. This will only do one file -
342		- # a better way would be to hand off all the files in this subdir at once.
343		- subdir_result = self._import_directory_recursively(directory_name, source_directory,
344		- split_path[1:], path_prefix)
	522	+ # Hand this off to the importer for that subdir.
	523	+
	524	+ # It would be advantageous to batch these together by
	525	+ # directory_name. However, we can't do it out of
	526	+ # order, since importing symlinks affects the results
	527	+ # of other imports.
	528	+ subdir_result = _import_directory_recursively(directory_name, source_directory,
	529	+ split_path[1:], path_prefix)
345	530	result.combine(subdir_result)
346	531	elif os.path.islink(import_file):
347	532	if self._check_replacement(entry, path_prefix, result):
348		- self._add_new_link(source_directory, entry)
	533	+ self._copy_link_from_filesystem(source_directory, entry)
349	534	result.files_written.append(relative_pathname)
350	535	elif os.path.isdir(import_file):
351	536	# A plain directory which already exists isn't a problem; just ignore it.
...	...	@@ -353,10 +538,78 @@ class CasBasedDirectory(Directory):
353	538	self._add_directory(entry)
354	539	elif os.path.isfile(import_file):
355	540	if self._check_replacement(entry, path_prefix, result):
356		- self._add_new_file(source_directory, entry)
	541	+ self._add_file(source_directory, entry, modified=relative_pathname in result.overwritten)
357	542	result.files_written.append(relative_pathname)
358	543	return result
359	544
	545	+ @staticmethod
	546	+ def _files_in_subdir(sorted_files, dirname):
	547	+ """Filters sorted_files and returns only the ones which have
	548	+ 'dirname' as a prefix, with that prefix removed.
	549	+
	550	+ """
	551	+ if not dirname.endswith(os.path.sep):
	552	+ dirname += os.path.sep
	553	+ return [f[len(dirname):] for f in sorted_files if f.startswith(dirname)]
	554	+
	555	+ def _partial_import_cas_into_cas(self, source_directory, files, path_prefix="", file_list_required=True):
	556	+ """ Import only the files and symlinks listed in 'files' from source_directory to this one.
	557	+ Args:
	558	+ source_directory (:class:`.CasBasedDirectory`): The directory to import from
	559	+ files ([str]): List of pathnames to import. Must be a list, not a generator.
	560	+ path_prefix (str): Prefix used to add entries to the file list result.
	561	+ file_list_required: Whether to update the file list while processing.
	562	+ """
	563	+ result = FileListResult()
	564	+ processed_directories = set()
	565	+ for f in files:
	566	+ fullname = os.path.join(path_prefix, f)
	567	+ components = f.split(os.path.sep)
	568	+ if len(components) > 1:
	569	+ # We are importing a thing which is in a subdirectory. We may have already seen this dirname
	570	+ # for a previous file.
	571	+ dirname = components[0]
	572	+ if dirname not in processed_directories:
	573	+ # Now strip off the first directory name and import files recursively.
	574	+ subcomponents = CasBasedDirectory._files_in_subdir(files, dirname)
	575	+ # We will fail at this point if there is a file or symlink to file called 'dirname'.
	576	+ if dirname in self.index:
	577	+ resolved_component = self._resolve(dirname, force_create=True)
	578	+ if isinstance(resolved_component, remote_execution_pb2.FileNode):
	579	+ dest_subdir = self._replace_anything_with_dir(dirname, path_prefix, result.overwritten)
	580	+ else:
	581	+ dest_subdir = resolved_component
	582	+ else:
	583	+ dest_subdir = self.descend(dirname, create=True)
	584	+ src_subdir = source_directory.descend(dirname)
	585	+ import_result = dest_subdir._partial_import_cas_into_cas(src_subdir, subcomponents,
	586	+ path_prefix=fullname,
	587	+ file_list_required=file_list_required)
	588	+ result.combine(import_result)
	589	+ processed_directories.add(dirname)
	590	+ elif isinstance(source_directory.index[f].buildstream_object, CasBasedDirectory):
	591	+ # The thing in the input file list is a directory on
	592	+ # its own. We don't need to do anything other than create it if it doesn't exist.
	593	+ # If we already have an entry with the same name that isn't a directory, that
	594	+ # will be dealt with when importing files in this directory.
	595	+ if f not in self.index:
	596	+ self.descend(f, create=True)
	597	+ else:
	598	+ # We're importing a file or symlink - replace anything with the same name.
	599	+ importable = self._check_replacement(f, path_prefix, result)
	600	+ if importable:
	601	+ item = source_directory.index[f].pb_object
	602	+ if isinstance(item, remote_execution_pb2.FileNode):
	603	+ filenode = self.pb2_directory.files.add(digest=item.digest, name=f,
	604	+ is_executable=item.is_executable)
	605	+ self.index[f] = IndexEntry(filenode, modified=True)
	606	+ else:
	607	+ assert isinstance(item, remote_execution_pb2.SymlinkNode)
	608	+ self._add_new_link_direct(name=f, target=item.target)
	609	+ else:
	610	+ result.ignored.append(os.path.join(path_prefix, f))
	611	+ return result
	612	+
360	613	def import_files(self, external_pathspec, *, files=None,
361	614	report_written=True, update_utimes=False,
362	615	can_link=False):
...	...	@@ -378,28 +631,27 @@ class CasBasedDirectory(Directory):
378	631
379	632	can_link (bool): Ignored, since hard links do not have any meaning within CAS.
380	633	"""
381		- if isinstance(external_pathspec, FileBasedDirectory):
382		- source_directory = external_pathspec._get_underlying_directory()
383		- elif isinstance(external_pathspec, CasBasedDirectory):
384		- # TODO: This transfers from one CAS to another via the
385		- # filesystem, which is very inefficient. Alter this so it
386		- # transfers refs across directly.
387		- with tempfile.TemporaryDirectory(prefix="roundtrip") as tmpdir:
388		- external_pathspec.export_files(tmpdir)
389		- if files is None:
390		- files = list_relative_paths(tmpdir)
391		- result = self._import_files_from_directory(tmpdir, files=files)
392		- return result
393		- else:
394		- source_directory = external_pathspec
395	634
396	635	if files is None:
397		- files = list_relative_paths(source_directory)
	636	+ if isinstance(external_pathspec, str):
	637	+ files = list_relative_paths(external_pathspec)
	638	+ else:
	639	+ assert isinstance(external_pathspec, Directory)
	640	+ files = external_pathspec.list_relative_paths()
	641	+
	642	+ if isinstance(external_pathspec, FileBasedDirectory):
	643	+ source_directory = external_pathspec.get_underlying_directory()
	644	+ result = self._import_files_from_directory(source_directory, files=files)
	645	+ elif isinstance(external_pathspec, str):
	646	+ source_directory = external_pathspec
	647	+ result = self._import_files_from_directory(source_directory, files=files)
	648	+ else:
	649	+ assert isinstance(external_pathspec, CasBasedDirectory)
	650	+ result = self._partial_import_cas_into_cas(external_pathspec, files=list(files))
398	651
399	652	# TODO: No notice is taken of report_written, update_utimes or can_link.
400	653	# Current behaviour is to fully populate the report, which is inefficient,
401	654	# but still correct.
402		- result = self._import_files_from_directory(source_directory, files=files)
403	655
404	656	# We need to recalculate and store the hashes of all directories both
405	657	# up and down the tree; we have changed our directory by importing files
...	...	@@ -511,6 +763,28 @@ class CasBasedDirectory(Directory):
511	763	else:
512	764	self._mark_directory_unmodified()
513	765
	766	+ def _lightweight_resolve_to_index(self, path):
	767	+ """A lightweight function for transforming paths into IndexEntry
	768	+ objects. This does not follow symlinks.
	769	+
	770	+ path: The string to resolve. This should be a series of path
	771	+ components separated by the protocol buffer path separator
	772	+ _pb2_path_sep.
	773	+
	774	+ Returns: the IndexEntry found, or None if any of the path components were not present.
	775	+
	776	+ """
	777	+ directory = self
	778	+ path_components = path.split(CasBasedDirectory._pb2_path_sep)
	779	+ for component in path_components[:-1]:
	780	+ if component not in directory.index:
	781	+ return None
	782	+ if isinstance(directory.index[component].buildstream_object, CasBasedDirectory):
	783	+ directory = directory.index[component].buildstream_object
	784	+ else:
	785	+ return None
	786	+ return directory.index.get(path_components[-1], None)
	787	+
514	788	def list_modified_paths(self):
515	789	"""Provide a list of relative paths which have been modified since the
516	790	last call to mark_unmodified.
...	...	@@ -518,29 +792,43 @@ class CasBasedDirectory(Directory):
518	792	Return value: List(str) - list of modified paths
519	793	"""
520	794
521		- filelist = []
522		- for (k, v) in self.index.items():
523		- if isinstance(v.buildstream_object, CasBasedDirectory):
524		- filelist.extend([k + os.path.sep + x for x in v.buildstream_object.list_modified_paths()])
525		- elif isinstance(v.pb_object, remote_execution_pb2.FileNode) and v.modified:
526		- filelist.append(k)
527		- return filelist
	795	+ for p in self.list_relative_paths():
	796	+ i = self._lightweight_resolve_to_index(p)
	797	+ if i and i.modified:
	798	+ yield p
528	799
529		- def list_relative_paths(self):
	800	+ def list_relative_paths(self, relpath=""):
530	801	"""Provide a list of all relative paths.
531	802
532		- NOTE: This list is not in the same order as utils.list_relative_paths.
533		-
534	803	Return value: List(str) - list of all paths
535	804	"""
536	805
537		- filelist = []
538		- for (k, v) in self.index.items():
539		- if isinstance(v.buildstream_object, CasBasedDirectory):
540		- filelist.extend([k + os.path.sep + x for x in v.buildstream_object.list_relative_paths()])
541		- elif isinstance(v.pb_object, remote_execution_pb2.FileNode):
542		- filelist.append(k)
543		- return filelist
	806	+ symlink_list = filter(lambda i: isinstance(i[1].pb_object, remote_execution_pb2.SymlinkNode),
	807	+ self.index.items())
	808	+ file_list = list(filter(lambda i: isinstance(i[1].pb_object, remote_execution_pb2.FileNode),
	809	+ self.index.items()))
	810	+ directory_list = filter(lambda i: isinstance(i[1].buildstream_object, CasBasedDirectory),
	811	+ self.index.items())
	812	+
	813	+ # We need to mimic the behaviour of os.walk, in which symlinks
	814	+ # to directories count as directories and symlinks to file or
	815	+ # broken symlinks count as files. os.walk doesn't follow
	816	+ # symlinks, so we don't recurse.
	817	+ for (k, v) in sorted(symlink_list):
	818	+ target = self._resolve(k, absolute_symlinks_resolve=True)
	819	+ if isinstance(target, CasBasedDirectory):
	820	+ yield os.path.join(relpath, k)
	821	+ else:
	822	+ file_list.append((k, v))
	823	+
	824	+ if file_list == [] and relpath != "":
	825	+ yield relpath
	826	+ else:
	827	+ for (k, v) in sorted(file_list):
	828	+ yield os.path.join(relpath, k)
	829	+
	830	+ for (k, v) in sorted(directory_list):
	831	+ yield from v.buildstream_object.list_relative_paths(relpath=os.path.join(relpath, k))
544	832
545	833	def recalculate_hash(self):
546	834	""" Recalcuates the hash for this directory and store the results in

tests/storage/virtual_directory_import.py

---

	1	+from hashlib import sha256
	2	+import os
	3	+import pytest
	4	+import random
	5	+import tempfile
	6	+from tests.testutils import cli
	7	+
	8	+from buildstream.storage._casbaseddirectory import CasBasedDirectory
	9	+from buildstream.storage._filebaseddirectory import FileBasedDirectory
	10	+from buildstream._artifactcache import ArtifactCache
	11	+from buildstream._artifactcache.cascache import CASCache
	12	+from buildstream import utils
	13	+
	14	+
	15	+# These are comparitive tests that check that FileBasedDirectory and
	16	+# CasBasedDirectory act identically.
	17	+
	18	+
	19	+class FakeArtifactCache():
	20	+ def __init__(self):
	21	+ self.cas = None
	22	+
	23	+
	24	+class FakeContext():
	25	+ def __init__(self):
	26	+ self.artifactdir = ''
	27	+ self.artifactcache = FakeArtifactCache()
	28	+
	29	+
	30	+# This is a set of example file system contents. It's a set of trees
	31	+# which are either expected to be problematic or were found to be
	32	+# problematic during random testing.
	33	+
	34	+# The test attempts to import each on top of each other to test
	35	+# importing works consistently. Each tuple is defined as (<filename>,
	36	+# <type>, <content>). Type can be 'F' (file), 'S' (symlink) or 'D'
	37	+# (directory) with content being the contents for a file or the
	38	+# destination for a symlink.
	39	+root_filesets = [
	40	+ [('a/b/c/textfile1', 'F', 'This is textfile 1\n')],
	41	+ [('a/b/c/textfile1', 'F', 'This is the replacement textfile 1\n')],
	42	+ [('a/b/d', 'D', '')],
	43	+ [('a/b/c', 'S', '/a/b/d')],
	44	+ [('a/b/d', 'S', '/a/b/c')],
	45	+ [('a/b/d', 'D', ''), ('a/b/c', 'S', '/a/b/d')],
	46	+ [('a/b/c', 'D', ''), ('a/b/d', 'S', '/a/b/c')],
	47	+ [('a/b', 'F', 'This is textfile 1\n')],
	48	+ [('a/b/c', 'F', 'This is textfile 1\n')],
	49	+ [('a/b/c', 'D', '')]
	50	+]
	51	+
	52	+empty_hash_ref = sha256().hexdigest()
	53	+RANDOM_SEED = 69105
	54	+NUM_RANDOM_TESTS = 10
	55	+
	56	+
	57	+def generate_import_roots(rootno, directory):
	58	+ rootname = "root{}".format(rootno)
	59	+ rootdir = os.path.join(directory, "content", rootname)
	60	+ if os.path.exists(rootdir):
	61	+ return
	62	+ for (path, typesymbol, content) in root_filesets[rootno - 1]:
	63	+ if typesymbol == 'F':
	64	+ (dirnames, filename) = os.path.split(path)
	65	+ os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
	66	+ with open(os.path.join(rootdir, dirnames, filename), "wt") as f:
	67	+ f.write(content)
	68	+ elif typesymbol == 'D':
	69	+ os.makedirs(os.path.join(rootdir, path), exist_ok=True)
	70	+ elif typesymbol == 'S':
	71	+ (dirnames, filename) = os.path.split(path)
	72	+ os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
	73	+ os.symlink(content, os.path.join(rootdir, path))
	74	+
	75	+
	76	+def generate_random_root(rootno, directory):
	77	+ # By seeding the random number generator, we ensure these tests
	78	+ # will be repeatable, at least until Python changes the random
	79	+ # number algorithm.
	80	+ random.seed(RANDOM_SEED + rootno)
	81	+ rootname = "root{}".format(rootno)
	82	+ rootdir = os.path.join(directory, "content", rootname)
	83	+ if os.path.exists(rootdir):
	84	+ return
	85	+ things = []
	86	+ locations = ['.']
	87	+ os.makedirs(rootdir)
	88	+ for i in range(0, 100):
	89	+ location = random.choice(locations)
	90	+ thingname = "node{}".format(i)
	91	+ thing = random.choice(['dir', 'link', 'file'])
	92	+ target = os.path.join(rootdir, location, thingname)
	93	+ if thing == 'dir':
	94	+ os.makedirs(target)
	95	+ locations.append(os.path.join(location, thingname))
	96	+ elif thing == 'file':
	97	+ with open(target, "wt") as f:
	98	+ f.write("This is node {}\n".format(i))
	99	+ elif thing == 'link':
	100	+ symlink_type = random.choice(['absolute', 'relative', 'broken'])
	101	+ if symlink_type == 'broken' or not things:
	102	+ os.symlink("/broken", target)
	103	+ elif symlink_type == 'absolute':
	104	+ symlink_destination = random.choice(things)
	105	+ os.symlink(symlink_destination, target)
	106	+ else:
	107	+ symlink_destination = random.choice(things)
	108	+ relative_link = os.path.relpath(symlink_destination, start=location)
	109	+ os.symlink(relative_link, target)
	110	+ things.append(os.path.join(location, thingname))
	111	+
	112	+
	113	+def file_contents(path):
	114	+ with open(path, "r") as f:
	115	+ result = f.read()
	116	+ return result
	117	+
	118	+
	119	+def file_contents_are(path, contents):
	120	+ return file_contents(path) == contents
	121	+
	122	+
	123	+def create_new_casdir(root_number, fake_context, tmpdir):
	124	+ d = CasBasedDirectory(fake_context)
	125	+ d.import_files(os.path.join(tmpdir, "content", "root{}".format(root_number)))
	126	+ assert d.ref.hash != empty_hash_ref
	127	+ return d
	128	+
	129	+
	130	+def create_new_filedir(root_number, tmpdir):
	131	+ root = os.path.join(tmpdir, "vdir")
	132	+ os.makedirs(root)
	133	+ d = FileBasedDirectory(root)
	134	+ d.import_files(os.path.join(tmpdir, "content", "root{}".format(root_number)))
	135	+ return d
	136	+
	137	+
	138	+def combinations(integer_range):
	139	+ for x in integer_range:
	140	+ for y in integer_range:
	141	+ yield (x, y)
	142	+
	143	+
	144	+def resolve_symlinks(path, root):
	145	+ """ A function to resolve symlinks inside 'path' components apart from the last one.
	146	+ For example, resolve_symlinks('/a/b/c/d', '/a/b')
	147	+ will return '/a/b/f/d' if /a/b/c is a symlink to /a/b/f. The final component of
	148	+ 'path' is not resolved, because we typically want to inspect the symlink found
	149	+ at that path, not its target.
	150	+
	151	+ """
	152	+ components = path.split(os.path.sep)
	153	+ location = root
	154	+ for i in range(0, len(components) - 1):
	155	+ location = os.path.join(location, components[i])
	156	+ if os.path.islink(location):
	157	+ # Resolve the link, add on all the remaining components
	158	+ target = os.path.join(os.readlink(location))
	159	+ tail = os.path.sep.join(components[i + 1:])
	160	+
	161	+ if target.startswith(os.path.sep):
	162	+ # Absolute link - relative to root
	163	+ location = os.path.join(root, target, tail)
	164	+ else:
	165	+ # Relative link - relative to symlink location
	166	+ location = os.path.join(location, target)
	167	+ return resolve_symlinks(location, root)
	168	+ # If we got here, no symlinks were found. Add on the final component and return.
	169	+ location = os.path.join(location, components[-1])
	170	+ return location
	171	+
	172	+
	173	+def directory_not_empty(path):
	174	+ return os.listdir(path)
	175	+
	176	+
	177	+def _import_test(tmpdir, original, overlay, generator_function, verify_contents=False):
	178	+ fake_context = FakeContext()
	179	+ fake_context.artifactcache.cas = CASCache(tmpdir)
	180	+ # Create some fake content
	181	+ generator_function(original, tmpdir)
	182	+ if original != overlay:
	183	+ generator_function(overlay, tmpdir)
	184	+
	185	+ d = create_new_casdir(original, fake_context, tmpdir)
	186	+
	187	+ duplicate_cas = create_new_casdir(original, fake_context, tmpdir)
	188	+
	189	+ assert duplicate_cas.ref.hash == d.ref.hash
	190	+
	191	+ d2 = create_new_casdir(overlay, fake_context, tmpdir)
	192	+ d.import_files(d2)
	193	+ export_dir = os.path.join(tmpdir, "output-{}-{}".format(original, overlay))
	194	+ roundtrip_dir = os.path.join(tmpdir, "roundtrip-{}-{}".format(original, overlay))
	195	+ d2.export_files(roundtrip_dir)
	196	+ d.export_files(export_dir)
	197	+
	198	+ if verify_contents:
	199	+ for item in root_filesets[overlay - 1]:
	200	+ (path, typename, content) = item
	201	+ realpath = resolve_symlinks(path, export_dir)
	202	+ if typename == 'F':
	203	+ if os.path.isdir(realpath) and directory_not_empty(realpath):
	204	+ # The file should not have overwritten the directory in this case.
	205	+ pass
	206	+ else:
	207	+ assert os.path.isfile(realpath), "{} did not exist in the combined virtual directory".format(path)
	208	+ assert file_contents_are(realpath, content)
	209	+ elif typename == 'S':
	210	+ if os.path.isdir(realpath) and directory_not_empty(realpath):
	211	+ # The symlink should not have overwritten the directory in this case.
	212	+ pass
	213	+ else:
	214	+ assert os.path.islink(realpath)
	215	+ assert os.readlink(realpath) == content
	216	+ elif typename == 'D':
	217	+ # We can't do any more tests than this because it
	218	+ # depends on things present in the original. Blank
	219	+ # directories here will be ignored and the original
	220	+ # left in place.
	221	+ assert os.path.lexists(realpath)
	222	+
	223	+ # Now do the same thing with filebaseddirectories and check the contents match
	224	+
	225	+ files = list(utils.list_relative_paths(roundtrip_dir))
	226	+ duplicate_cas._import_files_from_directory(roundtrip_dir, files=files)
	227	+ duplicate_cas._recalculate_recursing_down()
	228	+ if duplicate_cas.parent:
	229	+ duplicate_cas.parent._recalculate_recursing_up(duplicate_cas)
	230	+
	231	+ assert duplicate_cas.ref.hash == d.ref.hash
	232	+
	233	+
	234	+# It's possible to parameterize on both original and overlay values,
	235	+# but this leads to more tests being listed in the output than are
	236	+# comfortable.
	237	+@pytest.mark.parametrize("original", range(1, len(root_filesets) + 1))
	238	+def test_fixed_cas_import(cli, tmpdir, original):
	239	+ for overlay in range(1, len(root_filesets) + 1):
	240	+ _import_test(str(tmpdir), original, overlay, generate_import_roots, verify_contents=True)
	241	+
	242	+
	243	+@pytest.mark.parametrize("original", range(1, NUM_RANDOM_TESTS + 1))
	244	+def test_random_cas_import(cli, tmpdir, original):
	245	+ for overlay in range(1, NUM_RANDOM_TESTS + 1):
	246	+ _import_test(str(tmpdir), original, overlay, generate_random_root, verify_contents=False)
	247	+
	248	+
	249	+def _listing_test(tmpdir, root, generator_function):
	250	+ fake_context = FakeContext()
	251	+ fake_context.artifactcache.cas = CASCache(tmpdir)
	252	+ # Create some fake content
	253	+ generator_function(root, tmpdir)
	254	+
	255	+ d = create_new_filedir(root, tmpdir)
	256	+ filelist = list(d.list_relative_paths())
	257	+
	258	+ d2 = create_new_casdir(root, fake_context, tmpdir)
	259	+ filelist2 = list(d2.list_relative_paths())
	260	+
	261	+ assert filelist == filelist2
	262	+
	263	+
	264	+@pytest.mark.parametrize("root", range(1, 11))
	265	+def test_random_directory_listing(cli, tmpdir, root):
	266	+ _listing_test(str(tmpdir), root, generate_random_root)
	267	+
	268	+
	269	+@pytest.mark.parametrize("root", [1, 2, 3, 4, 5])
	270	+def test_fixed_directory_listing(cli, tmpdir, root):
	271	+ _listing_test(str(tmpdir), root, generate_import_roots)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.