[Notes] [Git][BuildStream/buildstream][master] 8 commits: _casbaseddirectory.py: Fix order in _recalculate_recursing_up()

[Jürg Billeter <gitlab mg gitlab com>]

Title: GitLab

Jürg Billeter pushed to branch master at BuildStream / buildstream

Commits:

• fb65af6c
  by Jürg Billeter at 2019-02-18T10:29:25Z

  _casbaseddirectory.py: Fix order in _recalculate_recursing_up()
  
  The directory needs to be serialized after the Digest for the
  subdirectory `caller` has been updated.
  

• f5660fa0
  by Jim MacArthur at 2019-02-18T10:29:25Z

  _casbaseddirectory.py: Correct typo for _get_underlying_directory
  

• cdcf0dc5
  by Jürg Billeter at 2019-02-18T10:29:25Z

  _casbaseddirectory.py: Remove unused find_root() method
  

• f9dd6ea2
  by Jim MacArthur at 2019-02-18T10:29:25Z

  storage: Add Directory.get_size()
  

• 050249bb
  by Jürg Billeter at 2019-02-18T10:29:25Z

  _casbaseddirectory.py: Add _get_digest() method
  

• 3b881efc
  by Jürg Billeter at 2019-02-18T10:29:25Z

  element.py: Expand scope of timed_activity in _cache_artifact()
  
  ArtifactCache.commit() is no longer the only expensive part and even
  more so with the upcoming change to use CASBasedDirectory.
  

• 3832c0d1
  by Jim MacArthur at 2019-02-18T10:29:25Z

  element.py: Alter _cache_artifact to use CasBasedDirectory
  

• ef85e3b2
  by Jürg Billeter at 2019-02-18T11:18:10Z

  Merge branch 'jmac/cache_artifacts_with_vdir' into 'master'
  
  Cache artifacts with virtual directories instead of filesystem.
  
  Closes #787
  
  See merge request BuildStream/buildstream!991

5 changed files:

• buildstream/_artifactcache.py
• buildstream/element.py
• buildstream/storage/_casbaseddirectory.py
• buildstream/storage/_filebaseddirectory.py
• buildstream/storage/directory.py

Changes:

buildstream/_artifactcache.py

---

...	...	@@ -588,13 +588,16 @@ class ArtifactCache():
588	588	#
589	589	# Args:
590	590	# element (Element): The Element commit an artifact for
591		- # content (str): The element's content directory
	591	+ # content (Directory): The element's content directory
592	592	# keys (list): The cache keys to use
593	593	#
594	594	def commit(self, element, content, keys):
595	595	refs = [element.get_artifact_name(key) for key in keys]
596	596
597		- self.cas.commit(refs, content)
	597	+ tree = content._get_digest()
	598	+
	599	+ for ref in refs:
	600	+ self.cas.set_ref(ref, tree)
598	601
599	602	# diff():
600	603	#

buildstream/element.py

---

...	...	@@ -103,6 +103,7 @@ from .types import _KeyStrength, CoreWarnings
103	103
104	104	from .storage.directory import Directory
105	105	from .storage._filebaseddirectory import FileBasedDirectory
	106	+from .storage._casbaseddirectory import CasBasedDirectory
106	107	from .storage.directory import VirtualDirectoryError
107	108
108	109
...	...	@@ -1670,106 +1671,109 @@ class Element(Plugin):
1670	1671	cleanup_rootdir()
1671	1672
1672	1673	def _cache_artifact(self, rootdir, sandbox, collect):
1673		- if collect is not None:
1674		- try:
1675		- sandbox_vroot = sandbox.get_virtual_directory()
1676		- collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
1677		- except VirtualDirectoryError:
1678		- # No collect directory existed
1679		- collectvdir = None
	1674	+ with self.timed_activity("Caching artifact"):
	1675	+ if collect is not None:
	1676	+ try:
	1677	+ sandbox_vroot = sandbox.get_virtual_directory()
	1678	+ collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
	1679	+ except VirtualDirectoryError:
	1680	+ # No collect directory existed
	1681	+ collectvdir = None
1680	1682
1681		- context = self._get_context()
	1683	+ context = self._get_context()
1682	1684
1683		- # Create artifact directory structure
1684		- assembledir = os.path.join(rootdir, 'artifact')
1685		- filesdir = os.path.join(assembledir, 'files')
1686		- logsdir = os.path.join(assembledir, 'logs')
1687		- metadir = os.path.join(assembledir, 'meta')
1688		- buildtreedir = os.path.join(assembledir, 'buildtree')
1689		- os.mkdir(assembledir)
1690		- if collect is not None and collectvdir is not None:
1691		- os.mkdir(filesdir)
1692		- os.mkdir(logsdir)
1693		- os.mkdir(metadir)
1694		- os.mkdir(buildtreedir)
1695		-
1696		- # Hard link files from collect dir to files directory
1697		- if collect is not None and collectvdir is not None:
1698		- collectvdir.export_files(filesdir, can_link=True)
1699		-
1700		- cache_buildtrees = context.cache_buildtrees
1701		- build_success = self.__build_result[0]
1702		-
1703		- # cache_buildtrees defaults to 'always', as such the
1704		- # default behaviour is to attempt to cache them. If only
1705		- # caching failed artifact buildtrees, then query the build
1706		- # result. Element types without a build-root dir will be cached
1707		- # with an empty buildtreedir regardless of this configuration.
1708		-
1709		- if cache_buildtrees == 'always' or (cache_buildtrees == 'failure' and not build_success):
1710		- try:
	1685	+ assemblevdir = CasBasedDirectory(cas_cache=context.artifactcache.cas, ref=None)
	1686	+ logsvdir = assemblevdir.descend("logs", create=True)
	1687	+ metavdir = assemblevdir.descend("meta", create=True)
	1688	+ buildtreevdir = assemblevdir.descend("buildtree", create=True)
	1689	+
	1690	+ # Create artifact directory structure
	1691	+ assembledir = os.path.join(rootdir, 'artifact')
	1692	+ logsdir = os.path.join(assembledir, 'logs')
	1693	+ metadir = os.path.join(assembledir, 'meta')
	1694	+ os.mkdir(assembledir)
	1695	+ os.mkdir(logsdir)
	1696	+ os.mkdir(metadir)
	1697	+
	1698	+ if collect is not None and collectvdir is not None:
	1699	+ filesvdir = assemblevdir.descend("files", create=True)
	1700	+ filesvdir.import_files(collectvdir)
	1701	+
	1702	+ cache_buildtrees = context.cache_buildtrees
	1703	+ build_success = self.__build_result[0]
	1704	+
	1705	+ # cache_buildtrees defaults to 'always', as such the
	1706	+ # default behaviour is to attempt to cache them. If only
	1707	+ # caching failed artifact buildtrees, then query the build
	1708	+ # result. Element types without a build-root dir will be cached
	1709	+ # with an empty buildtreedir regardless of this configuration.
	1710	+
	1711	+ if cache_buildtrees == 'always' or (cache_buildtrees == 'failure' and not build_success):
1711	1712	sandbox_vroot = sandbox.get_virtual_directory()
1712		- sandbox_build_dir = sandbox_vroot.descend(
1713		- self.get_variable('build-root').lstrip(os.sep).split(os.sep))
1714		- # Hard link files from build-root dir to buildtreedir directory
1715		- sandbox_build_dir.export_files(buildtreedir)
1716		- except VirtualDirectoryError:
1717		- # Directory could not be found. Pre-virtual
1718		- # directory behaviour was to continue silently
1719		- # if the directory could not be found.
1720		- pass
	1713	+ try:
	1714	+ sandbox_build_dir = sandbox_vroot.descend(
	1715	+ self.get_variable('build-root').lstrip(os.sep).split(os.sep))
	1716	+ buildtreevdir.import_files(sandbox_build_dir)
	1717	+ except VirtualDirectoryError:
	1718	+ # Directory could not be found. Pre-virtual
	1719	+ # directory behaviour was to continue silently
	1720	+ # if the directory could not be found.
	1721	+ pass
	1722	+
	1723	+ # Write some logs out to normal directories: logsdir and metadir
	1724	+ # Copy build log
	1725	+ log_filename = context.get_log_filename()
	1726	+ self._build_log_path = os.path.join(logsdir, 'build.log')
	1727	+ if log_filename:
	1728	+ shutil.copyfile(log_filename, self._build_log_path)
	1729	+
	1730	+ # Store public data
	1731	+ _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
	1732	+
	1733	+ # Store result
	1734	+ build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
	1735	+ if self.__build_result[2] is not None:
	1736	+ build_result_dict["detail"] = self.__build_result[2]
	1737	+ _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
	1738	+
	1739	+ # ensure we have cache keys
	1740	+ self._assemble_done()
	1741	+
	1742	+ # Store keys.yaml
	1743	+ _yaml.dump(_yaml.node_sanitize({
	1744	+ 'strong': self._get_cache_key(),
	1745	+ 'weak': self._get_cache_key(_KeyStrength.WEAK),
	1746	+ }), os.path.join(metadir, 'keys.yaml'))
	1747	+
	1748	+ # Store dependencies.yaml
	1749	+ _yaml.dump(_yaml.node_sanitize({
	1750	+ e.name: e._get_cache_key() for e in self.dependencies(Scope.BUILD)
	1751	+ }), os.path.join(metadir, 'dependencies.yaml'))
	1752	+
	1753	+ # Store workspaced.yaml
	1754	+ _yaml.dump(_yaml.node_sanitize({
	1755	+ 'workspaced': bool(self._get_workspace())
	1756	+ }), os.path.join(metadir, 'workspaced.yaml'))
	1757	+
	1758	+ # Store workspaced-dependencies.yaml
	1759	+ _yaml.dump(_yaml.node_sanitize({
	1760	+ 'workspaced-dependencies': [
	1761	+ e.name for e in self.dependencies(Scope.BUILD)
	1762	+ if e._get_workspace()
	1763	+ ]
	1764	+ }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
1721	1765
1722		- # Copy build log
1723		- log_filename = context.get_log_filename()
1724		- self._build_log_path = os.path.join(logsdir, 'build.log')
1725		- if log_filename:
1726		- shutil.copyfile(log_filename, self._build_log_path)
1727		-
1728		- # Store public data
1729		- _yaml.dump(_yaml.node_sanitize(self.__dynamic_public), os.path.join(metadir, 'public.yaml'))
1730		-
1731		- # Store result
1732		- build_result_dict = {"success": self.__build_result[0], "description": self.__build_result[1]}
1733		- if self.__build_result[2] is not None:
1734		- build_result_dict["detail"] = self.__build_result[2]
1735		- _yaml.dump(build_result_dict, os.path.join(metadir, 'build-result.yaml'))
1736		-
1737		- # ensure we have cache keys
1738		- self._assemble_done()
1739		-
1740		- # Store keys.yaml
1741		- _yaml.dump(_yaml.node_sanitize({
1742		- 'strong': self._get_cache_key(),
1743		- 'weak': self._get_cache_key(_KeyStrength.WEAK),
1744		- }), os.path.join(metadir, 'keys.yaml'))
1745		-
1746		- # Store dependencies.yaml
1747		- _yaml.dump(_yaml.node_sanitize({
1748		- e.name: e._get_cache_key() for e in self.dependencies(Scope.BUILD)
1749		- }), os.path.join(metadir, 'dependencies.yaml'))
1750		-
1751		- # Store workspaced.yaml
1752		- _yaml.dump(_yaml.node_sanitize({
1753		- 'workspaced': bool(self._get_workspace())
1754		- }), os.path.join(metadir, 'workspaced.yaml'))
1755		-
1756		- # Store workspaced-dependencies.yaml
1757		- _yaml.dump(_yaml.node_sanitize({
1758		- 'workspaced-dependencies': [
1759		- e.name for e in self.dependencies(Scope.BUILD)
1760		- if e._get_workspace()
1761		- ]
1762		- }), os.path.join(metadir, 'workspaced-dependencies.yaml'))
	1766	+ metavdir.import_files(metadir)
	1767	+ logsvdir.import_files(logsdir)
1763	1768
1764		- with self.timed_activity("Caching artifact"):
1765		- artifact_size = utils._get_dir_size(assembledir)
1766		- self.__artifacts.commit(self, assembledir, self.__get_cache_keys_for_commit())
1767		-
1768		- if collect is not None and collectvdir is None:
1769		- raise ElementError(
1770		- "Directory '{}' was not found inside the sandbox, "
1771		- "unable to collect artifact contents"
1772		- .format(collect))
	1769	+ artifact_size = assemblevdir.get_size()
	1770	+ self.__artifacts.commit(self, assemblevdir, self.__get_cache_keys_for_commit())
	1771	+
	1772	+ if collect is not None and collectvdir is None:
	1773	+ raise ElementError(
	1774	+ "Directory '{}' was not found inside the sandbox, "
	1775	+ "unable to collect artifact contents"
	1776	+ .format(collect))
1773	1777
1774	1778	return artifact_size
1775	1779

buildstream/storage/_casbaseddirectory.py

---

...	...	@@ -136,10 +136,10 @@ class CasBasedDirectory(Directory):
136	136	the parent).
137	137
138	138	"""
139		- self.ref = self.cas_cache.add_object(buffer=self.pb2_directory.SerializeToString())
140	139	if caller:
141	140	old_dir = self._find_pb2_entry(caller.filename)
142	141	self.cas_cache.add_object(digest=old_dir.digest, buffer=caller.pb2_directory.SerializeToString())
	142	+ self.ref = self.cas_cache.add_object(buffer=self.pb2_directory.SerializeToString())
143	143	if self.parent:
144	144	self.parent._recalculate_recursing_up(self)
145	145
...	...	@@ -277,14 +277,6 @@ class CasBasedDirectory(Directory):
277	277	directory_list))
278	278	return None
279	279
280		- def find_root(self):
281		- """ Finds the root of this directory tree by following 'parent' until there is
282		- no parent. """
283		- if self.parent:
284		- return self.parent.find_root()
285		- else:
286		- return self
287		-
288	280	def _check_replacement(self, name, path_prefix, fileListResult):
289	281	""" Checks whether 'name' exists, and if so, whether we can overwrite it.
290	282	If we can, add the name to 'overwritten_files' and delete the existing entry.
...	...	@@ -451,7 +443,7 @@ class CasBasedDirectory(Directory):
451	443	files = external_pathspec.list_relative_paths()
452	444
453	445	if isinstance(external_pathspec, FileBasedDirectory):
454		- source_directory = external_pathspec.get_underlying_directory()
	446	+ source_directory = external_pathspec._get_underlying_directory()
455	447	result = self._import_files_from_directory(source_directory, files=files)
456	448	elif isinstance(external_pathspec, str):
457	449	source_directory = external_pathspec
...	...	@@ -635,6 +627,18 @@ class CasBasedDirectory(Directory):
635	627	self._recalculate_recursing_up()
636	628	self._recalculate_recursing_down()
637	629
	630	+ def get_size(self):
	631	+ total = len(self.pb2_directory.SerializeToString())
	632	+ for i in self.index.values():
	633	+ if isinstance(i.buildstream_object, CasBasedDirectory):
	634	+ total += i.buildstream_object.get_size()
	635	+ elif isinstance(i.pb_object, remote_execution_pb2.FileNode):
	636	+ src_name = self.cas_cache.objpath(i.pb_object.digest)
	637	+ filesize = os.stat(src_name).st_size
	638	+ total += filesize
	639	+ # Symlink nodes are encoded as part of the directory serialization.
	640	+ return total
	641	+
638	642	def _get_identifier(self):
639	643	path = ""
640	644	if self.parent:
...	...	@@ -653,3 +657,15 @@ class CasBasedDirectory(Directory):
653	657	throw an exception. """
654	658	raise VirtualDirectoryError("_get_underlying_directory was called on a CAS-backed directory," +
655	659	" which has no underlying directory.")
	660	+
	661	+ # _get_digest():
	662	+ #
	663	+ # Return the Digest for this directory.
	664	+ #
	665	+ # Returns:
	666	+ # (Digest): The Digest protobuf object for the Directory protobuf
	667	+ #
	668	+ def _get_digest(self):
	669	+ if not self.ref:
	670	+ self.ref = self.cas_cache.add_object(buffer=self.pb2_directory.SerializeToString())
	671	+ return self.ref

buildstream/storage/_filebaseddirectory.py

---

...	...	@@ -30,6 +30,7 @@ See also: :ref:`sandboxing`.
30	30	import os
31	31	import time
32	32	from .directory import Directory, VirtualDirectoryError
	33	+from .. import utils
33	34	from ..utils import link_files, copy_files, list_relative_paths, _get_link_mtime, _magic_timestamp
34	35	from ..utils import _set_deterministic_user, _set_deterministic_mtime
35	36
...	...	@@ -201,6 +202,9 @@ class FileBasedDirectory(Directory):
201	202
202	203	return list_relative_paths(self.external_directory)
203	204
	205	+ def get_size(self):
	206	+ return utils._get_dir_size(self.external_directory)
	207	+
204	208	def __str__(self):
205	209	# This returns the whole path (since we don't know where the directory started)
206	210	# which exposes the sandbox directory; we will have to assume for the time being

buildstream/storage/directory.py

---

...	...	@@ -177,3 +177,9 @@ class Directory():
177	177
178	178	"""
179	179	raise NotImplementedError()
	180	+
	181	+ def get_size(self):
	182	+ """ Get an approximation of the storage space in bytes used by this directory
	183	+ and all files and subdirectories in it. Storage space varies by implementation
	184	+ and effective space used may be lower than this number due to deduplication. """
	185	+ raise NotImplementedError()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.