[Notes] [Git][BuildStream/buildstream][jmac/virtual_directories] 8 commits: element.py: Conversion to use virtual directories

[Jim MacArthur <gitlab mg gitlab com>]

Title: GitLab

Jim MacArthur pushed to branch jmac/virtual_directories at BuildStream / buildstream

Commits:

• 785642b6
  by Jim MacArthur at 2018-08-01T11:36:47Z

  element.py: Conversion to use virtual directories
  

• ed7c610d
  by Jim MacArthur at 2018-08-01T11:36:47Z

  plugins/elements/compose.py: Convert to virtual directories
  

• b030d0a4
  by Jim MacArthur at 2018-08-01T11:36:47Z

  plugins/elements/import.py: Convert to virtual directories
  

• 6bdfe94a
  by Jim MacArthur at 2018-08-01T11:36:47Z

  sandbox: Remove instances of get_directory
  
  sandbox/_mount.py, sandbox/_sandboxbwrap.py:
    Remove instances of get_directory
  

• 87a77e9f
  by Jim MacArthur at 2018-08-01T11:36:47Z

  plugins/elements/stack.py: Convert to virtual directories
  

• 206f6bb4
  by Jim MacArthur at 2018-08-01T11:36:47Z

  scriptelement.py: Convert to virtual directories
  

• 1de89f92
  by Jim MacArthur at 2018-08-01T11:36:47Z

  compose.py: Remove unused getmtime()
  

• b0ff6db1
  by Jim MacArthur at 2018-08-01T11:36:47Z

  sandbox.py: Use _root in _has_command
  
  This will need more attention when we bring in another virtual
  directory backend, however, we've said it is acceptable for the
  sandbox itself to access the underlying directory, and this is
  the best fix in the meantime.
  

8 changed files:

• buildstream/element.py
• buildstream/plugins/elements/compose.py
• buildstream/plugins/elements/import.py
• buildstream/plugins/elements/stack.py
• buildstream/sandbox/_mount.py
• buildstream/sandbox/_sandboxbwrap.py
• buildstream/sandbox/sandbox.py
• buildstream/scriptelement.py

Changes:

buildstream/element.py

---

...	...	@@ -80,7 +80,6 @@ from collections import Mapping, OrderedDict
80	80	from contextlib import contextmanager
81	81	from enum import Enum
82	82	import tempfile
83		-import time
84	83	import shutil
85	84
86	85	from . import _yaml
...	...	@@ -97,6 +96,10 @@ from . import _site
97	96	from ._platform import Platform
98	97	from .sandbox._config import SandboxConfig
99	98
	99	+from .storage.directory import Directory
	100	+from .storage._filebaseddirectory import FileBasedDirectory, VirtualDirectoryError
	101	+
	102	+
100	103	# _KeyStrength():
101	104	#
102	105	# Strength of cache key
...	...	@@ -198,7 +201,7 @@ class Element(Plugin):
198	201	"""Whether to raise exceptions if an element uses Sandbox.get_directory
199	202	instead of Sandbox.get_virtual_directory.
200	203
201		- *Since: 1.2*
	204	+ *Since: 1.4*
202	205	"""
203	206
204	207	def __init__(self, context, project, artifacts, meta, plugin_conf):
...	...	@@ -633,10 +636,10 @@ class Element(Plugin):
633	636
634	637	# Hard link it into the staging area
635	638	#
636		- basedir = sandbox.get_directory()
637		- stagedir = basedir \
	639	+ vbasedir = sandbox.get_virtual_directory()
	640	+ vstagedir = vbasedir \
638	641	if path is None \
639		- else os.path.join(basedir, path.lstrip(os.sep))
	642	+ else vbasedir.descend(path.lstrip(os.sep).split(os.sep))
640	643
641	644	files = list(self.__compute_splits(include, exclude, orphans))
642	645
...	...	@@ -648,15 +651,8 @@ class Element(Plugin):
648	651	link_files = files
649	652	copy_files = []
650	653
651		- link_result = utils.link_files(artifact, stagedir, files=link_files,
652		- report_written=True)
653		- copy_result = utils.copy_files(artifact, stagedir, files=copy_files,
654		- report_written=True)
655		-
656		- cur_time = time.time()
657		-
658		- for f in copy_result.files_written:
659		- os.utime(os.path.join(stagedir, f), times=(cur_time, cur_time))
	654	+ link_result = vstagedir.import_files(artifact, files=link_files, report_written=True, can_link=True)
	655	+ copy_result = vstagedir.import_files(artifact, files=copy_files, report_written=True, update_utimes=True)
660	656
661	657	return link_result.combine(copy_result)
662	658
...	...	@@ -1365,40 +1361,45 @@ class Element(Plugin):
1365	1361	sandbox._set_mount_source(directory, workspace.get_absolute_path())
1366	1362
1367	1363	# Stage all sources that need to be copied
1368		- sandbox_root = sandbox.get_directory()
1369		- host_directory = os.path.join(sandbox_root, directory.lstrip(os.sep))
1370		- self._stage_sources_at(host_directory, mount_workspaces=mount_workspaces)
	1364	+ sandbox_vroot = sandbox.get_virtual_directory()
	1365	+ host_vdirectory = sandbox_vroot.descend(directory.lstrip(os.sep).split(os.sep), create=True)
	1366	+ self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces)
1371	1367
1372	1368	# _stage_sources_at():
1373	1369	#
1374	1370	# Stage this element's sources to a directory
1375	1371	#
1376	1372	# Args:
1377		- # directory (str): An absolute path to stage the sources at
	1373	+ # vdirectory (:class:`.storage.Directory`): A virtual directory object to stage sources into.
1378	1374	# mount_workspaces (bool): mount workspaces if True, copy otherwise
1379	1375	#
1380		- def _stage_sources_at(self, directory, mount_workspaces=True):
	1376	+ def _stage_sources_at(self, vdirectory, mount_workspaces=True):
1381	1377	with self.timed_activity("Staging sources", silent_nested=True):
1382	1378
1383		- if os.path.isdir(directory) and os.listdir(directory):
1384		- raise ElementError("Staging directory '{}' is not empty".format(directory))
1385		-
1386		- workspace = self._get_workspace()
1387		- if workspace:
1388		- # If mount_workspaces is set and we're doing incremental builds,
1389		- # the workspace is already mounted into the sandbox.
1390		- if not (mount_workspaces and self.__can_build_incrementally()):
1391		- with self.timed_activity("Staging local files at {}".format(workspace.path)):
1392		- workspace.stage(directory)
1393		- else:
1394		- # No workspace, stage directly
1395		- for source in self.sources():
1396		- source._stage(directory)
1397		-
	1379	+ if not isinstance(vdirectory, Directory):
	1380	+ vdirectory = FileBasedDirectory(vdirectory)
	1381	+ if not vdirectory.is_empty():
	1382	+ raise ElementError("Staging directory '{}' is not empty".format(vdirectory))
	1383	+
	1384	+ with tempfile.TemporaryDirectory() as temp_staging_directory:
	1385	+
	1386	+ workspace = self._get_workspace()
	1387	+ if workspace:
	1388	+ # If mount_workspaces is set and we're doing incremental builds,
	1389	+ # the workspace is already mounted into the sandbox.
	1390	+ if not (mount_workspaces and self.__can_build_incrementally()):
	1391	+ with self.timed_activity("Staging local files at {}".format(workspace.path)):
	1392	+ workspace.stage(temp_staging_directory)
	1393	+ else:
	1394	+ # No workspace, stage directly
	1395	+ for source in self.sources():
	1396	+ source._stage(temp_staging_directory)
	1397	+
	1398	+ vdirectory.import_files(temp_staging_directory)
1398	1399	# Ensure deterministic mtime of sources at build time
1399		- utils._set_deterministic_mtime(directory)
	1400	+ vdirectory.set_deterministic_mtime()
1400	1401	# Ensure deterministic owners of sources at build time
1401		- utils._set_deterministic_user(directory)
	1402	+ vdirectory.set_deterministic_user()
1402	1403
1403	1404	# _set_required():
1404	1405	#
...	...	@@ -1514,7 +1515,7 @@ class Element(Plugin):
1514	1515	with _signals.terminator(cleanup_rootdir), \
1515	1516	self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox: # nopep8
1516	1517
1517		- sandbox_root = sandbox.get_directory()
	1518	+ sandbox_vroot = sandbox.get_virtual_directory()
1518	1519
1519	1520	# By default, the dynamic public data is the same as the static public data.
1520	1521	# The plugin's assemble() method may modify this, though.
...	...	@@ -1546,11 +1547,11 @@ class Element(Plugin):
1546	1547	#
1547	1548	workspace = self._get_workspace()
1548	1549	if workspace and self.__staged_sources_directory:
1549		- sandbox_root = sandbox.get_directory()
1550		- sandbox_path = os.path.join(sandbox_root,
1551		- self.__staged_sources_directory.lstrip(os.sep))
	1550	+ sandbox_vroot = sandbox.get_virtual_directory()
	1551	+ path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep)
	1552	+ sandbox_vpath = sandbox_vroot.descend(path_components)
1552	1553	try:
1553		- utils.copy_files(workspace.path, sandbox_path)
	1554	+ sandbox_vpath.import_files(workspace.path)
1554	1555	except UtilError as e:
1555	1556	self.warn("Failed to preserve workspace state for failed build sysroot: {}"
1556	1557	.format(e))
...	...	@@ -1562,7 +1563,11 @@ class Element(Plugin):
1562	1563	raise
1563	1564	finally:
1564	1565	if collect is not None:
1565		- collectdir = os.path.join(sandbox_root, collect.lstrip(os.sep))
	1566	+ try:
	1567	+ collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
	1568	+ except VirtualDirectoryError:
	1569	+ # No collect directory existed
	1570	+ collectvdir = None
1566	1571
1567	1572	# Create artifact directory structure
1568	1573	assembledir = os.path.join(rootdir, 'artifact')
...	...	@@ -1571,20 +1576,26 @@ class Element(Plugin):
1571	1576	metadir = os.path.join(assembledir, 'meta')
1572	1577	buildtreedir = os.path.join(assembledir, 'buildtree')
1573	1578	os.mkdir(assembledir)
1574		- if collect is not None and os.path.exists(collectdir):
	1579	+ if collect is not None and collectvdir is not None:
1575	1580	os.mkdir(filesdir)
1576	1581	os.mkdir(logsdir)
1577	1582	os.mkdir(metadir)
1578	1583	os.mkdir(buildtreedir)
1579	1584
1580	1585	# Hard link files from collect dir to files directory
1581		- if collect is not None and os.path.exists(collectdir):
1582		- utils.link_files(collectdir, filesdir)
1583		-
1584		- sandbox_build_dir = os.path.join(sandbox_root, self.get_variable('build-root').lstrip(os.sep))
1585		- # Hard link files from build-root dir to buildtreedir directory
1586		- if os.path.isdir(sandbox_build_dir):
1587		- utils.link_files(sandbox_build_dir, buildtreedir)
	1586	+ if collect is not None and collectvdir is not None:
	1587	+ collectvdir.export_files(filesdir, can_link=True)
	1588	+
	1589	+ try:
	1590	+ sandbox_build_dir = sandbox_vroot.descend(
	1591	+ self.get_variable('build-root').lstrip(os.sep).split(os.sep))
	1592	+ # Hard link files from build-root dir to buildtreedir directory
	1593	+ sandbox_build_dir.export_files(buildtreedir)
	1594	+ except VirtualDirectoryError:
	1595	+ # Directory could not be found. Pre-virtual
	1596	+ # directory behaviour was to continue silently
	1597	+ # if the directory could not be found.
	1598	+ pass
1588	1599
1589	1600	# Copy build log
1590	1601	log_filename = context.get_log_filename()
...	...	@@ -1632,7 +1643,7 @@ class Element(Plugin):
1632	1643	self.__artifact_size = utils._get_dir_size(assembledir)
1633	1644	self.__artifacts.commit(self, assembledir, self.__get_cache_keys_for_commit())
1634	1645
1635		- if collect is not None and not os.path.exists(collectdir):
	1646	+ if collect is not None and collectvdir is None:
1636	1647	raise ElementError(
1637	1648	"Directory '{}' was not found inside the sandbox, "
1638	1649	"unable to collect artifact contents"

buildstream/plugins/elements/compose.py

---

...	...	@@ -34,7 +34,6 @@ The default configuration and possible options are as such:
34	34	"""
35	35
36	36	import os
37		-from buildstream import utils
38	37	from buildstream import Element, Scope
39	38
40	39
...	...	@@ -56,6 +55,9 @@ class ComposeElement(Element):
56	55	# added, to reduce the potential for confusion
57	56	BST_FORBID_SOURCES = True
58	57
	58	+ # This plugin has been modified to avoid the use of Sandbox.get_directory
	59	+ BST_VIRTUAL_DIRECTORY = True
	60	+
59	61	def configure(self, node):
60	62	self.node_validate(node, [
61	63	'integrate', 'include', 'exclude', 'include-orphans'
...	...	@@ -104,7 +106,8 @@ class ComposeElement(Element):
104	106	orphans=self.include_orphans)
105	107	manifest.update(files)
106	108
107		- basedir = sandbox.get_directory()
	109	+ # Make a snapshot of all the files.
	110	+ vbasedir = sandbox.get_virtual_directory()
108	111	modified_files = set()
109	112	removed_files = set()
110	113	added_files = set()
...	...	@@ -116,38 +119,24 @@ class ComposeElement(Element):
116	119	if require_split:
117	120
118	121	# Make a snapshot of all the files before integration-commands are run.
119		- snapshot = {
120		- f: getmtime(os.path.join(basedir, f))
121		- for f in utils.list_relative_paths(basedir)
122		- }
	122	+ snapshot = set(vbasedir.list_relative_paths())
	123	+ vbasedir.mark_unmodified()
123	124
124	125	for dep in self.dependencies(Scope.BUILD):
125	126	dep.integrate(sandbox)
126	127
127	128	if require_split:
128		-
129	129	# Calculate added, modified and removed files
130		- basedir_contents = set(utils.list_relative_paths(basedir))
	130	+ post_integration_snapshot = vbasedir.list_relative_paths()
	131	+ modified_files = set(vbasedir.list_modified_paths())
	132	+ basedir_contents = set(post_integration_snapshot)
131	133	for path in manifest:
132		- if path in basedir_contents:
133		- if path in snapshot:
134		- preintegration_mtime = snapshot[path]
135		- if preintegration_mtime != getmtime(os.path.join(basedir, path)):
136		- modified_files.add(path)
137		- else:
138		- # If the path appears in the manifest but not the initial snapshot,
139		- # it may be a file staged inside a directory symlink. In this case
140		- # the path we got from the manifest won't show up in the snapshot
141		- # because utils.list_relative_paths() doesn't recurse into symlink
142		- # directories.
143		- pass
144		- elif path in snapshot:
	134	+ if path in snapshot and path not in basedir_contents:
145	135	removed_files.add(path)
146	136
147	137	for path in basedir_contents:
148	138	if path not in snapshot:
149	139	added_files.add(path)
150		-
151	140	self.info("Integration modified {}, added {} and removed {} files"
152	141	.format(len(modified_files), len(added_files), len(removed_files)))
153	142
...	...	@@ -166,8 +155,7 @@ class ComposeElement(Element):
166	155	# instead of into a subdir. The element assemble() method should
167	156	# support this in some way.
168	157	#
169		- installdir = os.path.join(basedir, 'buildstream', 'install')
170		- os.makedirs(installdir, exist_ok=True)
	158	+ installdir = vbasedir.descend(['buildstream', 'install'], create=True)
171	159
172	160	# We already saved the manifest for created files in the integration phase,
173	161	# now collect the rest of the manifest.
...	...	@@ -191,19 +179,12 @@ class ComposeElement(Element):
191	179
192	180	with self.timed_activity("Creating composition", detail=detail, silent_nested=True):
193	181	self.info("Composing {} files".format(len(manifest)))
194		- utils.link_files(basedir, installdir, files=manifest)
	182	+ installdir.import_files(vbasedir, files=manifest, can_link=True)
195	183
196	184	# And we're done
197	185	return os.path.join(os.sep, 'buildstream', 'install')
198	186
199	187
200		-# Like os.path.getmtime(), but doesnt explode on symlinks
201		-#
202		-def getmtime(path):
203		- stat = os.lstat(path)
204		- return stat.st_mtime
205		-
206		-
207	188	# Plugin entry point
208	189	def setup():
209	190	return ComposeElement

buildstream/plugins/elements/import.py

---

...	...	@@ -31,7 +31,6 @@ The empty configuration is as such:
31	31	"""
32	32
33	33	import os
34		-import shutil
35	34	from buildstream import Element, BuildElement, ElementError
36	35
37	36
...	...	@@ -39,6 +38,9 @@ from buildstream import Element, BuildElement, ElementError
39	38	class ImportElement(BuildElement):
40	39	# pylint: disable=attribute-defined-outside-init
41	40
	41	+ # This plugin has been modified to avoid the use of Sandbox.get_directory
	42	+ BST_VIRTUAL_DIRECTORY = True
	43	+
42	44	def configure(self, node):
43	45	self.source = self.node_subst_member(node, 'source')
44	46	self.target = self.node_subst_member(node, 'target')
...	...	@@ -68,27 +70,22 @@ class ImportElement(BuildElement):
68	70	# Do not mount workspaces as the files are copied from outside the sandbox
69	71	self._stage_sources_in_sandbox(sandbox, 'input', mount_workspaces=False)
70	72
71		- rootdir = sandbox.get_directory()
72		- inputdir = os.path.join(rootdir, 'input')
73		- outputdir = os.path.join(rootdir, 'output')
	73	+ rootdir = sandbox.get_virtual_directory()
	74	+ inputdir = rootdir.descend(['input'])
	75	+ outputdir = rootdir.descend(['output'], create=True)
74	76
75	77	# The directory to grab
76		- inputdir = os.path.join(inputdir, self.source.lstrip(os.sep))
77		- inputdir = inputdir.rstrip(os.sep)
	78	+ inputdir = inputdir.descend(self.source.strip(os.sep).split(os.sep))
78	79
79	80	# The output target directory
80		- outputdir = os.path.join(outputdir, self.target.lstrip(os.sep))
81		- outputdir = outputdir.rstrip(os.sep)
82		-
83		- # Ensure target directory parent
84		- os.makedirs(os.path.dirname(outputdir), exist_ok=True)
	81	+ outputdir = outputdir.descend(self.target.strip(os.sep).split(os.sep), create=True)
85	82
86		- if not os.path.exists(inputdir):
	83	+ if inputdir.is_empty():
87	84	raise ElementError("{}: No files were found inside directory '{}'"
88	85	.format(self, self.source))
89	86
90	87	# Move it over
91		- shutil.move(inputdir, outputdir)
	88	+ outputdir.import_files(inputdir)
92	89
93	90	# And we're done
94	91	return '/output'

buildstream/plugins/elements/stack.py

---

...	...	@@ -24,13 +24,15 @@ Stack elements are simply a symbolic element used for representing
24	24	a logical group of elements.
25	25	"""
26	26
27		-import os
28	27	from buildstream import Element
29	28
30	29
31	30	# Element implementation for the 'stack' kind.
32	31	class StackElement(Element):
33	32
	33	+ # This plugin has been modified to avoid the use of Sandbox.get_directory
	34	+ BST_VIRTUAL_DIRECTORY = True
	35	+
34	36	def configure(self, node):
35	37	pass
36	38
...	...	@@ -52,7 +54,7 @@ class StackElement(Element):
52	54
53	55	# Just create a dummy empty artifact, its existence is a statement
54	56	# that all this stack's dependencies are built.
55		- rootdir = sandbox.get_directory()
	57	+ vrootdir = sandbox.get_virtual_directory()
56	58
57	59	# XXX FIXME: This is currently needed because the artifact
58	60	# cache wont let us commit an empty artifact.
...	...	@@ -61,10 +63,7 @@ class StackElement(Element):
61	63	# the actual artifact data in a subdirectory, then we
62	64	# will be able to store some additional state in the
63	65	# artifact cache, and we can also remove this hack.
64		- outputdir = os.path.join(rootdir, 'output', 'bst')
65		-
66		- # Ensure target directory parent
67		- os.makedirs(os.path.dirname(outputdir), exist_ok=True)
	66	+ vrootdir.descend(['output', 'bst'], create=True)
68	67
69	68	# And we're done
70	69	return '/output'

buildstream/sandbox/_mount.py

---

...	...	@@ -32,7 +32,8 @@ from .._fuse import SafeHardlinks
32	32	class Mount():
33	33	def __init__(self, sandbox, mount_point, safe_hardlinks):
34	34	scratch_directory = sandbox._get_scratch_directory()
35		- root_directory = sandbox.get_directory()
	35	+ # Getting external_directory here is acceptable as we're part of the sandbox code.
	36	+ root_directory = sandbox.get_virtual_directory().external_directory
36	37
37	38	self.mount_point = mount_point
38	39	self.safe_hardlinks = safe_hardlinks

buildstream/sandbox/_sandboxbwrap.py

---

...	...	@@ -56,7 +56,9 @@ class SandboxBwrap(Sandbox):
56	56
57	57	def run(self, command, flags, *, cwd=None, env=None):
58	58	stdout, stderr = self._get_output()
59		- root_directory = self.get_directory()
	59	+
	60	+ # Allowable access to underlying storage as we're part of the sandbox
	61	+ root_directory = self.get_virtual_directory().external_directory
60	62
61	63	# Fallback to the sandbox default settings for
62	64	# the cwd and env.

buildstream/sandbox/sandbox.py

---

...	...	@@ -316,11 +316,11 @@ class Sandbox():
316	316	def _has_command(self, command, env=None):
317	317	if os.path.isabs(command):
318	318	return os.path.exists(os.path.join(
319		- self.get_directory(), command.lstrip(os.sep)))
	319	+ self._root, command.lstrip(os.sep)))
320	320
321	321	for path in env.get('PATH').split(':'):
322	322	if os.path.exists(os.path.join(
323		- self.get_directory(), path.lstrip(os.sep), command)):
	323	+ self._root, path.lstrip(os.sep), command)):
324	324	return True
325	325
326	326	return False

buildstream/scriptelement.py

---

...	...	@@ -243,9 +243,8 @@ class ScriptElement(Element):
243	243	with self.timed_activity("Staging {} at {}"
244	244	.format(element.name, item['destination']),
245	245	silent_nested=True):
246		- real_dstdir = os.path.join(sandbox.get_directory(),
247		- item['destination'].lstrip(os.sep))
248		- os.makedirs(os.path.dirname(real_dstdir), exist_ok=True)
	246	+ virtual_dstdir = sandbox.get_virtual_directory()
	247	+ virtual_dstdir.descend(item['destination'].lstrip(os.sep).split(os.sep), create=True)
249	248	element.stage_dependency_artifacts(sandbox, Scope.RUN, path=item['destination'])
250	249
251	250	for item in self.__layout:
...	...	@@ -263,8 +262,8 @@ class ScriptElement(Element):
263	262	for dep in element.dependencies(Scope.RUN):
264	263	dep.integrate(sandbox)
265	264
266		- os.makedirs(os.path.join(sandbox.get_directory(), self.__install_root.lstrip(os.sep)),
267		- exist_ok=True)
	265	+ install_root_path_components = self.__install_root.lstrip(os.sep).split(os.sep)
	266	+ sandbox.get_virtual_directory().descend(install_root_path_components, create=True)
268	267
269	268	def assemble(self, sandbox):
270	269

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.