[Notes] [Git][BuildStream/buildstream][relative_workspaces] Implementing relative workspaces

[Phillip Smyth <gitlab mg gitlab com>]

Title: GitLab

Phillip Smyth pushed to branch relative_workspaces at BuildStream / buildstream

Commits:

• 70a82dcf
  by Phillip Smyth at 2018-08-09T17:47:03Z

  Implementing relative workspaces
  
  This fixes #191
  
  A note has been added to NEWS explaining backwards
  compatibility issues
  

6 changed files:

• NEWS
• buildstream/_frontend/widget.py
• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/element.py
• tests/frontend/workspace.py

Changes:

NEWS

---

...	...	@@ -17,6 +17,10 @@ buildstream 1.1.5
17	17	to not be scheduled and fail during artifact assembly,
18	18	and display the retry prompt during an interactive session.
19	19
	20	+ o Due to enabling the use of relative workspaces, "Legacy" workspaces
	21	+ may need to be closed and remade before the changes will affect them.
	22	+ Downgrading after using this feature may result in workspaces
	23	+ not functioning correctly
20	24
21	25	=================
22	26	buildstream 1.1.4

buildstream/_frontend/widget.py

---

...	...	@@ -418,7 +418,9 @@ class LogLine(Widget):
418	418	if "%{workspace-dirs" in format_:
419	419	workspace = element._get_workspace()
420	420	if workspace is not None:
421		- path = workspace.path.replace(os.getenv('HOME', '/root'), '~')
	421	+ path = workspace.get_absolute_path()
	422	+ if path.startswith("~/"):
	423	+ path = os.path.join(os.getenv('HOME', '/root'), path[2:])
422	424	line = p.fmt_subst(line, 'workspace-dirs', "Workspace: {}".format(path))
423	425	else:
424	426	line = p.fmt_subst(

buildstream/_stream.py

---

...	...	@@ -460,7 +460,7 @@ class Stream():
460	460	selection=PipelineSelection.REDIRECT,
461	461	track_selection=PipelineSelection.REDIRECT)
462	462	target = elements[0]
463		- workdir = os.path.abspath(directory)
	463	+ directory = os.path.abspath(directory)
464	464
465	465	if not list(target.sources()):
466	466	build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
...	...	@@ -476,7 +476,7 @@ class Stream():
476	476	workspace = workspaces.get_workspace(target._get_full_name())
477	477	if workspace and not force:
478	478	raise StreamError("Workspace '{}' is already defined at: {}"
479		- .format(target.name, workspace.path))
	479	+ .format(target.name, workspace.get_absolute_path()))
480	480
481	481	# If we're going to checkout, we need at least a fetch,
482	482	# if we were asked to track first, we're going to fetch anyway.
...	...	@@ -502,7 +502,7 @@ class Stream():
502	502	except OSError as e:
503	503	raise StreamError("Failed to create workspace directory: {}".format(e)) from e
504	504
505		- workspaces.create_workspace(target._get_full_name(), workdir)
	505	+ workspaces.create_workspace(target._get_full_name(), directory)
506	506
507	507	if not no_checkout:
508	508	with target.timed_activity("Staging sources to {}".format(directory)):
...	...	@@ -526,12 +526,12 @@ class Stream():
526	526	# Remove workspace directory if prompted
527	527	if remove_dir:
528	528	with self._context.timed_activity("Removing workspace directory {}"
529		- .format(workspace.path)):
	529	+ .format(workspace.get_absolute_path())):
530	530	try:
531		- shutil.rmtree(workspace.path)
	531	+ shutil.rmtree(workspace.get_absolute_path())
532	532	except OSError as e:
533	533	raise StreamError("Could not remove '{}': {}"
534		- .format(workspace.path, e)) from e
	534	+ .format(workspace.get_absolute_path(), e)) from e
535	535
536	536	# Delete the workspace and save the configuration
537	537	workspaces.delete_workspace(element_name)
...	...	@@ -574,28 +574,30 @@ class Stream():
574	574
575	575	for element in elements:
576	576	workspace = workspaces.get_workspace(element._get_full_name())
577		-
	577	+ workspace_path = workspace.get_absolute_path()
578	578	if soft:
579	579	workspace.prepared = False
580	580	self._message(MessageType.INFO, "Reset workspace state for {} at: {}"
581		- .format(element.name, workspace.path))
	581	+ .format(element.name, workspace_path))
582	582	continue
583	583
584	584	with element.timed_activity("Removing workspace directory {}"
585		- .format(workspace.path)):
	585	+ .format(workspace_path)):
586	586	try:
587		- shutil.rmtree(workspace.path)
	587	+ shutil.rmtree(workspace_path)
588	588	except OSError as e:
589	589	raise StreamError("Could not remove '{}': {}"
590		- .format(workspace.path, e)) from e
	590	+ .format(workspace_path, e)) from e
591	591
592	592	workspaces.delete_workspace(element._get_full_name())
593		- workspaces.create_workspace(element._get_full_name(), workspace.path)
	593	+ workspaces.create_workspace(element._get_full_name(), workspace_path)
594	594
595		- with element.timed_activity("Staging sources to {}".format(workspace.path)):
	595	+ with element.timed_activity("Staging sources to {}".format(workspace_path)):
596	596	element._open_workspace()
597	597
598		- self._message(MessageType.INFO, "Reset workspace for {} at: {}".format(element.name, workspace.path))
	598	+ self._message(MessageType.INFO,
	599	+ "Reset workspace for {} at: {}".format(element.name,
	600	+ workspace_path))
599	601
600	602	workspaces.save_config()
601	603
...	...	@@ -632,7 +634,7 @@ class Stream():
632	634	for element_name, workspace_ in self._context.get_workspaces().list():
633	635	workspace_detail = {
634	636	'element': element_name,
635		- 'directory': workspace_.path,
	637	+ 'directory': workspace_.get_absolute_path(),
636	638	}
637	639	workspaces.append(workspace_detail)
638	640

buildstream/_workspaces.py

---

...	...	@@ -26,14 +26,6 @@ from ._exceptions import LoadError, LoadErrorReason
26	26
27	27	BST_WORKSPACE_FORMAT_VERSION = 3
28	28
29		-# Hold on to a list of members which get serialized
30		-_WORKSPACE_MEMBERS = [
31		- 'prepared',
32		- 'path',
33		- 'last_successful',
34		- 'running_files'
35		-]
36		-
37	29
38	30	# Workspace()
39	31	#
...	...	@@ -56,7 +48,7 @@ class Workspace():
56	48	def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
57	49	self.prepared = prepared
58	50	self.last_successful = last_successful
59		- self.path = path
	51	+ self._path = path
60	52	self.running_files = running_files if running_files is not None else {}
61	53
62	54	self._toplevel_project = toplevel_project
...	...	@@ -64,14 +56,20 @@ class Workspace():
64	56
65	57	# to_dict()
66	58	#
67		- # Convert this object to a dict for serialization purposes
	59	+ # Convert a list of members which get serialized to a dict for serialization purposes
68	60	#
69	61	# Returns:
70	62	# (dict) A dict representation of the workspace
71	63	#
72	64	def to_dict(self):
73		- return {key: val for key, val in self.__dict__.items()
74		- if key in _WORKSPACE_MEMBERS and val is not None}
	65	+ ret = {
	66	+ 'prepared': self.prepared,
	67	+ 'path': self._path,
	68	+ 'running_files': self.running_files
	69	+ }
	70	+ if self.last_successful is not None:
	71	+ ret["last_successful"] = self.last_successful
	72	+ return ret
75	73
76	74	# from_dict():
77	75	#
...	...	@@ -103,15 +101,7 @@ class Workspace():
103	101	# True if the workspace differs from 'other', otherwise False
104	102	#
105	103	def differs(self, other):
106		-
107		- for member in _WORKSPACE_MEMBERS:
108		- member_a = getattr(self, member)
109		- member_b = getattr(other, member)
110		-
111		- if member_a != member_b:
112		- return True
113		-
114		- return False
	104	+ return self.to_dict() != other.to_dict()
115	105
116	106	# invalidate_key()
117	107	#
...	...	@@ -133,7 +123,7 @@ class Workspace():
133	123	if os.path.isdir(fullpath):
134	124	utils.copy_files(fullpath, directory)
135	125	else:
136		- destfile = os.path.join(directory, os.path.basename(self.path))
	126	+ destfile = os.path.join(directory, os.path.basename(self.get_absolute_path()))
137	127	utils.safe_copy(fullpath, destfile)
138	128
139	129	# add_running_files()
...	...	@@ -189,7 +179,7 @@ class Workspace():
189	179	filelist = utils.list_relative_paths(fullpath)
190	180	filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
191	181	else:
192		- filelist = [(self.path, fullpath)]
	182	+ filelist = [(self.get_absolute_path(), fullpath)]
193	183
194	184	self._key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist]
195	185
...	...	@@ -200,7 +190,7 @@ class Workspace():
200	190	# Returns: The absolute path of the element's workspace.
201	191	#
202	192	def get_absolute_path(self):
203		- return os.path.join(self._toplevel_project.directory, self.path)
	193	+ return os.path.join(self._toplevel_project.directory, self._path)
204	194
205	195
206	196	# Workspaces()
...	...	@@ -236,6 +226,9 @@ class Workspaces():
236	226	# path (str) - The path in which the workspace should be kept
237	227	#
238	228	def create_workspace(self, element_name, path):
	229	+ if path.startswith(self._toplevel_project.directory):
	230	+ path = os.path.relpath(path, self._toplevel_project.directory)
	231	+
239	232	self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
240	233
241	234	return self._workspaces[element_name]

buildstream/element.py

---

...	...	@@ -1403,7 +1403,8 @@ class Element(Plugin):
1403	1403	# If mount_workspaces is set and we're doing incremental builds,
1404	1404	# the workspace is already mounted into the sandbox.
1405	1405	if not (mount_workspaces and self.__can_build_incrementally()):
1406		- with self.timed_activity("Staging local files at {}".format(workspace.path)):
	1406	+ with self.timed_activity("Staging local files at {}"
	1407	+ .format(workspace.get_absolute_path())):
1407	1408	workspace.stage(temp_staging_directory)
1408	1409	else:
1409	1410	# No workspace, stage directly
...	...	@@ -1411,6 +1412,7 @@ class Element(Plugin):
1411	1412	source._stage(temp_staging_directory)
1412	1413
1413	1414	vdirectory.import_files(temp_staging_directory)
	1415	+
1414	1416	# Ensure deterministic mtime of sources at build time
1415	1417	vdirectory.set_deterministic_mtime()
1416	1418	# Ensure deterministic owners of sources at build time
...	...	@@ -1566,7 +1568,7 @@ class Element(Plugin):
1566	1568	path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep)
1567	1569	sandbox_vpath = sandbox_vroot.descend(path_components)
1568	1570	try:
1569		- sandbox_vpath.import_files(workspace.path)
	1571	+ sandbox_vpath.import_files(workspace.get_absolute_path())
1570	1572	except UtilError as e:
1571	1573	self.warn("Failed to preserve workspace state for failed build sysroot: {}"
1572	1574	.format(e))
...	...	@@ -1893,7 +1895,7 @@ class Element(Plugin):
1893	1895	source._init_workspace(temp)
1894	1896
1895	1897	# Now hardlink the files into the workspace target.
1896		- utils.link_files(temp, workspace.path)
	1898	+ utils.link_files(temp, workspace.get_absolute_path())
1897	1899
1898	1900	# _get_workspace():
1899	1901	#

tests/frontend/workspace.py

---

...	...	@@ -18,12 +18,13 @@ DATA_DIR = os.path.join(
18	18	)
19	19
20	20
21		-def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
22		- project = os.path.join(datafiles.dirname, datafiles.basename)
23		- bin_files_path = os.path.join(project, 'files', 'bin-files')
24		- element_path = os.path.join(project, 'elements')
	21	+def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None):
	22	+ if not workspace_dir:
	23	+ workspace_dir = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
	24	+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
	25	+ bin_files_path = os.path.join(project_path, 'files', 'bin-files')
	26	+ element_path = os.path.join(project_path, 'elements')
25	27	element_name = 'workspace-test-{}{}.bst'.format(kind, suffix)
26		- workspace = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
27	28
28	29	# Create our repo object of the given source type with
29	30	# the bin files, and then collect the initial ref.
...	...	@@ -45,7 +46,7 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
45	46	element_name))
46	47
47	48	# Assert that there is no reference, a track & fetch is needed
48		- state = cli.get_element_state(project, element_name)
	49	+ state = cli.get_element_state(project_path, element_name)
49	50	if track:
50	51	assert state == 'no reference'
51	52	else:
...	...	@@ -56,20 +57,20 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
56	57	args = ['workspace', 'open']
57	58	if track:
58	59	args.append('--track')
59		- args.extend([element_name, workspace])
	60	+ args.extend([element_name, workspace_dir])
	61	+ result = cli.run(project=project_path, args=args)
60	62
61		- result = cli.run(project=project, args=args)
62	63	result.assert_success()
63	64
64	65	# Assert that we are now buildable because the source is
65	66	# now cached.
66		- assert cli.get_element_state(project, element_name) == 'buildable'
	67	+ assert cli.get_element_state(project_path, element_name) == 'buildable'
67	68
68	69	# Check that the executable hello file is found in the workspace
69		- filename = os.path.join(workspace, 'usr', 'bin', 'hello')
	70	+ filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
70	71	assert os.path.exists(filename)
71	72
72		- return (element_name, project, workspace)
	73	+ return (element_name, project_path, workspace_dir)
73	74
74	75
75	76	@pytest.mark.datafiles(DATA_DIR)
...	...	@@ -190,6 +191,45 @@ def test_close(cli, tmpdir, datafiles, kind):
190	191	assert not os.path.exists(workspace)
191	192
192	193
	194	+@pytest.mark.datafiles(DATA_DIR)
	195	+def test_close_external_after_move_project(cli, tmpdir, datafiles):
	196	+ tmp_parent = os.path.dirname(str(tmpdir))
	197	+ workspace_dir = os.path.join(tmp_parent, "workspace")
	198	+ element_name, project_path, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, "", workspace_dir)
	199	+ assert os.path.exists(workspace_dir)
	200	+ tmp_dir = os.path.join(tmp_parent, 'external_project')
	201	+ shutil.move(project_path, tmp_dir)
	202	+ assert os.path.exists(tmp_dir)
	203	+
	204	+ # Close the workspace
	205	+ result = cli.run(configure=False, project=tmp_dir, args=[
	206	+ 'workspace', 'close', '--remove-dir', element_name
	207	+ ])
	208	+ result.assert_success()
	209	+
	210	+ # Assert the workspace dir has been deleted
	211	+ assert not os.path.exists(workspace_dir)
	212	+ shutil.move(tmp_dir, project_path)
	213	+
	214	+
	215	+@pytest.mark.datafiles(DATA_DIR)
	216	+def test_close_internal_after_move_project(cli, tmpdir, datafiles):
	217	+ element_name, project, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
	218	+ tmp_dir = os.path.join(os.path.dirname(str(tmpdir)), 'external_project')
	219	+ shutil.move(str(tmpdir), tmp_dir)
	220	+ assert os.path.exists(tmp_dir)
	221	+
	222	+ # Close the workspace
	223	+ result = cli.run(configure=False, project=tmp_dir, args=[
	224	+ 'workspace', 'close', '--remove-dir', element_name
	225	+ ])
	226	+ result.assert_success()
	227	+
	228	+ # Assert the workspace dir has been deleted
	229	+ workspace = os.path.join(tmp_dir, 'workspace')
	230	+ assert not os.path.exists(workspace)
	231	+
	232	+
193	233	@pytest.mark.datafiles(DATA_DIR)
194	234	def test_close_removed(cli, tmpdir, datafiles):
195	235	element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.