[Notes] [Git][BuildStream/buildstream][relative_workspaces] Implementing relative workspaces

[Phillip Smyth <gitlab mg gitlab com>]

Title: GitLab

Phillip Smyth pushed to branch relative_workspaces at BuildStream / buildstream

Commits:

• a6ea277a
  by Phillip Smyth at 2018-08-02T13:54:37Z

  Implementing relative workspaces
  

5 changed files:

• buildstream/_frontend/widget.py
• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/element.py
• tests/frontend/workspace.py

Changes:

buildstream/_frontend/widget.py

---

...	...	@@ -415,7 +415,7 @@ class LogLine(Widget):
415	415	if "%{workspace-dirs" in format_:
416	416	workspace = element._get_workspace()
417	417	if workspace is not None:
418		- path = workspace.path.replace(os.getenv('HOME', '/root'), '~')
	418	+ path = workspace.get_absolute_path().replace(os.getenv('HOME', '/root'), '~')
419	419	line = p.fmt_subst(line, 'workspace-dirs', "Workspace: {}".format(path))
420	420	else:
421	421	line = p.fmt_subst(

buildstream/_stream.py

---

...	...	@@ -478,7 +478,7 @@ class Stream():
478	478	workspace = workspaces.get_workspace(target._get_full_name())
479	479	if workspace and not force:
480	480	raise StreamError("Workspace '{}' is already defined at: {}"
481		- .format(target.name, workspace.path))
	481	+ .format(target.name, workspace.get_absolute_path()))
482	482
483	483	# If we're going to checkout, we need at least a fetch,
484	484	# if we were asked to track first, we're going to fetch anyway.
...	...	@@ -500,14 +500,14 @@ class Stream():
500	500	workspaces.save_config()
501	501	shutil.rmtree(directory)
502	502	try:
503		- os.makedirs(directory, exist_ok=True)
	503	+ os.makedirs(workdir, exist_ok=True)
504	504	except OSError as e:
505	505	raise StreamError("Failed to create workspace directory: {}".format(e)) from e
506	506
507	507	workspaces.create_workspace(target._get_full_name(), workdir)
508	508
509	509	if not no_checkout:
510		- with target.timed_activity("Staging sources to {}".format(directory)):
	510	+ with target.timed_activity("Staging sources to {}".format(workdir)):
511	511	target._open_workspace()
512	512
513	513	workspaces.save_config()
...	...	@@ -528,12 +528,12 @@ class Stream():
528	528	# Remove workspace directory if prompted
529	529	if remove_dir:
530	530	with self._context.timed_activity("Removing workspace directory {}"
531		- .format(workspace.path)):
	531	+ .format(workspace.get_absolute_path())):
532	532	try:
533		- shutil.rmtree(workspace.path)
	533	+ shutil.rmtree(workspace.get_absolute_path())
534	534	except OSError as e:
535	535	raise StreamError("Could not remove '{}': {}"
536		- .format(workspace.path, e)) from e
	536	+ .format(workspace.get_absolute_path(), e)) from e
537	537
538	538	# Delete the workspace and save the configuration
539	539	workspaces.delete_workspace(element_name)
...	...	@@ -576,28 +576,30 @@ class Stream():
576	576
577	577	for element in elements:
578	578	workspace = workspaces.get_workspace(element._get_full_name())
579		-
	579	+ workspace_path = workspace.get_absolute_path()
580	580	if soft:
581	581	workspace.prepared = False
582	582	self._message(MessageType.INFO, "Reset workspace state for {} at: {}"
583		- .format(element.name, workspace.path))
	583	+ .format(element.name, workspace_path))
584	584	continue
585	585
586	586	with element.timed_activity("Removing workspace directory {}"
587		- .format(workspace.path)):
	587	+ .format(workspace_path)):
588	588	try:
589		- shutil.rmtree(workspace.path)
	589	+ shutil.rmtree(workspace_path)
590	590	except OSError as e:
591	591	raise StreamError("Could not remove '{}': {}"
592		- .format(workspace.path, e)) from e
	592	+ .format(workspace_path, e)) from e
593	593
594	594	workspaces.delete_workspace(element._get_full_name())
595		- workspaces.create_workspace(element._get_full_name(), workspace.path)
	595	+ workspaces.create_workspace(element._get_full_name(), workspace_path)
596	596
597		- with element.timed_activity("Staging sources to {}".format(workspace.path)):
	597	+ with element.timed_activity("Staging sources to {}".format(workspace_path)):
598	598	element._open_workspace()
599	599
600		- self._message(MessageType.INFO, "Reset workspace for {} at: {}".format(element.name, workspace.path))
	600	+ self._message(MessageType.INFO,
	601	+ "Reset workspace for {} at: {}".format(element.name,
	602	+ workspace_path))
601	603
602	604	workspaces.save_config()
603	605
...	...	@@ -634,7 +636,7 @@ class Stream():
634	636	for element_name, workspace_ in self._context.get_workspaces().list():
635	637	workspace_detail = {
636	638	'element': element_name,
637		- 'directory': workspace_.path,
	639	+ 'directory': workspace_.get_absolute_path(),
638	640	}
639	641	workspaces.append(workspace_detail)
640	642

buildstream/_workspaces.py

---

...	...	@@ -26,14 +26,6 @@ from ._exceptions import LoadError, LoadErrorReason
26	26
27	27	BST_WORKSPACE_FORMAT_VERSION = 3
28	28
29		-# Hold on to a list of members which get serialized
30		-_WORKSPACE_MEMBERS = [
31		- 'prepared',
32		- 'path',
33		- 'last_successful',
34		- 'running_files'
35		-]
36		-
37	29
38	30	# Workspace()
39	31	#
...	...	@@ -56,7 +48,7 @@ class Workspace():
56	48	def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
57	49	self.prepared = prepared
58	50	self.last_successful = last_successful
59		- self.path = path
	51	+ self._path = path
60	52	self.running_files = running_files if running_files is not None else {}
61	53
62	54	self._toplevel_project = toplevel_project
...	...	@@ -64,14 +56,18 @@ class Workspace():
64	56
65	57	# to_dict()
66	58	#
67		- # Convert this object to a dict for serialization purposes
	59	+ # Convert a list of members which get serialized to a dict for serialization purposes
68	60	#
69	61	# Returns:
70	62	# (dict) A dict representation of the workspace
71	63	#
72	64	def to_dict(self):
73		- return {key: val for key, val in self.__dict__.items()
74		- if key in _WORKSPACE_MEMBERS and val is not None}
	65	+ return {
	66	+ 'prepared': self.prepared,
	67	+ 'path': self._path,
	68	+ 'last_successful': self.last_successful,
	69	+ 'running_files': self.running_files
	70	+ }
75	71
76	72	# from_dict():
77	73	#
...	...	@@ -103,15 +99,7 @@ class Workspace():
103	99	# True if the workspace differs from 'other', otherwise False
104	100	#
105	101	def differs(self, other):
106		-
107		- for member in _WORKSPACE_MEMBERS:
108		- member_a = getattr(self, member)
109		- member_b = getattr(other, member)
110		-
111		- if member_a != member_b:
112		- return True
113		-
114		- return False
	102	+ return self.to_dict() != other.to_dict()
115	103
116	104	# invalidate_key()
117	105	#
...	...	@@ -133,7 +121,7 @@ class Workspace():
133	121	if os.path.isdir(fullpath):
134	122	utils.copy_files(fullpath, directory)
135	123	else:
136		- destfile = os.path.join(directory, os.path.basename(self.path))
	124	+ destfile = os.path.join(directory, os.path.basename(self.get_absolute_path()))
137	125	utils.safe_copy(fullpath, destfile)
138	126
139	127	# add_running_files()
...	...	@@ -189,7 +177,7 @@ class Workspace():
189	177	filelist = utils.list_relative_paths(fullpath)
190	178	filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
191	179	else:
192		- filelist = [(self.path, fullpath)]
	180	+ filelist = [(self.get_absolute_path(), fullpath)]
193	181
194	182	self._key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist]
195	183
...	...	@@ -200,7 +188,7 @@ class Workspace():
200	188	# Returns: The absolute path of the element's workspace.
201	189	#
202	190	def get_absolute_path(self):
203		- return os.path.join(self._toplevel_project.directory, self.path)
	191	+ return os.path.join(self._toplevel_project.directory, self._path)
204	192
205	193
206	194	# Workspaces()
...	...	@@ -236,6 +224,9 @@ class Workspaces():
236	224	# path (str) - The path in which the workspace should be kept
237	225	#
238	226	def create_workspace(self, element_name, path):
	227	+ if path.startswith(self._toplevel_project.directory):
	228	+ path = os.path.relpath(path, self._toplevel_project.directory)
	229	+
239	230	self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
240	231
241	232	return self._workspaces[element_name]

buildstream/element.py

---

...	...	@@ -1324,7 +1324,7 @@ class Element(Plugin):
1324	1324	# If mount_workspaces is set and we're doing incremental builds,
1325	1325	# the workspace is already mounted into the sandbox.
1326	1326	if not (mount_workspaces and self.__can_build_incrementally()):
1327		- with self.timed_activity("Staging local files at {}".format(workspace.path)):
	1327	+ with self.timed_activity("Staging local files at {}".format(workspace.get_absolute_path())):
1328	1328	workspace.stage(directory)
1329	1329	else:
1330	1330	# No workspace, stage directly
...	...	@@ -1484,7 +1484,7 @@ class Element(Plugin):
1484	1484	sandbox_path = os.path.join(sandbox_root,
1485	1485	self.__staged_sources_directory.lstrip(os.sep))
1486	1486	try:
1487		- utils.copy_files(workspace.path, sandbox_path)
	1487	+ utils.copy_files(workspace.get_absolute_path(), sandbox_path)
1488	1488	except UtilError as e:
1489	1489	self.warn("Failed to preserve workspace state for failed build sysroot: {}"
1490	1490	.format(e))
...	...	@@ -1786,7 +1786,7 @@ class Element(Plugin):
1786	1786	source._init_workspace(temp)
1787	1787
1788	1788	# Now hardlink the files into the workspace target.
1789		- utils.link_files(temp, workspace.path)
	1789	+ utils.link_files(temp, workspace.get_absolute_path())
1790	1790
1791	1791	# _get_workspace():
1792	1792	#

tests/frontend/workspace.py

---

...	...	@@ -19,7 +19,12 @@ DATA_DIR = os.path.join(
19	19
20	20
21	21	def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
	22	+ if os.path.isdir(os.path.join(str(tmpdir), 'repo')):
	23	+ shutil.rmtree(os.path.join(str(tmpdir), 'repo'))
	24	+
22	25	project = os.path.join(datafiles.dirname, datafiles.basename)
	26	+
	27	+ assert os.path.exists(project)
23	28	bin_files_path = os.path.join(project, 'files', 'bin-files')
24	29	element_path = os.path.join(project, 'elements')
25	30	element_name = 'workspace-test-{}{}.bst'.format(kind, suffix)
...	...	@@ -59,6 +64,7 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
59	64	args.extend([element_name, workspace])
60	65
61	66	result = cli.run(project=project, args=args)
	67	+
62	68	result.assert_success()
63	69
64	70	# Assert that we are now buildable because the source is
...	...	@@ -190,6 +196,24 @@ def test_close(cli, tmpdir, datafiles, kind):
190	196	assert not os.path.exists(workspace)
191	197
192	198
	199	+@pytest.mark.datafiles(DATA_DIR)
	200	+def test_close_after_move_project(cli, tmpdir, datafiles):
	201	+ element_name, project, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
	202	+ tmp_dir = os.path.join(os.path.dirname(str(tmpdir)), 'external_workspace')
	203	+ shutil.move(str(tmpdir), tmp_dir)
	204	+ assert os.path.exists(tmp_dir)
	205	+
	206	+ # Close the workspace
	207	+ result = cli.run(configure=False, project=tmp_dir, args=[
	208	+ 'workspace', 'close', '--remove-dir', element_name
	209	+ ])
	210	+ result.assert_success()
	211	+
	212	+ # Assert the workspace dir has been deleted
	213	+ workspace = os.path.join(tmp_dir, 'workspace')
	214	+ assert not os.path.exists(workspace)
	215	+
	216	+
193	217	@pytest.mark.datafiles(DATA_DIR)
194	218	def test_close_removed(cli, tmpdir, datafiles):
195	219	element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
...	...	@@ -388,6 +412,7 @@ def test_build(cli, tmpdir, datafiles, kind, strict):
388	412	# Build modified workspace
389	413	assert cli.get_element_state(project, element_name) == 'buildable'
390	414	assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
	415	+
391	416	result = cli.run(project=project, args=['build', element_name])
392	417	result.assert_success()
393	418	assert cli.get_element_state(project, element_name) == 'cached'
...	...	@@ -561,6 +586,7 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
561	586	"workspaces": {
562	587	"alpha.bst": {
563	588	"prepared": False,
	589	+ "last_successful": None,
564	590	"path": "/workspaces/bravo",
565	591	"running_files": {}
566	592	}
...	...	@@ -576,6 +602,7 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
576	602	"workspaces": {
577	603	"alpha.bst": {
578	604	"prepared": False,
	605	+ "last_successful": None,
579	606	"path": "/workspaces/bravo",
580	607	"running_files": {}
581	608	}
...	...	@@ -594,6 +621,7 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
594	621	"workspaces": {
595	622	"alpha.bst": {
596	623	"prepared": False,
	624	+ "last_successful": None,
597	625	"path": "/workspaces/bravo",
598	626	"running_files": {}
599	627	}
...	...	@@ -639,6 +667,7 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
639	667	"workspaces": {
640	668	"alpha.bst": {
641	669	"prepared": True,
	670	+ "last_successful": None,
642	671	"path": "/workspaces/bravo",
643	672	"running_files": {}
644	673	}

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.