Phillip Smyth pushed to branch relative_workspaces at BuildStream / buildstream
Commits:
-
a10814eb
by Phillip Smyth at 2018-08-03T14:01:50Z
5 changed files:
- buildstream/_frontend/widget.py
- buildstream/_stream.py
- buildstream/_workspaces.py
- buildstream/element.py
- tests/frontend/workspace.py
Changes:
... | ... | @@ -418,7 +418,9 @@ class LogLine(Widget): |
418 | 418 |
if "%{workspace-dirs" in format_:
|
419 | 419 |
workspace = element._get_workspace()
|
420 | 420 |
if workspace is not None:
|
421 |
- path = workspace.path.replace(os.getenv('HOME', '/root'), '~')
|
|
421 |
+ path = workspace.get_absolute_path()
|
|
422 |
+ if path.startswith("~/"):
|
|
423 |
+ path = os.path.join(os.getenv('HOME', '/root'), path[2:])
|
|
422 | 424 |
line = p.fmt_subst(line, 'workspace-dirs', "Workspace: {}".format(path))
|
423 | 425 |
else:
|
424 | 426 |
line = p.fmt_subst(
|
... | ... | @@ -461,7 +461,7 @@ class Stream(): |
461 | 461 |
selection=PipelineSelection.REDIRECT,
|
462 | 462 |
track_selection=PipelineSelection.REDIRECT)
|
463 | 463 |
target = elements[0]
|
464 |
- workdir = os.path.abspath(directory)
|
|
464 |
+ directory = os.path.abspath(directory)
|
|
465 | 465 |
|
466 | 466 |
if not list(target.sources()):
|
467 | 467 |
build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
|
... | ... | @@ -477,7 +477,7 @@ class Stream(): |
477 | 477 |
workspace = workspaces.get_workspace(target._get_full_name())
|
478 | 478 |
if workspace and not force:
|
479 | 479 |
raise StreamError("Workspace '{}' is already defined at: {}"
|
480 |
- .format(target.name, workspace.path))
|
|
480 |
+ .format(target.name, workspace.get_absolute_path()))
|
|
481 | 481 |
|
482 | 482 |
# If we're going to checkout, we need at least a fetch,
|
483 | 483 |
# if we were asked to track first, we're going to fetch anyway.
|
... | ... | @@ -503,7 +503,7 @@ class Stream(): |
503 | 503 |
except OSError as e:
|
504 | 504 |
raise StreamError("Failed to create workspace directory: {}".format(e)) from e
|
505 | 505 |
|
506 |
- workspaces.create_workspace(target._get_full_name(), workdir)
|
|
506 |
+ workspaces.create_workspace(target._get_full_name(), directory)
|
|
507 | 507 |
|
508 | 508 |
if not no_checkout:
|
509 | 509 |
with target.timed_activity("Staging sources to {}".format(directory)):
|
... | ... | @@ -527,12 +527,12 @@ class Stream(): |
527 | 527 |
# Remove workspace directory if prompted
|
528 | 528 |
if remove_dir:
|
529 | 529 |
with self._context.timed_activity("Removing workspace directory {}"
|
530 |
- .format(workspace.path)):
|
|
530 |
+ .format(workspace.get_absolute_path())):
|
|
531 | 531 |
try:
|
532 |
- shutil.rmtree(workspace.path)
|
|
532 |
+ shutil.rmtree(workspace.get_absolute_path())
|
|
533 | 533 |
except OSError as e:
|
534 | 534 |
raise StreamError("Could not remove '{}': {}"
|
535 |
- .format(workspace.path, e)) from e
|
|
535 |
+ .format(workspace.get_absolute_path(), e)) from e
|
|
536 | 536 |
|
537 | 537 |
# Delete the workspace and save the configuration
|
538 | 538 |
workspaces.delete_workspace(element_name)
|
... | ... | @@ -575,28 +575,30 @@ class Stream(): |
575 | 575 |
|
576 | 576 |
for element in elements:
|
577 | 577 |
workspace = workspaces.get_workspace(element._get_full_name())
|
578 |
- |
|
578 |
+ workspace_path = workspace.get_absolute_path()
|
|
579 | 579 |
if soft:
|
580 | 580 |
workspace.prepared = False
|
581 | 581 |
self._message(MessageType.INFO, "Reset workspace state for {} at: {}"
|
582 |
- .format(element.name, workspace.path))
|
|
582 |
+ .format(element.name, workspace_path))
|
|
583 | 583 |
continue
|
584 | 584 |
|
585 | 585 |
with element.timed_activity("Removing workspace directory {}"
|
586 |
- .format(workspace.path)):
|
|
586 |
+ .format(workspace_path)):
|
|
587 | 587 |
try:
|
588 |
- shutil.rmtree(workspace.path)
|
|
588 |
+ shutil.rmtree(workspace_path)
|
|
589 | 589 |
except OSError as e:
|
590 | 590 |
raise StreamError("Could not remove '{}': {}"
|
591 |
- .format(workspace.path, e)) from e
|
|
591 |
+ .format(workspace_path, e)) from e
|
|
592 | 592 |
|
593 | 593 |
workspaces.delete_workspace(element._get_full_name())
|
594 |
- workspaces.create_workspace(element._get_full_name(), workspace.path)
|
|
594 |
+ workspaces.create_workspace(element._get_full_name(), workspace_path)
|
|
595 | 595 |
|
596 |
- with element.timed_activity("Staging sources to {}".format(workspace.path)):
|
|
596 |
+ with element.timed_activity("Staging sources to {}".format(workspace_path)):
|
|
597 | 597 |
element._open_workspace()
|
598 | 598 |
|
599 |
- self._message(MessageType.INFO, "Reset workspace for {} at: {}".format(element.name, workspace.path))
|
|
599 |
+ self._message(MessageType.INFO,
|
|
600 |
+ "Reset workspace for {} at: {}".format(element.name,
|
|
601 |
+ workspace_path))
|
|
600 | 602 |
|
601 | 603 |
workspaces.save_config()
|
602 | 604 |
|
... | ... | @@ -633,7 +635,7 @@ class Stream(): |
633 | 635 |
for element_name, workspace_ in self._context.get_workspaces().list():
|
634 | 636 |
workspace_detail = {
|
635 | 637 |
'element': element_name,
|
636 |
- 'directory': workspace_.path,
|
|
638 |
+ 'directory': workspace_.get_absolute_path(),
|
|
637 | 639 |
}
|
638 | 640 |
workspaces.append(workspace_detail)
|
639 | 641 |
|
... | ... | @@ -26,14 +26,6 @@ from ._exceptions import LoadError, LoadErrorReason |
26 | 26 |
|
27 | 27 |
BST_WORKSPACE_FORMAT_VERSION = 3
|
28 | 28 |
|
29 |
-# Hold on to a list of members which get serialized
|
|
30 |
-_WORKSPACE_MEMBERS = [
|
|
31 |
- 'prepared',
|
|
32 |
- 'path',
|
|
33 |
- 'last_successful',
|
|
34 |
- 'running_files'
|
|
35 |
-]
|
|
36 |
- |
|
37 | 29 |
|
38 | 30 |
# Workspace()
|
39 | 31 |
#
|
... | ... | @@ -56,7 +48,7 @@ class Workspace(): |
56 | 48 |
def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
|
57 | 49 |
self.prepared = prepared
|
58 | 50 |
self.last_successful = last_successful
|
59 |
- self.path = path
|
|
51 |
+ self._path = path
|
|
60 | 52 |
self.running_files = running_files if running_files is not None else {}
|
61 | 53 |
|
62 | 54 |
self._toplevel_project = toplevel_project
|
... | ... | @@ -64,14 +56,21 @@ class Workspace(): |
64 | 56 |
|
65 | 57 |
# to_dict()
|
66 | 58 |
#
|
67 |
- # Convert this object to a dict for serialization purposes
|
|
59 |
+ # Convert a list of members which get serialized to a dict for serialization purposes
|
|
68 | 60 |
#
|
69 | 61 |
# Returns:
|
70 | 62 |
# (dict) A dict representation of the workspace
|
71 | 63 |
#
|
72 | 64 |
def to_dict(self):
|
73 |
- return {key: val for key, val in self.__dict__.items()
|
|
74 |
- if key in _WORKSPACE_MEMBERS and val is not None}
|
|
65 |
+ ret = {
|
|
66 |
+ 'prepared': self.prepared,
|
|
67 |
+ 'path': self._path,
|
|
68 |
+ 'running_files': self.running_files
|
|
69 |
+ }
|
|
70 |
+ if self.last_successful is not None:
|
|
71 |
+ ret["last_successful"] = self.last_successful
|
|
72 |
+ return ret
|
|
73 |
+ |
|
75 | 74 |
|
76 | 75 |
# from_dict():
|
77 | 76 |
#
|
... | ... | @@ -103,15 +102,7 @@ class Workspace(): |
103 | 102 |
# True if the workspace differs from 'other', otherwise False
|
104 | 103 |
#
|
105 | 104 |
def differs(self, other):
|
106 |
- |
|
107 |
- for member in _WORKSPACE_MEMBERS:
|
|
108 |
- member_a = getattr(self, member)
|
|
109 |
- member_b = getattr(other, member)
|
|
110 |
- |
|
111 |
- if member_a != member_b:
|
|
112 |
- return True
|
|
113 |
- |
|
114 |
- return False
|
|
105 |
+ return self.to_dict() != other.to_dict()
|
|
115 | 106 |
|
116 | 107 |
# invalidate_key()
|
117 | 108 |
#
|
... | ... | @@ -133,7 +124,7 @@ class Workspace(): |
133 | 124 |
if os.path.isdir(fullpath):
|
134 | 125 |
utils.copy_files(fullpath, directory)
|
135 | 126 |
else:
|
136 |
- destfile = os.path.join(directory, os.path.basename(self.path))
|
|
127 |
+ destfile = os.path.join(directory, os.path.basename(self.get_absolute_path()))
|
|
137 | 128 |
utils.safe_copy(fullpath, destfile)
|
138 | 129 |
|
139 | 130 |
# add_running_files()
|
... | ... | @@ -189,7 +180,7 @@ class Workspace(): |
189 | 180 |
filelist = utils.list_relative_paths(fullpath)
|
190 | 181 |
filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
|
191 | 182 |
else:
|
192 |
- filelist = [(self.path, fullpath)]
|
|
183 |
+ filelist = [(self.get_absolute_path(), fullpath)]
|
|
193 | 184 |
|
194 | 185 |
self._key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist]
|
195 | 186 |
|
... | ... | @@ -200,7 +191,7 @@ class Workspace(): |
200 | 191 |
# Returns: The absolute path of the element's workspace.
|
201 | 192 |
#
|
202 | 193 |
def get_absolute_path(self):
|
203 |
- return os.path.join(self._toplevel_project.directory, self.path)
|
|
194 |
+ return os.path.join(self._toplevel_project.directory, self._path)
|
|
204 | 195 |
|
205 | 196 |
|
206 | 197 |
# Workspaces()
|
... | ... | @@ -236,6 +227,9 @@ class Workspaces(): |
236 | 227 |
# path (str) - The path in which the workspace should be kept
|
237 | 228 |
#
|
238 | 229 |
def create_workspace(self, element_name, path):
|
230 |
+ if path.startswith(self._toplevel_project.directory):
|
|
231 |
+ path = os.path.relpath(path, self._toplevel_project.directory)
|
|
232 |
+ |
|
239 | 233 |
self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
|
240 | 234 |
|
241 | 235 |
return self._workspaces[element_name]
|
... | ... | @@ -1394,7 +1394,8 @@ class Element(Plugin): |
1394 | 1394 |
# If mount_workspaces is set and we're doing incremental builds,
|
1395 | 1395 |
# the workspace is already mounted into the sandbox.
|
1396 | 1396 |
if not (mount_workspaces and self.__can_build_incrementally()):
|
1397 |
- with self.timed_activity("Staging local files at {}".format(workspace.path)):
|
|
1397 |
+ with self.timed_activity("Staging local files at {}"
|
|
1398 |
+ .format(workspace.get_absolute_path())):
|
|
1398 | 1399 |
workspace.stage(temp_staging_directory)
|
1399 | 1400 |
else:
|
1400 | 1401 |
# No workspace, stage directly
|
... | ... | @@ -1402,6 +1403,7 @@ class Element(Plugin): |
1402 | 1403 |
source._stage(temp_staging_directory)
|
1403 | 1404 |
|
1404 | 1405 |
vdirectory.import_files(temp_staging_directory)
|
1406 |
+ |
|
1405 | 1407 |
# Ensure deterministic mtime of sources at build time
|
1406 | 1408 |
vdirectory.set_deterministic_mtime()
|
1407 | 1409 |
# Ensure deterministic owners of sources at build time
|
... | ... | @@ -1557,7 +1559,7 @@ class Element(Plugin): |
1557 | 1559 |
path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep)
|
1558 | 1560 |
sandbox_vpath = sandbox_vroot.descend(path_components)
|
1559 | 1561 |
try:
|
1560 |
- sandbox_vpath.import_files(workspace.path)
|
|
1562 |
+ sandbox_vpath.import_files(workspace.get_absolute_path())
|
|
1561 | 1563 |
except UtilError as e:
|
1562 | 1564 |
self.warn("Failed to preserve workspace state for failed build sysroot: {}"
|
1563 | 1565 |
.format(e))
|
... | ... | @@ -1884,7 +1886,7 @@ class Element(Plugin): |
1884 | 1886 |
source._init_workspace(temp)
|
1885 | 1887 |
|
1886 | 1888 |
# Now hardlink the files into the workspace target.
|
1887 |
- utils.link_files(temp, workspace.path)
|
|
1889 |
+ utils.link_files(temp, workspace.get_absolute_path())
|
|
1888 | 1890 |
|
1889 | 1891 |
# _get_workspace():
|
1890 | 1892 |
#
|
... | ... | @@ -18,12 +18,13 @@ DATA_DIR = os.path.join( |
18 | 18 |
)
|
19 | 19 |
|
20 | 20 |
|
21 |
-def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
|
|
22 |
- project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
23 |
- bin_files_path = os.path.join(project, 'files', 'bin-files')
|
|
24 |
- element_path = os.path.join(project, 'elements')
|
|
21 |
+def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None):
|
|
22 |
+ if not workspace_dir:
|
|
23 |
+ workspace_dir = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
|
|
24 |
+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
|
|
25 |
+ bin_files_path = os.path.join(project_path, 'files', 'bin-files')
|
|
26 |
+ element_path = os.path.join(project_path, 'elements')
|
|
25 | 27 |
element_name = 'workspace-test-{}{}.bst'.format(kind, suffix)
|
26 |
- workspace = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
|
|
27 | 28 |
|
28 | 29 |
# Create our repo object of the given source type with
|
29 | 30 |
# the bin files, and then collect the initial ref.
|
... | ... | @@ -45,7 +46,7 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''): |
45 | 46 |
element_name))
|
46 | 47 |
|
47 | 48 |
# Assert that there is no reference, a track & fetch is needed
|
48 |
- state = cli.get_element_state(project, element_name)
|
|
49 |
+ state = cli.get_element_state(project_path, element_name)
|
|
49 | 50 |
if track:
|
50 | 51 |
assert state == 'no reference'
|
51 | 52 |
else:
|
... | ... | @@ -56,20 +57,20 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''): |
56 | 57 |
args = ['workspace', 'open']
|
57 | 58 |
if track:
|
58 | 59 |
args.append('--track')
|
59 |
- args.extend([element_name, workspace])
|
|
60 |
+ args.extend([element_name, workspace_dir])
|
|
61 |
+ result = cli.run(project=project_path, args=args)
|
|
60 | 62 |
|
61 |
- result = cli.run(project=project, args=args)
|
|
62 | 63 |
result.assert_success()
|
63 | 64 |
|
64 | 65 |
# Assert that we are now buildable because the source is
|
65 | 66 |
# now cached.
|
66 |
- assert cli.get_element_state(project, element_name) == 'buildable'
|
|
67 |
+ assert cli.get_element_state(project_path, element_name) == 'buildable'
|
|
67 | 68 |
|
68 | 69 |
# Check that the executable hello file is found in the workspace
|
69 |
- filename = os.path.join(workspace, 'usr', 'bin', 'hello')
|
|
70 |
+ filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
|
|
70 | 71 |
assert os.path.exists(filename)
|
71 | 72 |
|
72 |
- return (element_name, project, workspace)
|
|
73 |
+ return (element_name, project_path, workspace_dir)
|
|
73 | 74 |
|
74 | 75 |
|
75 | 76 |
@pytest.mark.datafiles(DATA_DIR)
|
... | ... | @@ -190,6 +191,45 @@ def test_close(cli, tmpdir, datafiles, kind): |
190 | 191 |
assert not os.path.exists(workspace)
|
191 | 192 |
|
192 | 193 |
|
194 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
195 |
+def test_close_external_after_move_project(cli, tmpdir, datafiles):
|
|
196 |
+ tmp_parent = os.path.dirname(str(tmpdir))
|
|
197 |
+ workspace_dir = os.path.join(tmp_parent, "workspace")
|
|
198 |
+ element_name, project_path, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, "", workspace_dir)
|
|
199 |
+ assert os.path.exists(workspace_dir)
|
|
200 |
+ tmp_dir = os.path.join(tmp_parent, 'external_project')
|
|
201 |
+ shutil.move(project_path, tmp_dir)
|
|
202 |
+ assert os.path.exists(tmp_dir)
|
|
203 |
+ |
|
204 |
+ # Close the workspace
|
|
205 |
+ result = cli.run(configure=False, project=tmp_dir, args=[
|
|
206 |
+ 'workspace', 'close', '--remove-dir', element_name
|
|
207 |
+ ])
|
|
208 |
+ result.assert_success()
|
|
209 |
+ |
|
210 |
+ # Assert the workspace dir has been deleted
|
|
211 |
+ assert not os.path.exists(workspace_dir)
|
|
212 |
+ shutil.move(tmp_dir, project_path)
|
|
213 |
+ |
|
214 |
+ |
|
215 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
216 |
+def test_close_internal_after_move_project(cli, tmpdir, datafiles):
|
|
217 |
+ element_name, project, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
|
|
218 |
+ tmp_dir = os.path.join(os.path.dirname(str(tmpdir)), 'external_project')
|
|
219 |
+ shutil.move(str(tmpdir), tmp_dir)
|
|
220 |
+ assert os.path.exists(tmp_dir)
|
|
221 |
+ |
|
222 |
+ # Close the workspace
|
|
223 |
+ result = cli.run(configure=False, project=tmp_dir, args=[
|
|
224 |
+ 'workspace', 'close', '--remove-dir', element_name
|
|
225 |
+ ])
|
|
226 |
+ result.assert_success()
|
|
227 |
+ |
|
228 |
+ # Assert the workspace dir has been deleted
|
|
229 |
+ workspace = os.path.join(tmp_dir, 'workspace')
|
|
230 |
+ assert not os.path.exists(workspace)
|
|
231 |
+ |
|
232 |
+ |
|
193 | 233 |
@pytest.mark.datafiles(DATA_DIR)
|
194 | 234 |
def test_close_removed(cli, tmpdir, datafiles):
|
195 | 235 |
element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
|