Jonathan Maw pushed to branch jonathan/workspace-fragment-create at BuildStream / buildstream
Commits:
-
7738f6df
by Daniel Silverstone at 2018-11-09T11:25:44Z
-
62f59eaa
by Valentin David at 2018-11-09T13:30:06Z
-
35ec6b6a
by Jim MacArthur at 2018-11-09T14:58:22Z
-
b93100ec
by Jim MacArthur at 2018-11-09T14:58:22Z
-
78691fa0
by Jim MacArthur at 2018-11-09T14:58:22Z
-
7ce6581b
by Jim MacArthur at 2018-11-09T15:25:46Z
-
6f4351ec
by Benjamin Schubert at 2018-11-09T16:53:42Z
-
e6e03451
by richardmaw-codethink at 2018-11-12T11:05:15Z
-
ae5ccd76
by Phillip Smyth at 2018-11-12T14:57:51Z
-
56a54161
by Phillip Smyth at 2018-11-12T14:57:51Z
-
e55a9703
by Jürg Billeter at 2018-11-12T15:44:45Z
-
36919a8b
by Jonathan Maw at 2018-11-12T17:23:54Z
-
1ef0b448
by Jonathan Maw at 2018-11-12T17:23:54Z
-
d93ccd59
by Jonathan Maw at 2018-11-12T17:23:54Z
-
a2b2fde9
by Jonathan Maw at 2018-11-12T17:23:54Z
-
c788330e
by Jonathan Maw at 2018-11-12T17:23:54Z
-
65a6e9b7
by Jonathan Maw at 2018-11-12T17:23:54Z
-
d3ed4542
by Jonathan Maw at 2018-11-12T17:23:54Z
-
bb7b407f
by Jonathan Maw at 2018-11-12T17:23:54Z
-
dc1d5f7f
by Jonathan Maw at 2018-11-12T17:23:54Z
-
51bc2dd3
by Jonathan Maw at 2018-11-12T17:23:54Z
-
42e1c85f
by Jonathan Maw at 2018-11-12T17:23:54Z
-
61fb721c
by Jonathan Maw at 2018-11-12T17:23:55Z
13 changed files:
- buildstream/_frontend/cli.py
- buildstream/_project.py
- buildstream/_stream.py
- buildstream/_workspaces.py
- buildstream/plugin.py
- buildstream/source.py
- buildstream/storage/_casbaseddirectory.py
- buildstream/utils.py
- + tests/frontend/invalid_element_path/project.conf
- tests/frontend/show.py
- tests/frontend/workspace.py
- tests/integration/shell.py
- + tests/storage/virtual_directory_import.py
Changes:
... | ... | @@ -59,18 +59,9 @@ def complete_target(args, incomplete): |
59 | 59 |
:return: all the possible user-specified completions for the param
|
60 | 60 |
"""
|
61 | 61 |
|
62 |
+ from .. import utils
|
|
62 | 63 |
project_conf = 'project.conf'
|
63 | 64 |
|
64 |
- def ensure_project_dir(directory):
|
|
65 |
- directory = os.path.abspath(directory)
|
|
66 |
- while not os.path.isfile(os.path.join(directory, project_conf)):
|
|
67 |
- parent_dir = os.path.dirname(directory)
|
|
68 |
- if directory == parent_dir:
|
|
69 |
- break
|
|
70 |
- directory = parent_dir
|
|
71 |
- |
|
72 |
- return directory
|
|
73 |
- |
|
74 | 65 |
# First resolve the directory, in case there is an
|
75 | 66 |
# active --directory/-C option
|
76 | 67 |
#
|
... | ... | @@ -89,7 +80,7 @@ def complete_target(args, incomplete): |
89 | 80 |
else:
|
90 | 81 |
# Check if this directory or any of its parent directories
|
91 | 82 |
# contain a project config file
|
92 |
- base_directory = ensure_project_dir(base_directory)
|
|
83 |
+ base_directory = utils._search_upward_for_file(base_directory, project_conf)
|
|
93 | 84 |
|
94 | 85 |
# Now parse the project.conf just to find the element path,
|
95 | 86 |
# this is unfortunately a bit heavy.
|
... | ... | @@ -713,10 +704,12 @@ def workspace_open(app, no_checkout, force, track_, element, directory): |
713 | 704 |
help="Remove the path that contains the closed workspace")
|
714 | 705 |
@click.option('--all', '-a', 'all_', default=False, is_flag=True,
|
715 | 706 |
help="Close all open workspaces")
|
707 |
+@click.option('--force', '-f', default=False, is_flag=True,
|
|
708 |
+ help="Always close the workspace and/or delete your changes")
|
|
716 | 709 |
@click.argument('elements', nargs=-1,
|
717 | 710 |
type=click.Path(readable=False))
|
718 | 711 |
@click.pass_obj
|
719 |
-def workspace_close(app, remove_dir, all_, elements):
|
|
712 |
+def workspace_close(app, remove_dir, all_, force, elements):
|
|
720 | 713 |
"""Close a workspace"""
|
721 | 714 |
|
722 | 715 |
if not (all_ or elements):
|
... | ... | @@ -735,15 +728,25 @@ def workspace_close(app, remove_dir, all_, elements): |
735 | 728 |
|
736 | 729 |
elements = app.stream.redirect_element_names(elements)
|
737 | 730 |
|
738 |
- # Check that the workspaces in question exist
|
|
731 |
+ # Check that the workspaces in question exist, and that it's safe to
|
|
732 |
+ # remove them.
|
|
739 | 733 |
nonexisting = []
|
740 | 734 |
for element_name in elements:
|
741 | 735 |
if not app.stream.workspace_exists(element_name):
|
742 | 736 |
nonexisting.append(element_name)
|
737 |
+ if app.stream.workspace_is_required(element_name):
|
|
738 |
+ if app.interactive:
|
|
739 |
+ click.echo("Removing '{}' will prevent you from running buildstream commands".format(element_name))
|
|
740 |
+ if not click.confirm('Are you sure you want to close this workspace?'):
|
|
741 |
+ click.echo('Aborting', err=True)
|
|
742 |
+ sys.exit(-1)
|
|
743 |
+ elif not force:
|
|
744 |
+ raise AppError("Cannot close workspaces. Workspace {} is being used to load the project"
|
|
745 |
+ .format(element_name), reason='closing-required-workspace')
|
|
743 | 746 |
if nonexisting:
|
744 | 747 |
raise AppError("Workspace does not exist", detail="\n".join(nonexisting))
|
745 | 748 |
|
746 |
- if app.interactive and remove_dir:
|
|
749 |
+ if app.interactive and remove_dir and not force:
|
|
747 | 750 |
if not click.confirm('This will remove all your changes, are you sure?'):
|
748 | 751 |
click.echo('Aborting', err=True)
|
749 | 752 |
sys.exit(-1)
|
... | ... | @@ -40,6 +40,7 @@ from .element import Element |
40 | 40 |
from ._message import Message, MessageType
|
41 | 41 |
from ._includes import Includes
|
42 | 42 |
from ._platform import Platform
|
43 |
+from ._workspaces import WorkspaceLocal
|
|
43 | 44 |
|
44 | 45 |
|
45 | 46 |
# Project Configuration file
|
... | ... | @@ -94,8 +95,8 @@ class Project(): |
94 | 95 |
# The project name
|
95 | 96 |
self.name = None
|
96 | 97 |
|
97 |
- # The project directory
|
|
98 |
- self.directory = self._ensure_project_dir(directory)
|
|
98 |
+ # The project directory, and whether the project was found from an external workspace
|
|
99 |
+ self.directory, self._required_workspace_element = self._find_project_dir(directory)
|
|
99 | 100 |
|
100 | 101 |
# Absolute path to where elements are loaded from within the project
|
101 | 102 |
self.element_path = None
|
... | ... | @@ -219,6 +220,19 @@ class Project(): |
219 | 220 |
|
220 | 221 |
return self._cache_key
|
221 | 222 |
|
223 |
+ def _validate_node(self, node):
|
|
224 |
+ _yaml.node_validate(node, [
|
|
225 |
+ 'format-version',
|
|
226 |
+ 'element-path', 'variables',
|
|
227 |
+ 'environment', 'environment-nocache',
|
|
228 |
+ 'split-rules', 'elements', 'plugins',
|
|
229 |
+ 'aliases', 'name',
|
|
230 |
+ 'artifacts', 'options',
|
|
231 |
+ 'fail-on-overlap', 'shell', 'fatal-warnings',
|
|
232 |
+ 'ref-storage', 'sandbox', 'mirrors', 'remote-execution',
|
|
233 |
+ 'sources', '(@)'
|
|
234 |
+ ])
|
|
235 |
+ |
|
222 | 236 |
# create_element()
|
223 | 237 |
#
|
224 | 238 |
# Instantiate and return an element
|
... | ... | @@ -357,6 +371,14 @@ class Project(): |
357 | 371 |
|
358 | 372 |
self._load_second_pass()
|
359 | 373 |
|
374 |
+ # required_workspace_element()
|
|
375 |
+ #
|
|
376 |
+ # Returns the element whose workspace is required to load this project,
|
|
377 |
+ # if any.
|
|
378 |
+ #
|
|
379 |
+ def required_workspace_element(self):
|
|
380 |
+ return self._required_workspace_element
|
|
381 |
+ |
|
360 | 382 |
# cleanup()
|
361 | 383 |
#
|
362 | 384 |
# Cleans up resources used loading elements
|
... | ... | @@ -402,6 +424,8 @@ class Project(): |
402 | 424 |
"Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
|
403 | 425 |
.format(format_version, major, minor, BST_FORMAT_VERSION))
|
404 | 426 |
|
427 |
+ self._validate_node(pre_config_node)
|
|
428 |
+ |
|
405 | 429 |
# FIXME:
|
406 | 430 |
#
|
407 | 431 |
# Performing this check manually in the absense
|
... | ... | @@ -467,16 +491,7 @@ class Project(): |
467 | 491 |
|
468 | 492 |
self._load_pass(config, self.config)
|
469 | 493 |
|
470 |
- _yaml.node_validate(config, [
|
|
471 |
- 'format-version',
|
|
472 |
- 'element-path', 'variables',
|
|
473 |
- 'environment', 'environment-nocache',
|
|
474 |
- 'split-rules', 'elements', 'plugins',
|
|
475 |
- 'aliases', 'name',
|
|
476 |
- 'artifacts', 'options',
|
|
477 |
- 'fail-on-overlap', 'shell', 'fatal-warnings',
|
|
478 |
- 'ref-storage', 'sandbox', 'mirrors', 'remote-execution'
|
|
479 |
- ])
|
|
494 |
+ self._validate_node(config)
|
|
480 | 495 |
|
481 | 496 |
#
|
482 | 497 |
# Now all YAML composition is done, from here on we just load
|
... | ... | @@ -645,7 +660,7 @@ class Project(): |
645 | 660 |
# Source url aliases
|
646 | 661 |
output._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
|
647 | 662 |
|
648 |
- # _ensure_project_dir()
|
|
663 |
+ # _find_project_dir()
|
|
649 | 664 |
#
|
650 | 665 |
# Returns path of the project directory, if a configuration file is found
|
651 | 666 |
# in given directory or any of its parent directories.
|
... | ... | @@ -656,18 +671,24 @@ class Project(): |
656 | 671 |
# Raises:
|
657 | 672 |
# LoadError if project.conf is not found
|
658 | 673 |
#
|
659 |
- def _ensure_project_dir(self, directory):
|
|
660 |
- directory = os.path.abspath(directory)
|
|
661 |
- while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
|
|
662 |
- parent_dir = os.path.dirname(directory)
|
|
663 |
- if directory == parent_dir:
|
|
674 |
+ # Returns:
|
|
675 |
+ # (str) - the directory that contains the project, and
|
|
676 |
+ # (str) - the name of the element required to find the project, or an empty string
|
|
677 |
+ def _find_project_dir(self, directory):
|
|
678 |
+ workspace_element = ""
|
|
679 |
+ project_directory = utils._search_upward_for_file(directory, _PROJECT_CONF_FILE)
|
|
680 |
+ if not project_directory:
|
|
681 |
+ workspace_local = WorkspaceLocal.load(directory)
|
|
682 |
+ if workspace_local:
|
|
683 |
+ project_directory = workspace_local.get_default_path()
|
|
684 |
+ workspace_element = workspace_local.get_default_element()
|
|
685 |
+ else:
|
|
664 | 686 |
raise LoadError(
|
665 | 687 |
LoadErrorReason.MISSING_PROJECT_CONF,
|
666 | 688 |
'{} not found in current directory or any of its parent directories'
|
667 | 689 |
.format(_PROJECT_CONF_FILE))
|
668 |
- directory = parent_dir
|
|
669 | 690 |
|
670 |
- return directory
|
|
691 |
+ return project_directory, workspace_element
|
|
671 | 692 |
|
672 | 693 |
def _load_plugin_factories(self, config, output):
|
673 | 694 |
plugin_source_origins = [] # Origins of custom sources
|
... | ... | @@ -32,6 +32,7 @@ from ._exceptions import StreamError, ImplError, BstError, set_last_task_error |
32 | 32 |
from ._message import Message, MessageType
|
33 | 33 |
from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
|
34 | 34 |
from ._pipeline import Pipeline, PipelineSelection
|
35 |
+from ._workspaces import WorkspaceLocal
|
|
35 | 36 |
from . import utils, _yaml, _site
|
36 | 37 |
from . import Scope, Consistency
|
37 | 38 |
|
... | ... | @@ -516,6 +517,10 @@ class Stream(): |
516 | 517 |
with target.timed_activity("Staging sources to {}".format(directory)):
|
517 | 518 |
target._open_workspace()
|
518 | 519 |
|
520 |
+ project = self._context.get_toplevel_project()
|
|
521 |
+ workspace_local = WorkspaceLocal.create(directory, project.directory, target._get_full_name())
|
|
522 |
+ workspace_local.write()
|
|
523 |
+ |
|
519 | 524 |
workspaces.save_config()
|
520 | 525 |
self._message(MessageType.INFO, "Saved workspace configuration")
|
521 | 526 |
|
... | ... | @@ -540,6 +545,11 @@ class Stream(): |
540 | 545 |
except OSError as e:
|
541 | 546 |
raise StreamError("Could not remove '{}': {}"
|
542 | 547 |
.format(workspace.get_absolute_path(), e)) from e
|
548 |
+ else:
|
|
549 |
+ # TODO: At some point, closing a workspace only deletes the file if no projects are using it.
|
|
550 |
+ workspace_local = WorkspaceLocal.load(workspace.get_absolute_path())
|
|
551 |
+ if workspace_local:
|
|
552 |
+ workspace_local.delete()
|
|
543 | 553 |
|
544 | 554 |
# Delete the workspace and save the configuration
|
545 | 555 |
workspaces.delete_workspace(element_name)
|
... | ... | @@ -633,6 +643,20 @@ class Stream(): |
633 | 643 |
|
634 | 644 |
return False
|
635 | 645 |
|
646 |
+ # workspace_is_required()
|
|
647 |
+ #
|
|
648 |
+ # Checks whether the workspace belonging to element_name is required to
|
|
649 |
+ # load the project
|
|
650 |
+ #
|
|
651 |
+ # Args:
|
|
652 |
+ # element_name (str): The element whose workspace may be required
|
|
653 |
+ #
|
|
654 |
+ # Returns:
|
|
655 |
+ # (bool): True if the workspace is required
|
|
656 |
+ def workspace_is_required(self, element_name):
|
|
657 |
+ required_elm = self._project.required_workspace_element()
|
|
658 |
+ return required_elm == element_name
|
|
659 |
+ |
|
636 | 660 |
# workspace_list
|
637 | 661 |
#
|
638 | 662 |
# Serializes the workspaces and dumps them in YAML to stdout.
|
... | ... | @@ -25,6 +25,149 @@ from ._exceptions import LoadError, LoadErrorReason |
25 | 25 |
|
26 | 26 |
|
27 | 27 |
BST_WORKSPACE_FORMAT_VERSION = 3
|
28 |
+BST_WORKSPACE_LOCAL_FORMAT_VERSION = 1
|
|
29 |
+WORKSPACE_LOCAL_FILE = ".bstproject.yaml"
|
|
30 |
+ |
|
31 |
+ |
|
32 |
+# WorkspaceLocal()
|
|
33 |
+#
|
|
34 |
+# An object to contain various helper functions and data required for
|
|
35 |
+# referring from a workspace back to buildstream.
|
|
36 |
+#
|
|
37 |
+# Args:
|
|
38 |
+# directory (str): The directory that the workspace exists in
|
|
39 |
+# project_path (str): The project path used to refer back
|
|
40 |
+# to buildstream projects.
|
|
41 |
+# element_name (str): The name of the element used to create this workspace.
|
|
42 |
+class WorkspaceLocal():
|
|
43 |
+ def __init__(self, directory, project_path="", element_name=""):
|
|
44 |
+ self._projects = []
|
|
45 |
+ self._directory = directory
|
|
46 |
+ |
|
47 |
+ assert (project_path and element_name) or (not project_path and not element_name)
|
|
48 |
+ if project_path:
|
|
49 |
+ self._add_project(project_path, element_name)
|
|
50 |
+ |
|
51 |
+ # get_default_path()
|
|
52 |
+ #
|
|
53 |
+ # Retrieves the default path to a project.
|
|
54 |
+ #
|
|
55 |
+ # Returns:
|
|
56 |
+ # (str): The path to a project
|
|
57 |
+ def get_default_path(self):
|
|
58 |
+ return self._projects[0]['project-path']
|
|
59 |
+ |
|
60 |
+ # get_default_element()
|
|
61 |
+ #
|
|
62 |
+ # Retrieves the name of the element that owns this workspace.
|
|
63 |
+ #
|
|
64 |
+ # Returns:
|
|
65 |
+ # (str): The name of an element
|
|
66 |
+ def get_default_element(self):
|
|
67 |
+ return self._projects[0]['element-name']
|
|
68 |
+ |
|
69 |
+ # to_dict()
|
|
70 |
+ #
|
|
71 |
+ # Turn the members data into a dict for serialization purposes
|
|
72 |
+ #
|
|
73 |
+ # Returns:
|
|
74 |
+ # (dict): A dict representation of the WorkspaceLocal
|
|
75 |
+ #
|
|
76 |
+ def to_dict(self):
|
|
77 |
+ ret = {
|
|
78 |
+ 'projects': self._projects,
|
|
79 |
+ 'format-version': BST_WORKSPACE_LOCAL_FORMAT_VERSION,
|
|
80 |
+ }
|
|
81 |
+ return ret
|
|
82 |
+ |
|
83 |
+ # from_dict()
|
|
84 |
+ #
|
|
85 |
+ # Loads a new WorkspaceLocal from a simple dictionary
|
|
86 |
+ #
|
|
87 |
+ # Args:
|
|
88 |
+ # directory (str): The directory that the workspace exists in
|
|
89 |
+ # dictionary (dict): The dict to generate a WorkspaceLocal from
|
|
90 |
+ #
|
|
91 |
+ # Returns:
|
|
92 |
+ # (WorkspaceLocal): A newly instantiated WorkspaceLocal
|
|
93 |
+ @classmethod
|
|
94 |
+ def from_dict(cls, directory, dictionary):
|
|
95 |
+ # Only know how to handle one format-version at the moment.
|
|
96 |
+ format_version = int(dictionary['format-version'])
|
|
97 |
+ assert format_version == BST_WORKSPACE_LOCAL_FORMAT_VERSION, \
|
|
98 |
+ "Format version {} not found in {}".format(BST_WORKSPACE_LOCAL_FORMAT_VERSION, dictionary)
|
|
99 |
+ |
|
100 |
+ workspace_local = cls(directory)
|
|
101 |
+ for item in dictionary['projects']:
|
|
102 |
+ workspace_local._add_project(item['project-path'], item['element-name'])
|
|
103 |
+ |
|
104 |
+ return workspace_local
|
|
105 |
+ |
|
106 |
+ # create()
|
|
107 |
+ #
|
|
108 |
+ # Creates a new WorkspaceLocal
|
|
109 |
+ #
|
|
110 |
+ # Args:
|
|
111 |
+ # directory (str): The directory that the workspace exists in
|
|
112 |
+ # project_path (str): The path to the project to store
|
|
113 |
+ # element_name (str): The name of the element within the project
|
|
114 |
+ #
|
|
115 |
+ # Returns:
|
|
116 |
+ # (WorkspaceLocal): The created WorkspaceLocal
|
|
117 |
+ @classmethod
|
|
118 |
+ def create(cls, directory, project_path, element_name):
|
|
119 |
+ # TODO: Load WorkspaceLocal if it exists, and maybe add project_path to it
|
|
120 |
+ return cls(directory, project_path, element_name)
|
|
121 |
+ |
|
122 |
+ # load()
|
|
123 |
+ #
|
|
124 |
+ # Loads the WorkspaceLocal for a given directory. This directory may be a
|
|
125 |
+ # subdirectory of the workspace's directory.
|
|
126 |
+ #
|
|
127 |
+ # Args:
|
|
128 |
+ # directory (str): The directory
|
|
129 |
+ # Returns:
|
|
130 |
+ # (WorkspaceLocal): The created WorkspaceLocal, if in a workspace, or
|
|
131 |
+ # (NoneType): None, if the directory is not inside a workspace.
|
|
132 |
+ @classmethod
|
|
133 |
+ def load(cls, directory):
|
|
134 |
+ local_dir = cls.search_for_dir(directory)
|
|
135 |
+ if local_dir:
|
|
136 |
+ workspace_file = os.path.join(local_dir, WORKSPACE_LOCAL_FILE)
|
|
137 |
+ data_dict = _yaml.load(workspace_file)
|
|
138 |
+ return cls.from_dict(local_dir, data_dict)
|
|
139 |
+ else:
|
|
140 |
+ return None
|
|
141 |
+ |
|
142 |
+ # write()
|
|
143 |
+ #
|
|
144 |
+ # Writes the WorkspaceLocal to disk
|
|
145 |
+ def write(self):
|
|
146 |
+ os.makedirs(self._directory, exist_ok=True)
|
|
147 |
+ _yaml.dump(self.to_dict(), self._get_filename())
|
|
148 |
+ |
|
149 |
+ # delete()
|
|
150 |
+ #
|
|
151 |
+ # Deletes the WorkspaceLocal from disk, if it exists.
|
|
152 |
+ def delete(self):
|
|
153 |
+ try:
|
|
154 |
+ os.unlink(self._get_filename())
|
|
155 |
+ except FileNotFoundError:
|
|
156 |
+ pass
|
|
157 |
+ |
|
158 |
+ # search_for_dir()
|
|
159 |
+ #
|
|
160 |
+ # Returns the directory that contains the workspace local file,
|
|
161 |
+ # searching upwards from search_dir.
|
|
162 |
+ @staticmethod
|
|
163 |
+ def search_for_dir(search_dir):
|
|
164 |
+ return utils._search_upward_for_file(search_dir, WORKSPACE_LOCAL_FILE)
|
|
165 |
+ |
|
166 |
+ def _get_filename(self):
|
|
167 |
+ return os.path.join(self._directory, WORKSPACE_LOCAL_FILE)
|
|
168 |
+ |
|
169 |
+ def _add_project(self, project_path, element_name):
|
|
170 |
+ self._projects.append({'project-path': project_path, 'element-name': element_name})
|
|
28 | 171 |
|
29 | 172 |
|
30 | 173 |
# Workspace()
|
... | ... | @@ -174,10 +317,15 @@ class Workspace(): |
174 | 317 |
if recalculate or self._key is None:
|
175 | 318 |
fullpath = self.get_absolute_path()
|
176 | 319 |
|
320 |
+ excluded_files = [WORKSPACE_LOCAL_FILE]
|
|
321 |
+ |
|
177 | 322 |
# Get a list of tuples of the the project relative paths and fullpaths
|
178 | 323 |
if os.path.isdir(fullpath):
|
179 | 324 |
filelist = utils.list_relative_paths(fullpath)
|
180 |
- filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
|
|
325 |
+ filelist = [
|
|
326 |
+ (relpath, os.path.join(fullpath, relpath)) for relpath in filelist
|
|
327 |
+ if relpath not in excluded_files
|
|
328 |
+ ]
|
|
181 | 329 |
else:
|
182 | 330 |
filelist = [(self.get_absolute_path(), fullpath)]
|
183 | 331 |
|
... | ... | @@ -111,6 +111,7 @@ Class Reference |
111 | 111 |
|
112 | 112 |
import os
|
113 | 113 |
import subprocess
|
114 |
+import sys
|
|
114 | 115 |
from contextlib import contextmanager
|
115 | 116 |
from weakref import WeakValueDictionary
|
116 | 117 |
|
... | ... | @@ -190,7 +191,7 @@ class Plugin(): |
190 | 191 |
# Dont send anything through the Message() pipeline at destruction time,
|
191 | 192 |
# any subsequent lookup of plugin by unique id would raise KeyError.
|
192 | 193 |
if self.__context.log_debug:
|
193 |
- print("DEBUG: Destroyed: {}".format(self))
|
|
194 |
+ sys.stderr.write("DEBUG: Destroyed: {}\n".format(self))
|
|
194 | 195 |
|
195 | 196 |
def __str__(self):
|
196 | 197 |
return "{kind} {typetag} at {provenance}".format(
|
... | ... | @@ -973,32 +973,34 @@ class Source(Plugin): |
973 | 973 |
# the items of source_fetchers, if it happens to be a generator.
|
974 | 974 |
#
|
975 | 975 |
source_fetchers = iter(source_fetchers)
|
976 |
- try:
|
|
977 | 976 |
|
978 |
- while True:
|
|
977 |
+ while True:
|
|
979 | 978 |
|
980 |
- with context.silence():
|
|
979 |
+ with context.silence():
|
|
980 |
+ try:
|
|
981 | 981 |
fetcher = next(source_fetchers)
|
982 |
- |
|
983 |
- alias = fetcher._get_alias()
|
|
984 |
- for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
|
|
985 |
- try:
|
|
986 |
- fetcher.fetch(uri)
|
|
987 |
- # FIXME: Need to consider temporary vs. permanent failures,
|
|
988 |
- # and how this works with retries.
|
|
989 |
- except BstError as e:
|
|
990 |
- last_error = e
|
|
991 |
- continue
|
|
992 |
- |
|
993 |
- # No error, we're done with this fetcher
|
|
982 |
+ except StopIteration:
|
|
983 |
+ # as per PEP479, we are not allowed to let StopIteration
|
|
984 |
+ # thrown from a context manager.
|
|
985 |
+ # Catching it here and breaking instead.
|
|
994 | 986 |
break
|
995 | 987 |
|
996 |
- else:
|
|
997 |
- # No break occurred, raise the last detected error
|
|
998 |
- raise last_error
|
|
988 |
+ alias = fetcher._get_alias()
|
|
989 |
+ for uri in project.get_alias_uris(alias, first_pass=self.__first_pass):
|
|
990 |
+ try:
|
|
991 |
+ fetcher.fetch(uri)
|
|
992 |
+ # FIXME: Need to consider temporary vs. permanent failures,
|
|
993 |
+ # and how this works with retries.
|
|
994 |
+ except BstError as e:
|
|
995 |
+ last_error = e
|
|
996 |
+ continue
|
|
999 | 997 |
|
1000 |
- except StopIteration:
|
|
1001 |
- pass
|
|
998 |
+ # No error, we're done with this fetcher
|
|
999 |
+ break
|
|
1000 |
+ |
|
1001 |
+ else:
|
|
1002 |
+ # No break occurred, raise the last detected error
|
|
1003 |
+ raise last_error
|
|
1002 | 1004 |
|
1003 | 1005 |
# Default codepath is to reinstantiate the Source
|
1004 | 1006 |
#
|
... | ... | @@ -30,7 +30,6 @@ See also: :ref:`sandboxing`. |
30 | 30 |
from collections import OrderedDict
|
31 | 31 |
|
32 | 32 |
import os
|
33 |
-import tempfile
|
|
34 | 33 |
import stat
|
35 | 34 |
|
36 | 35 |
from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
|
... | ... | @@ -51,6 +50,183 @@ class IndexEntry(): |
51 | 50 |
self.modified = modified
|
52 | 51 |
|
53 | 52 |
|
53 |
+class ResolutionException(VirtualDirectoryError):
|
|
54 |
+ """ Superclass of all exceptions that can be raised by
|
|
55 |
+ CasBasedDirectory._resolve. Should not be used outside this module. """
|
|
56 |
+ pass
|
|
57 |
+ |
|
58 |
+ |
|
59 |
+class InfiniteSymlinkException(ResolutionException):
|
|
60 |
+ """ Raised when an infinite symlink loop is found. """
|
|
61 |
+ pass
|
|
62 |
+ |
|
63 |
+ |
|
64 |
+class AbsoluteSymlinkException(ResolutionException):
|
|
65 |
+ """Raised if we try to follow an absolute symlink (i.e. one whose
|
|
66 |
+ target starts with the path separator) and we have disallowed
|
|
67 |
+ following such symlinks.
|
|
68 |
+ """
|
|
69 |
+ pass
|
|
70 |
+ |
|
71 |
+ |
|
72 |
+class UnexpectedFileException(ResolutionException):
|
|
73 |
+ """Raised if we were found a file where a directory or symlink was
|
|
74 |
+ expected, for example we try to resolve a symlink pointing to
|
|
75 |
+ /a/b/c but /a/b is a file.
|
|
76 |
+ """
|
|
77 |
+ def __init__(self, message=""):
|
|
78 |
+ """Allow constructor with no arguments, since this can be raised in
|
|
79 |
+ places where there isn't sufficient information to write the
|
|
80 |
+ message.
|
|
81 |
+ """
|
|
82 |
+ super().__init__(message)
|
|
83 |
+ |
|
84 |
+ |
|
85 |
+class _Resolver():
|
|
86 |
+ """A class for resolving symlinks inside CAS-based directories. As
|
|
87 |
+ well as providing a namespace for some functions, this also
|
|
88 |
+ contains two flags which are constant throughout one resolution
|
|
89 |
+ operation and the 'seen_objects' list used to detect infinite
|
|
90 |
+ symlink loops.
|
|
91 |
+ |
|
92 |
+ """
|
|
93 |
+ |
|
94 |
+ def __init__(self, absolute_symlinks_resolve=True, force_create=False):
|
|
95 |
+ self.absolute_symlinks_resolve = absolute_symlinks_resolve
|
|
96 |
+ self.force_create = force_create
|
|
97 |
+ self.seen_objects = []
|
|
98 |
+ |
|
99 |
+ def resolve(self, name, directory):
|
|
100 |
+ """Resolves any name to an object. If the name points to a symlink in
|
|
101 |
+ the directory, it returns the thing it points to,
|
|
102 |
+ recursively.
|
|
103 |
+ |
|
104 |
+ Returns a CasBasedDirectory, FileNode or None. None indicates
|
|
105 |
+ either that 'target' does not exist in this directory, or is a
|
|
106 |
+ symlink chain which points to a nonexistent name (broken
|
|
107 |
+ symlink).
|
|
108 |
+ |
|
109 |
+ Raises:
|
|
110 |
+ |
|
111 |
+ - InfiniteSymlinkException if 'name' points to an infinite
|
|
112 |
+ symlink loop.
|
|
113 |
+ - AbsoluteSymlinkException if 'name' points to an absolute
|
|
114 |
+ symlink and absolute_symlinks_resolve is False.
|
|
115 |
+ - UnexpectedFileException if at any point during resolution we
|
|
116 |
+ find a file which we expected to be a directory or symlink.
|
|
117 |
+ |
|
118 |
+ If force_create is set, this will attempt to create
|
|
119 |
+ directories to make symlinks and directories resolve. Files
|
|
120 |
+ present in symlink target paths will also be removed and
|
|
121 |
+ replaced with directories. If force_create is off, this will
|
|
122 |
+ never alter 'directory'.
|
|
123 |
+ |
|
124 |
+ """
|
|
125 |
+ |
|
126 |
+ # First check for nonexistent things or 'normal' objects and return them
|
|
127 |
+ if name not in directory.index:
|
|
128 |
+ return None
|
|
129 |
+ index_entry = directory.index[name]
|
|
130 |
+ if isinstance(index_entry.buildstream_object, Directory):
|
|
131 |
+ return index_entry.buildstream_object
|
|
132 |
+ elif isinstance(index_entry.pb_object, remote_execution_pb2.FileNode):
|
|
133 |
+ return index_entry.pb_object
|
|
134 |
+ |
|
135 |
+ # Now we must be dealing with a symlink.
|
|
136 |
+ assert isinstance(index_entry.pb_object, remote_execution_pb2.SymlinkNode)
|
|
137 |
+ |
|
138 |
+ symlink_object = index_entry.pb_object
|
|
139 |
+ if symlink_object in self.seen_objects:
|
|
140 |
+ # Infinite symlink loop detected
|
|
141 |
+ message = ("Infinite symlink loop found during resolution. " +
|
|
142 |
+ "First repeated element is {}".format(name))
|
|
143 |
+ raise InfiniteSymlinkException(message=message)
|
|
144 |
+ |
|
145 |
+ self.seen_objects.append(symlink_object)
|
|
146 |
+ |
|
147 |
+ components = symlink_object.target.split(CasBasedDirectory._pb2_path_sep)
|
|
148 |
+ absolute = symlink_object.target.startswith(CasBasedDirectory._pb2_absolute_path_prefix)
|
|
149 |
+ |
|
150 |
+ if absolute:
|
|
151 |
+ if self.absolute_symlinks_resolve:
|
|
152 |
+ directory = directory.find_root()
|
|
153 |
+ # Discard the first empty element
|
|
154 |
+ components.pop(0)
|
|
155 |
+ else:
|
|
156 |
+ # Unresolvable absolute symlink
|
|
157 |
+ message = "{} is an absolute symlink, which was disallowed during resolution".format(name)
|
|
158 |
+ raise AbsoluteSymlinkException(message=message)
|
|
159 |
+ |
|
160 |
+ resolution = directory
|
|
161 |
+ while components and isinstance(resolution, CasBasedDirectory):
|
|
162 |
+ c = components.pop(0)
|
|
163 |
+ directory = resolution
|
|
164 |
+ |
|
165 |
+ try:
|
|
166 |
+ resolution = self._resolve_path_component(c, directory, components)
|
|
167 |
+ except UnexpectedFileException as original:
|
|
168 |
+ errormsg = ("Reached a file called {} while trying to resolve a symlink; " +
|
|
169 |
+ "cannot proceed. The remaining path components are {}.")
|
|
170 |
+ raise UnexpectedFileException(errormsg.format(c, components)) from original
|
|
171 |
+ |
|
172 |
+ return resolution
|
|
173 |
+ |
|
174 |
+ def _resolve_path_component(self, c, directory, components_remaining):
|
|
175 |
+ if c == ".":
|
|
176 |
+ resolution = directory
|
|
177 |
+ elif c == "..":
|
|
178 |
+ if directory.parent is not None:
|
|
179 |
+ resolution = directory.parent
|
|
180 |
+ else:
|
|
181 |
+ # If directory.parent *is* None, this is an attempt to
|
|
182 |
+ # access '..' from the root, which is valid under
|
|
183 |
+ # POSIX; it just returns the root.
|
|
184 |
+ resolution = directory
|
|
185 |
+ elif c in directory.index:
|
|
186 |
+ try:
|
|
187 |
+ resolution = self._resolve_through_files(c, directory, components_remaining)
|
|
188 |
+ except UnexpectedFileException as original:
|
|
189 |
+ errormsg = ("Reached a file called {} while trying to resolve a symlink; " +
|
|
190 |
+ "cannot proceed. The remaining path components are {}.")
|
|
191 |
+ raise UnexpectedFileException(errormsg.format(c, components_remaining)) from original
|
|
192 |
+ else:
|
|
193 |
+ # c is not in our index
|
|
194 |
+ if self.force_create:
|
|
195 |
+ resolution = directory.descend(c, create=True)
|
|
196 |
+ else:
|
|
197 |
+ resolution = None
|
|
198 |
+ return resolution
|
|
199 |
+ |
|
200 |
+ def _resolve_through_files(self, c, directory, require_traversable):
|
|
201 |
+ """A wrapper to resolve() which deals with files being found
|
|
202 |
+ in the middle of paths, for example trying to resolve a symlink
|
|
203 |
+ which points to /usr/lib64/libfoo when 'lib64' is a file.
|
|
204 |
+ |
|
205 |
+ require_traversable: If this is True, never return a file
|
|
206 |
+ node. Instead, if force_create is set, destroy the file node,
|
|
207 |
+ then create and return a normal directory in its place. If
|
|
208 |
+ force_create is off, throws ResolutionException.
|
|
209 |
+ |
|
210 |
+ """
|
|
211 |
+ resolved_thing = self.resolve(c, directory)
|
|
212 |
+ |
|
213 |
+ if isinstance(resolved_thing, remote_execution_pb2.FileNode):
|
|
214 |
+ if require_traversable:
|
|
215 |
+ # We have components still to resolve, but one of the path components
|
|
216 |
+ # is a file.
|
|
217 |
+ if self.force_create:
|
|
218 |
+ directory.delete_entry(c)
|
|
219 |
+ resolved_thing = directory.descend(c, create=True)
|
|
220 |
+ else:
|
|
221 |
+ # This is a signal that we hit a file, but don't
|
|
222 |
+ # have the data to give a proper message, so the
|
|
223 |
+ # caller should reraise this with a proper
|
|
224 |
+ # description.
|
|
225 |
+ raise UnexpectedFileException()
|
|
226 |
+ |
|
227 |
+ return resolved_thing
|
|
228 |
+ |
|
229 |
+ |
|
54 | 230 |
# CasBasedDirectory intentionally doesn't call its superclass constuctor,
|
55 | 231 |
# which is meant to be unimplemented.
|
56 | 232 |
# pylint: disable=super-init-not-called
|
... | ... | @@ -168,29 +344,34 @@ class CasBasedDirectory(Directory): |
168 | 344 |
self.index[name] = IndexEntry(dirnode, buildstream_object=newdir)
|
169 | 345 |
return newdir
|
170 | 346 |
|
171 |
- def _add_new_file(self, basename, filename):
|
|
347 |
+ def _add_file(self, basename, filename, modified=False):
|
|
172 | 348 |
filenode = self.pb2_directory.files.add()
|
173 | 349 |
filenode.name = filename
|
174 | 350 |
self.cas_cache.add_object(digest=filenode.digest, path=os.path.join(basename, filename))
|
175 | 351 |
is_executable = os.access(os.path.join(basename, filename), os.X_OK)
|
176 | 352 |
filenode.is_executable = is_executable
|
177 |
- self.index[filename] = IndexEntry(filenode, modified=(filename in self.index))
|
|
353 |
+ self.index[filename] = IndexEntry(filenode, modified=modified or filename in self.index)
|
|
178 | 354 |
|
179 |
- def _add_new_link(self, basename, filename):
|
|
180 |
- existing_link = self._find_pb2_entry(filename)
|
|
355 |
+ def _copy_link_from_filesystem(self, basename, filename):
|
|
356 |
+ self._add_new_link_direct(filename, os.readlink(os.path.join(basename, filename)))
|
|
357 |
+ |
|
358 |
+ def _add_new_link_direct(self, name, target):
|
|
359 |
+ existing_link = self._find_pb2_entry(name)
|
|
181 | 360 |
if existing_link:
|
182 | 361 |
symlinknode = existing_link
|
183 | 362 |
else:
|
184 | 363 |
symlinknode = self.pb2_directory.symlinks.add()
|
185 |
- symlinknode.name = filename
|
|
364 |
+ assert isinstance(symlinknode, remote_execution_pb2.SymlinkNode)
|
|
365 |
+ symlinknode.name = name
|
|
186 | 366 |
# A symlink node has no digest.
|
187 |
- symlinknode.target = os.readlink(os.path.join(basename, filename))
|
|
188 |
- self.index[filename] = IndexEntry(symlinknode, modified=(existing_link is not None))
|
|
367 |
+ symlinknode.target = target
|
|
368 |
+ self.index[name] = IndexEntry(symlinknode, modified=(existing_link is not None))
|
|
189 | 369 |
|
190 | 370 |
def delete_entry(self, name):
|
191 | 371 |
for collection in [self.pb2_directory.files, self.pb2_directory.symlinks, self.pb2_directory.directories]:
|
192 |
- if name in collection:
|
|
193 |
- collection.remove(name)
|
|
372 |
+ for thing in collection:
|
|
373 |
+ if thing.name == name:
|
|
374 |
+ collection.remove(thing)
|
|
194 | 375 |
if name in self.index:
|
195 | 376 |
del self.index[name]
|
196 | 377 |
|
... | ... | @@ -231,9 +412,13 @@ class CasBasedDirectory(Directory): |
231 | 412 |
if isinstance(entry, CasBasedDirectory):
|
232 | 413 |
return entry.descend(subdirectory_spec[1:], create)
|
233 | 414 |
else:
|
415 |
+ # May be a symlink
|
|
416 |
+ target = self._resolve(subdirectory_spec[0], force_create=create)
|
|
417 |
+ if isinstance(target, CasBasedDirectory):
|
|
418 |
+ return target
|
|
234 | 419 |
error = "Cannot descend into {}, which is a '{}' in the directory {}"
|
235 | 420 |
raise VirtualDirectoryError(error.format(subdirectory_spec[0],
|
236 |
- type(entry).__name__,
|
|
421 |
+ type(self.index[subdirectory_spec[0]].pb_object).__name__,
|
|
237 | 422 |
self))
|
238 | 423 |
else:
|
239 | 424 |
if create:
|
... | ... | @@ -254,36 +439,9 @@ class CasBasedDirectory(Directory): |
254 | 439 |
else:
|
255 | 440 |
return self
|
256 | 441 |
|
257 |
- def _resolve_symlink_or_directory(self, name):
|
|
258 |
- """Used only by _import_files_from_directory. Tries to resolve a
|
|
259 |
- directory name or symlink name. 'name' must be an entry in this
|
|
260 |
- directory. It must be a single symlink or directory name, not a path
|
|
261 |
- separated by path separators. If it's an existing directory name, it
|
|
262 |
- just returns the Directory object for that. If it's a symlink, it will
|
|
263 |
- attempt to find the target of the symlink and return that as a
|
|
264 |
- Directory object.
|
|
265 |
- |
|
266 |
- If a symlink target doesn't exist, it will attempt to create it
|
|
267 |
- as a directory as long as it's within this directory tree.
|
|
268 |
- """
|
|
269 |
- |
|
270 |
- if isinstance(self.index[name].buildstream_object, Directory):
|
|
271 |
- return self.index[name].buildstream_object
|
|
272 |
- # OK then, it's a symlink
|
|
273 |
- symlink = self._find_pb2_entry(name)
|
|
274 |
- absolute = symlink.target.startswith(CasBasedDirectory._pb2_absolute_path_prefix)
|
|
275 |
- if absolute:
|
|
276 |
- root = self.find_root()
|
|
277 |
- else:
|
|
278 |
- root = self
|
|
279 |
- directory = root
|
|
280 |
- components = symlink.target.split(CasBasedDirectory._pb2_path_sep)
|
|
281 |
- for c in components:
|
|
282 |
- if c == "..":
|
|
283 |
- directory = directory.parent
|
|
284 |
- else:
|
|
285 |
- directory = directory.descend(c, create=True)
|
|
286 |
- return directory
|
|
442 |
+ def _resolve(self, name, absolute_symlinks_resolve=True, force_create=False):
|
|
443 |
+ resolver = _Resolver(absolute_symlinks_resolve, force_create)
|
|
444 |
+ return resolver.resolve(name, self)
|
|
287 | 445 |
|
288 | 446 |
def _check_replacement(self, name, path_prefix, fileListResult):
|
289 | 447 |
""" Checks whether 'name' exists, and if so, whether we can overwrite it.
|
... | ... | @@ -297,6 +455,7 @@ class CasBasedDirectory(Directory): |
297 | 455 |
return True
|
298 | 456 |
if (isinstance(existing_entry,
|
299 | 457 |
(remote_execution_pb2.FileNode, remote_execution_pb2.SymlinkNode))):
|
458 |
+ self.delete_entry(name)
|
|
300 | 459 |
fileListResult.overwritten.append(relative_pathname)
|
301 | 460 |
return True
|
302 | 461 |
elif isinstance(existing_entry, remote_execution_pb2.DirectoryNode):
|
... | ... | @@ -314,23 +473,44 @@ class CasBasedDirectory(Directory): |
314 | 473 |
.format(name, type(existing_entry)))
|
315 | 474 |
return False # In case asserts are disabled
|
316 | 475 |
|
317 |
- def _import_directory_recursively(self, directory_name, source_directory, remaining_path, path_prefix):
|
|
318 |
- """ _import_directory_recursively and _import_files_from_directory will be called alternately
|
|
319 |
- as a directory tree is descended. """
|
|
320 |
- if directory_name in self.index:
|
|
321 |
- subdir = self._resolve_symlink_or_directory(directory_name)
|
|
322 |
- else:
|
|
323 |
- subdir = self._add_directory(directory_name)
|
|
324 |
- new_path_prefix = os.path.join(path_prefix, directory_name)
|
|
325 |
- subdir_result = subdir._import_files_from_directory(os.path.join(source_directory, directory_name),
|
|
326 |
- [os.path.sep.join(remaining_path)],
|
|
327 |
- path_prefix=new_path_prefix)
|
|
328 |
- return subdir_result
|
|
476 |
+ def _replace_anything_with_dir(self, name, path_prefix, overwritten_files_list):
|
|
477 |
+ self.delete_entry(name)
|
|
478 |
+ subdir = self._add_directory(name)
|
|
479 |
+ overwritten_files_list.append(os.path.join(path_prefix, name))
|
|
480 |
+ return subdir
|
|
329 | 481 |
|
330 | 482 |
def _import_files_from_directory(self, source_directory, files, path_prefix=""):
|
331 |
- """ Imports files from a traditional directory """
|
|
483 |
+ """ Imports files from a traditional directory. """
|
|
484 |
+ |
|
485 |
+ def _ensure_followable(name, path_prefix):
|
|
486 |
+ """ Makes sure 'name' is a directory or symlink to a directory which can be descended into. """
|
|
487 |
+ if isinstance(self.index[name].buildstream_object, Directory):
|
|
488 |
+ return self.descend(name)
|
|
489 |
+ try:
|
|
490 |
+ target = self._resolve(name, force_create=True)
|
|
491 |
+ except InfiniteSymlinkException:
|
|
492 |
+ return self._replace_anything_with_dir(name, path_prefix, result.overwritten)
|
|
493 |
+ if isinstance(target, CasBasedDirectory):
|
|
494 |
+ return target
|
|
495 |
+ elif isinstance(target, remote_execution_pb2.FileNode):
|
|
496 |
+ return self._replace_anything_with_dir(name, path_prefix, result.overwritten)
|
|
497 |
+ return target
|
|
498 |
+ |
|
499 |
+ def _import_directory_recursively(directory_name, source_directory, remaining_path, path_prefix):
|
|
500 |
+ """ _import_directory_recursively and _import_files_from_directory will be called alternately
|
|
501 |
+ as a directory tree is descended. """
|
|
502 |
+ if directory_name in self.index:
|
|
503 |
+ subdir = _ensure_followable(directory_name, path_prefix)
|
|
504 |
+ else:
|
|
505 |
+ subdir = self._add_directory(directory_name)
|
|
506 |
+ new_path_prefix = os.path.join(path_prefix, directory_name)
|
|
507 |
+ subdir_result = subdir._import_files_from_directory(os.path.join(source_directory, directory_name),
|
|
508 |
+ [os.path.sep.join(remaining_path)],
|
|
509 |
+ path_prefix=new_path_prefix)
|
|
510 |
+ return subdir_result
|
|
511 |
+ |
|
332 | 512 |
result = FileListResult()
|
333 |
- for entry in sorted(files):
|
|
513 |
+ for entry in files:
|
|
334 | 514 |
split_path = entry.split(os.path.sep)
|
335 | 515 |
# The actual file on the FS we're importing
|
336 | 516 |
import_file = os.path.join(source_directory, entry)
|
... | ... | @@ -338,14 +518,18 @@ class CasBasedDirectory(Directory): |
338 | 518 |
relative_pathname = os.path.join(path_prefix, entry)
|
339 | 519 |
if len(split_path) > 1:
|
340 | 520 |
directory_name = split_path[0]
|
341 |
- # Hand this off to the importer for that subdir. This will only do one file -
|
|
342 |
- # a better way would be to hand off all the files in this subdir at once.
|
|
343 |
- subdir_result = self._import_directory_recursively(directory_name, source_directory,
|
|
344 |
- split_path[1:], path_prefix)
|
|
521 |
+ # Hand this off to the importer for that subdir.
|
|
522 |
+ |
|
523 |
+ # It would be advantageous to batch these together by
|
|
524 |
+ # directory_name. However, we can't do it out of
|
|
525 |
+ # order, since importing symlinks affects the results
|
|
526 |
+ # of other imports.
|
|
527 |
+ subdir_result = _import_directory_recursively(directory_name, source_directory,
|
|
528 |
+ split_path[1:], path_prefix)
|
|
345 | 529 |
result.combine(subdir_result)
|
346 | 530 |
elif os.path.islink(import_file):
|
347 | 531 |
if self._check_replacement(entry, path_prefix, result):
|
348 |
- self._add_new_link(source_directory, entry)
|
|
532 |
+ self._copy_link_from_filesystem(source_directory, entry)
|
|
349 | 533 |
result.files_written.append(relative_pathname)
|
350 | 534 |
elif os.path.isdir(import_file):
|
351 | 535 |
# A plain directory which already exists isn't a problem; just ignore it.
|
... | ... | @@ -353,10 +537,78 @@ class CasBasedDirectory(Directory): |
353 | 537 |
self._add_directory(entry)
|
354 | 538 |
elif os.path.isfile(import_file):
|
355 | 539 |
if self._check_replacement(entry, path_prefix, result):
|
356 |
- self._add_new_file(source_directory, entry)
|
|
540 |
+ self._add_file(source_directory, entry, modified=relative_pathname in result.overwritten)
|
|
357 | 541 |
result.files_written.append(relative_pathname)
|
358 | 542 |
return result
|
359 | 543 |
|
544 |
+ @staticmethod
|
|
545 |
+ def _files_in_subdir(sorted_files, dirname):
|
|
546 |
+ """Filters sorted_files and returns only the ones which have
|
|
547 |
+ 'dirname' as a prefix, with that prefix removed.
|
|
548 |
+ |
|
549 |
+ """
|
|
550 |
+ if not dirname.endswith(os.path.sep):
|
|
551 |
+ dirname += os.path.sep
|
|
552 |
+ return [f[len(dirname):] for f in sorted_files if f.startswith(dirname)]
|
|
553 |
+ |
|
554 |
+ def _partial_import_cas_into_cas(self, source_directory, files, path_prefix="", file_list_required=True):
|
|
555 |
+ """ Import only the files and symlinks listed in 'files' from source_directory to this one.
|
|
556 |
+ Args:
|
|
557 |
+ source_directory (:class:`.CasBasedDirectory`): The directory to import from
|
|
558 |
+ files ([str]): List of pathnames to import. Must be a list, not a generator.
|
|
559 |
+ path_prefix (str): Prefix used to add entries to the file list result.
|
|
560 |
+ file_list_required: Whether to update the file list while processing.
|
|
561 |
+ """
|
|
562 |
+ result = FileListResult()
|
|
563 |
+ processed_directories = set()
|
|
564 |
+ for f in files:
|
|
565 |
+ fullname = os.path.join(path_prefix, f)
|
|
566 |
+ components = f.split(os.path.sep)
|
|
567 |
+ if len(components) > 1:
|
|
568 |
+ # We are importing a thing which is in a subdirectory. We may have already seen this dirname
|
|
569 |
+ # for a previous file.
|
|
570 |
+ dirname = components[0]
|
|
571 |
+ if dirname not in processed_directories:
|
|
572 |
+ # Now strip off the first directory name and import files recursively.
|
|
573 |
+ subcomponents = CasBasedDirectory._files_in_subdir(files, dirname)
|
|
574 |
+ # We will fail at this point if there is a file or symlink to file called 'dirname'.
|
|
575 |
+ if dirname in self.index:
|
|
576 |
+ resolved_component = self._resolve(dirname, force_create=True)
|
|
577 |
+ if isinstance(resolved_component, remote_execution_pb2.FileNode):
|
|
578 |
+ dest_subdir = self._replace_anything_with_dir(dirname, path_prefix, result.overwritten)
|
|
579 |
+ else:
|
|
580 |
+ dest_subdir = resolved_component
|
|
581 |
+ else:
|
|
582 |
+ dest_subdir = self.descend(dirname, create=True)
|
|
583 |
+ src_subdir = source_directory.descend(dirname)
|
|
584 |
+ import_result = dest_subdir._partial_import_cas_into_cas(src_subdir, subcomponents,
|
|
585 |
+ path_prefix=fullname,
|
|
586 |
+ file_list_required=file_list_required)
|
|
587 |
+ result.combine(import_result)
|
|
588 |
+ processed_directories.add(dirname)
|
|
589 |
+ elif isinstance(source_directory.index[f].buildstream_object, CasBasedDirectory):
|
|
590 |
+ # The thing in the input file list is a directory on
|
|
591 |
+ # its own. We don't need to do anything other than create it if it doesn't exist.
|
|
592 |
+ # If we already have an entry with the same name that isn't a directory, that
|
|
593 |
+ # will be dealt with when importing files in this directory.
|
|
594 |
+ if f not in self.index:
|
|
595 |
+ self.descend(f, create=True)
|
|
596 |
+ else:
|
|
597 |
+ # We're importing a file or symlink - replace anything with the same name.
|
|
598 |
+ importable = self._check_replacement(f, path_prefix, result)
|
|
599 |
+ if importable:
|
|
600 |
+ item = source_directory.index[f].pb_object
|
|
601 |
+ if isinstance(item, remote_execution_pb2.FileNode):
|
|
602 |
+ filenode = self.pb2_directory.files.add(digest=item.digest, name=f,
|
|
603 |
+ is_executable=item.is_executable)
|
|
604 |
+ self.index[f] = IndexEntry(filenode, modified=True)
|
|
605 |
+ else:
|
|
606 |
+ assert isinstance(item, remote_execution_pb2.SymlinkNode)
|
|
607 |
+ self._add_new_link_direct(name=f, target=item.target)
|
|
608 |
+ else:
|
|
609 |
+ result.ignored.append(os.path.join(path_prefix, f))
|
|
610 |
+ return result
|
|
611 |
+ |
|
360 | 612 |
def import_files(self, external_pathspec, *, files=None,
|
361 | 613 |
report_written=True, update_utimes=False,
|
362 | 614 |
can_link=False):
|
... | ... | @@ -378,28 +630,27 @@ class CasBasedDirectory(Directory): |
378 | 630 |
|
379 | 631 |
can_link (bool): Ignored, since hard links do not have any meaning within CAS.
|
380 | 632 |
"""
|
381 |
- if isinstance(external_pathspec, FileBasedDirectory):
|
|
382 |
- source_directory = external_pathspec._get_underlying_directory()
|
|
383 |
- elif isinstance(external_pathspec, CasBasedDirectory):
|
|
384 |
- # TODO: This transfers from one CAS to another via the
|
|
385 |
- # filesystem, which is very inefficient. Alter this so it
|
|
386 |
- # transfers refs across directly.
|
|
387 |
- with tempfile.TemporaryDirectory(prefix="roundtrip") as tmpdir:
|
|
388 |
- external_pathspec.export_files(tmpdir)
|
|
389 |
- if files is None:
|
|
390 |
- files = list_relative_paths(tmpdir)
|
|
391 |
- result = self._import_files_from_directory(tmpdir, files=files)
|
|
392 |
- return result
|
|
393 |
- else:
|
|
394 |
- source_directory = external_pathspec
|
|
395 | 633 |
|
396 | 634 |
if files is None:
|
397 |
- files = list_relative_paths(source_directory)
|
|
635 |
+ if isinstance(external_pathspec, str):
|
|
636 |
+ files = list_relative_paths(external_pathspec)
|
|
637 |
+ else:
|
|
638 |
+ assert isinstance(external_pathspec, Directory)
|
|
639 |
+ files = external_pathspec.list_relative_paths()
|
|
640 |
+ |
|
641 |
+ if isinstance(external_pathspec, FileBasedDirectory):
|
|
642 |
+ source_directory = external_pathspec.get_underlying_directory()
|
|
643 |
+ result = self._import_files_from_directory(source_directory, files=files)
|
|
644 |
+ elif isinstance(external_pathspec, str):
|
|
645 |
+ source_directory = external_pathspec
|
|
646 |
+ result = self._import_files_from_directory(source_directory, files=files)
|
|
647 |
+ else:
|
|
648 |
+ assert isinstance(external_pathspec, CasBasedDirectory)
|
|
649 |
+ result = self._partial_import_cas_into_cas(external_pathspec, files=list(files))
|
|
398 | 650 |
|
399 | 651 |
# TODO: No notice is taken of report_written, update_utimes or can_link.
|
400 | 652 |
# Current behaviour is to fully populate the report, which is inefficient,
|
401 | 653 |
# but still correct.
|
402 |
- result = self._import_files_from_directory(source_directory, files=files)
|
|
403 | 654 |
|
404 | 655 |
# We need to recalculate and store the hashes of all directories both
|
405 | 656 |
# up and down the tree; we have changed our directory by importing files
|
... | ... | @@ -511,6 +762,28 @@ class CasBasedDirectory(Directory): |
511 | 762 |
else:
|
512 | 763 |
self._mark_directory_unmodified()
|
513 | 764 |
|
765 |
+ def _lightweight_resolve_to_index(self, path):
|
|
766 |
+ """A lightweight function for transforming paths into IndexEntry
|
|
767 |
+ objects. This does not follow symlinks.
|
|
768 |
+ |
|
769 |
+ path: The string to resolve. This should be a series of path
|
|
770 |
+ components separated by the protocol buffer path separator
|
|
771 |
+ _pb2_path_sep.
|
|
772 |
+ |
|
773 |
+ Returns: the IndexEntry found, or None if any of the path components were not present.
|
|
774 |
+ |
|
775 |
+ """
|
|
776 |
+ directory = self
|
|
777 |
+ path_components = path.split(CasBasedDirectory._pb2_path_sep)
|
|
778 |
+ for component in path_components[:-1]:
|
|
779 |
+ if component not in directory.index:
|
|
780 |
+ return None
|
|
781 |
+ if isinstance(directory.index[component].buildstream_object, CasBasedDirectory):
|
|
782 |
+ directory = directory.index[component].buildstream_object
|
|
783 |
+ else:
|
|
784 |
+ return None
|
|
785 |
+ return directory.index.get(path_components[-1], None)
|
|
786 |
+ |
|
514 | 787 |
def list_modified_paths(self):
|
515 | 788 |
"""Provide a list of relative paths which have been modified since the
|
516 | 789 |
last call to mark_unmodified.
|
... | ... | @@ -518,29 +791,43 @@ class CasBasedDirectory(Directory): |
518 | 791 |
Return value: List(str) - list of modified paths
|
519 | 792 |
"""
|
520 | 793 |
|
521 |
- filelist = []
|
|
522 |
- for (k, v) in self.index.items():
|
|
523 |
- if isinstance(v.buildstream_object, CasBasedDirectory):
|
|
524 |
- filelist.extend([k + os.path.sep + x for x in v.buildstream_object.list_modified_paths()])
|
|
525 |
- elif isinstance(v.pb_object, remote_execution_pb2.FileNode) and v.modified:
|
|
526 |
- filelist.append(k)
|
|
527 |
- return filelist
|
|
794 |
+ for p in self.list_relative_paths():
|
|
795 |
+ i = self._lightweight_resolve_to_index(p)
|
|
796 |
+ if i and i.modified:
|
|
797 |
+ yield p
|
|
528 | 798 |
|
529 |
- def list_relative_paths(self):
|
|
799 |
+ def list_relative_paths(self, relpath=""):
|
|
530 | 800 |
"""Provide a list of all relative paths.
|
531 | 801 |
|
532 |
- NOTE: This list is not in the same order as utils.list_relative_paths.
|
|
533 |
- |
|
534 | 802 |
Return value: List(str) - list of all paths
|
535 | 803 |
"""
|
536 | 804 |
|
537 |
- filelist = []
|
|
538 |
- for (k, v) in self.index.items():
|
|
539 |
- if isinstance(v.buildstream_object, CasBasedDirectory):
|
|
540 |
- filelist.extend([k + os.path.sep + x for x in v.buildstream_object.list_relative_paths()])
|
|
541 |
- elif isinstance(v.pb_object, remote_execution_pb2.FileNode):
|
|
542 |
- filelist.append(k)
|
|
543 |
- return filelist
|
|
805 |
+ symlink_list = filter(lambda i: isinstance(i[1].pb_object, remote_execution_pb2.SymlinkNode),
|
|
806 |
+ self.index.items())
|
|
807 |
+ file_list = list(filter(lambda i: isinstance(i[1].pb_object, remote_execution_pb2.FileNode),
|
|
808 |
+ self.index.items()))
|
|
809 |
+ directory_list = filter(lambda i: isinstance(i[1].buildstream_object, CasBasedDirectory),
|
|
810 |
+ self.index.items())
|
|
811 |
+ |
|
812 |
+ # We need to mimic the behaviour of os.walk, in which symlinks
|
|
813 |
+ # to directories count as directories and symlinks to file or
|
|
814 |
+ # broken symlinks count as files. os.walk doesn't follow
|
|
815 |
+ # symlinks, so we don't recurse.
|
|
816 |
+ for (k, v) in sorted(symlink_list):
|
|
817 |
+ target = self._resolve(k, absolute_symlinks_resolve=True)
|
|
818 |
+ if isinstance(target, CasBasedDirectory):
|
|
819 |
+ yield os.path.join(relpath, k)
|
|
820 |
+ else:
|
|
821 |
+ file_list.append((k, v))
|
|
822 |
+ |
|
823 |
+ if file_list == [] and relpath != "":
|
|
824 |
+ yield relpath
|
|
825 |
+ else:
|
|
826 |
+ for (k, v) in sorted(file_list):
|
|
827 |
+ yield os.path.join(relpath, k)
|
|
828 |
+ |
|
829 |
+ for (k, v) in sorted(directory_list):
|
|
830 |
+ yield from v.buildstream_object.list_relative_paths(relpath=os.path.join(relpath, k))
|
|
544 | 831 |
|
545 | 832 |
def recalculate_hash(self):
|
546 | 833 |
""" Recalcuates the hash for this directory and store the results in
|
... | ... | @@ -1199,3 +1199,17 @@ def _deduplicate(iterable, key=None): |
1199 | 1199 |
def _get_link_mtime(path):
|
1200 | 1200 |
path_stat = os.lstat(path)
|
1201 | 1201 |
return path_stat.st_mtime
|
1202 |
+ |
|
1203 |
+ |
|
1204 |
+# Returns the first directory to contain filename, or an empty string if
|
|
1205 |
+# none found
|
|
1206 |
+#
|
|
1207 |
+def _search_upward_for_file(directory, filename):
|
|
1208 |
+ directory = os.path.abspath(directory)
|
|
1209 |
+ while not os.path.isfile(os.path.join(directory, filename)):
|
|
1210 |
+ parent_dir = os.path.dirname(directory)
|
|
1211 |
+ if directory == parent_dir:
|
|
1212 |
+ return ""
|
|
1213 |
+ directory = parent_dir
|
|
1214 |
+ |
|
1215 |
+ return directory
|
1 |
+# Project config for frontend build test
|
|
2 |
+name: test
|
|
3 |
+ |
|
4 |
+elephant-path: elements
|
... | ... | @@ -36,6 +36,19 @@ def test_show(cli, datafiles, target, format, expected): |
36 | 36 |
.format(expected, result.output))
|
37 | 37 |
|
38 | 38 |
|
39 |
+@pytest.mark.datafiles(os.path.join(
|
|
40 |
+ os.path.dirname(os.path.realpath(__file__)),
|
|
41 |
+ "invalid_element_path",
|
|
42 |
+))
|
|
43 |
+def test_show_invalid_element_path(cli, datafiles):
|
|
44 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
45 |
+ result = cli.run(project=project, silent=True, args=[
|
|
46 |
+ 'show',
|
|
47 |
+ "foo.bst"])
|
|
48 |
+ |
|
49 |
+ result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
|
|
50 |
+ |
|
51 |
+ |
|
39 | 52 |
@pytest.mark.datafiles(DATA_DIR)
|
40 | 53 |
@pytest.mark.parametrize("target,except_,expected", [
|
41 | 54 |
('target.bst', 'import-bin.bst', ['import-dev.bst', 'compose-all.bst', 'target.bst']),
|
... | ... | @@ -29,6 +29,7 @@ import shutil |
29 | 29 |
import subprocess
|
30 | 30 |
from ruamel.yaml.comments import CommentedSet
|
31 | 31 |
from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
|
32 |
+from tests.testutils import create_artifact_share
|
|
32 | 33 |
|
33 | 34 |
from buildstream import _yaml
|
34 | 35 |
from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
|
... | ... | @@ -93,6 +94,13 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir |
93 | 94 |
|
94 | 95 |
result.assert_success()
|
95 | 96 |
|
97 |
+ # Assert that a .bstproject.yaml file has been created
|
|
98 |
+ # and contains the path to the project
|
|
99 |
+ bstproject_path = os.path.join(workspace_dir, '.bstproject.yaml')
|
|
100 |
+ assert os.path.exists(bstproject_path)
|
|
101 |
+ with open(bstproject_path) as f:
|
|
102 |
+ assert project_path in f.read()
|
|
103 |
+ |
|
96 | 104 |
# Assert that we are now buildable because the source is
|
97 | 105 |
# now cached.
|
98 | 106 |
assert cli.get_element_state(project_path, element_name) == 'buildable'
|
... | ... | @@ -148,6 +156,10 @@ def test_open_force(cli, tmpdir, datafiles, kind): |
148 | 156 |
# Assert the workspace dir still exists
|
149 | 157 |
assert os.path.exists(workspace)
|
150 | 158 |
|
159 |
+ # Assert the bstproject doesn't exist
|
|
160 |
+ bstproject_path = os.path.join(workspace, '.bstproject.yaml')
|
|
161 |
+ assert not os.path.exists(bstproject_path)
|
|
162 |
+ |
|
151 | 163 |
# Now open the workspace again with --force, this should happily succeed
|
152 | 164 |
result = cli.run(project=project, args=[
|
153 | 165 |
'workspace', 'open', '--force', element_name, workspace
|
... | ... | @@ -436,9 +448,12 @@ def test_list(cli, tmpdir, datafiles): |
436 | 448 |
@pytest.mark.datafiles(DATA_DIR)
|
437 | 449 |
@pytest.mark.parametrize("kind", repo_kinds)
|
438 | 450 |
@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
|
439 |
-def test_build(cli, tmpdir, datafiles, kind, strict):
|
|
451 |
+@pytest.mark.parametrize("call_from", [("project"), ("workspace")])
|
|
452 |
+def test_build(cli, tmpdir_factory, datafiles, kind, strict, call_from):
|
|
453 |
+ tmpdir = tmpdir_factory.mktemp('')
|
|
440 | 454 |
element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
|
441 | 455 |
checkout = os.path.join(str(tmpdir), 'checkout')
|
456 |
+ args_pre = ['-C', workspace] if call_from == "project" else []
|
|
442 | 457 |
|
443 | 458 |
# Modify workspace
|
444 | 459 |
shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
|
... | ... | @@ -461,15 +476,14 @@ def test_build(cli, tmpdir, datafiles, kind, strict): |
461 | 476 |
# Build modified workspace
|
462 | 477 |
assert cli.get_element_state(project, element_name) == 'buildable'
|
463 | 478 |
assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
|
464 |
- result = cli.run(project=project, args=['build', element_name])
|
|
479 |
+ result = cli.run(project=project, args=args_pre + ['build', element_name])
|
|
465 | 480 |
result.assert_success()
|
466 | 481 |
assert cli.get_element_state(project, element_name) == 'cached'
|
467 | 482 |
assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
|
468 | 483 |
|
469 | 484 |
# Checkout the result
|
470 |
- result = cli.run(project=project, args=[
|
|
471 |
- 'checkout', element_name, checkout
|
|
472 |
- ])
|
|
485 |
+ result = cli.run(project=project,
|
|
486 |
+ args=args_pre + ['checkout', element_name, checkout])
|
|
473 | 487 |
result.assert_success()
|
474 | 488 |
|
475 | 489 |
# Check that the pony.conf from the modified workspace exists
|
... | ... | @@ -876,3 +890,129 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles): |
876 | 890 |
result = cli.run(project=project, args=["build", element_name])
|
877 | 891 |
assert "BUG" not in result.stderr
|
878 | 892 |
assert cli.get_element_state(project, element_name) != "cached"
|
893 |
+ |
|
894 |
+ |
|
895 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
896 |
+def test_external_fetch(cli, datafiles, tmpdir_factory):
|
|
897 |
+ # Fetching from a workspace outside a project doesn't fail horribly
|
|
898 |
+ tmpdir = tmpdir_factory.mktemp('')
|
|
899 |
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
|
|
900 |
+ |
|
901 |
+ result = cli.run(project=project, args=['-C', workspace, 'fetch', element_name])
|
|
902 |
+ result.assert_success()
|
|
903 |
+ |
|
904 |
+ # We already fetched it by opening the workspace, but we're also checking
|
|
905 |
+ # `bst show` works here
|
|
906 |
+ assert cli.get_element_state(project, element_name) == 'buildable'
|
|
907 |
+ |
|
908 |
+ |
|
909 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
910 |
+def test_external_push_pull(cli, datafiles, tmpdir_factory):
|
|
911 |
+ # Pushing and pulling to/from an artifact cache works from an external workspace
|
|
912 |
+ tmpdir = tmpdir_factory.mktemp('')
|
|
913 |
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
|
|
914 |
+ |
|
915 |
+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
|
|
916 |
+ result = cli.run(project=project, args=['-C', workspace, 'build', element_name])
|
|
917 |
+ result.assert_success()
|
|
918 |
+ |
|
919 |
+ cli.configure({
|
|
920 |
+ 'artifacts': {'url': share.repo, 'push': True}
|
|
921 |
+ })
|
|
922 |
+ |
|
923 |
+ result = cli.run(project=project, args=['-C', workspace, 'push', element_name])
|
|
924 |
+ result.assert_success()
|
|
925 |
+ |
|
926 |
+ result = cli.run(project=project, args=['-C', workspace, 'pull', '--deps', 'all', 'target.bst'])
|
|
927 |
+ result.assert_success()
|
|
928 |
+ |
|
929 |
+ |
|
930 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
931 |
+def test_external_track(cli, datafiles, tmpdir_factory):
|
|
932 |
+ # Tracking does not get horribly confused
|
|
933 |
+ tmpdir = tmpdir_factory.mktemp('')
|
|
934 |
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", True)
|
|
935 |
+ |
|
936 |
+ # The workspace is necessarily already tracked, so we only care that
|
|
937 |
+ # there's no weird errors.
|
|
938 |
+ result = cli.run(project=project, args=['-C', workspace, 'track', element_name])
|
|
939 |
+ result.assert_success()
|
|
940 |
+ |
|
941 |
+ |
|
942 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
943 |
+def test_external_open_other(cli, datafiles, tmpdir_factory):
|
|
944 |
+ # From inside an external workspace, open another workspace
|
|
945 |
+ tmpdir1 = tmpdir_factory.mktemp('')
|
|
946 |
+ tmpdir2 = tmpdir_factory.mktemp('')
|
|
947 |
+ # Making use of the assumption that it's the same project in both invocations of open_workspace
|
|
948 |
+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
|
|
949 |
+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
|
|
950 |
+ |
|
951 |
+ # Closing the other element first, because I'm too lazy to create an
|
|
952 |
+ # element without opening it
|
|
953 |
+ result = cli.run(project=project, args=['workspace', 'close', beta_element])
|
|
954 |
+ result.assert_success()
|
|
955 |
+ |
|
956 |
+ result = cli.run(project=project, args=[
|
|
957 |
+ '-C', alpha_workspace, 'workspace', 'open', '--force', beta_element, beta_workspace
|
|
958 |
+ ])
|
|
959 |
+ result.assert_success()
|
|
960 |
+ |
|
961 |
+ |
|
962 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
963 |
+def test_external_close_other(cli, datafiles, tmpdir_factory):
|
|
964 |
+ # From inside an external workspace, close the other workspace
|
|
965 |
+ tmpdir1 = tmpdir_factory.mktemp('')
|
|
966 |
+ tmpdir2 = tmpdir_factory.mktemp('')
|
|
967 |
+ # Making use of the assumption that it's the same project in both invocations of open_workspace
|
|
968 |
+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
|
|
969 |
+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
|
|
970 |
+ |
|
971 |
+ result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element])
|
|
972 |
+ result.assert_success()
|
|
973 |
+ |
|
974 |
+ |
|
975 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
976 |
+@pytest.mark.parametrize("force", [("force"), ("no-force")])
|
|
977 |
+def test_external_close_self(cli, datafiles, tmpdir_factory, force):
|
|
978 |
+ # From inside an external workspace, close it
|
|
979 |
+ # This is unwise, so is only allowed if --force
|
|
980 |
+ tmpdir1 = tmpdir_factory.mktemp('')
|
|
981 |
+ tmpdir2 = tmpdir_factory.mktemp('')
|
|
982 |
+ # Making use of the assumption that it's the same project in both invocations of open_workspace
|
|
983 |
+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
|
|
984 |
+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
|
|
985 |
+ |
|
986 |
+ args = ['-C', alpha_workspace, 'workspace', 'close']
|
|
987 |
+ if force == "force":
|
|
988 |
+ args.append('--force')
|
|
989 |
+ args.append(alpha_element)
|
|
990 |
+ |
|
991 |
+ result = cli.run(project=project, args=args)
|
|
992 |
+ if force == "force":
|
|
993 |
+ result.assert_success()
|
|
994 |
+ else:
|
|
995 |
+ # TODO: Proper error domain and reason
|
|
996 |
+ result.assert_main_error(ErrorDomain.APP, 'closing-required-workspace')
|
|
997 |
+ |
|
998 |
+ |
|
999 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
1000 |
+def test_external_reset_other(cli, datafiles, tmpdir_factory):
|
|
1001 |
+ tmpdir1 = tmpdir_factory.mktemp('')
|
|
1002 |
+ tmpdir2 = tmpdir_factory.mktemp('')
|
|
1003 |
+ # Making use of the assumption that it's the same project in both invocations of open_workspace
|
|
1004 |
+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
|
|
1005 |
+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
|
|
1006 |
+ |
|
1007 |
+ result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'reset', beta_element])
|
|
1008 |
+ result.assert_success()
|
|
1009 |
+ |
|
1010 |
+ |
|
1011 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
1012 |
+def test_external_list(cli, datafiles, tmpdir_factory):
|
|
1013 |
+ tmpdir = tmpdir_factory.mktemp('')
|
|
1014 |
+ # Making use of the assumption that it's the same project in both invocations of open_workspace
|
|
1015 |
+ element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
|
|
1016 |
+ |
|
1017 |
+ result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
|
|
1018 |
+ result.assert_success()
|
... | ... | @@ -339,3 +339,28 @@ def test_integration_devices(cli, tmpdir, datafiles): |
339 | 339 |
|
340 | 340 |
result = execute_shell(cli, project, ["true"], element=element_name)
|
341 | 341 |
assert result.exit_code == 0
|
342 |
+ |
|
343 |
+ |
|
344 |
+# Test that a shell can be opened from an external workspace
|
|
345 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
346 |
+@pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
|
|
347 |
+def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell):
|
|
348 |
+ tmpdir = tmpdir_factory.mktemp("")
|
|
349 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
350 |
+ element_name = 'autotools/amhello.bst'
|
|
351 |
+ workspace_dir = os.path.join(str(tmpdir), 'workspace')
|
|
352 |
+ |
|
353 |
+ result = cli.run(project=project, args=[
|
|
354 |
+ 'workspace', 'open', element_name, workspace_dir
|
|
355 |
+ ])
|
|
356 |
+ result.assert_success()
|
|
357 |
+ |
|
358 |
+ result = cli.run(project=project, args=['-C', workspace_dir, 'build', element_name])
|
|
359 |
+ result.assert_success()
|
|
360 |
+ |
|
361 |
+ command = ['shell']
|
|
362 |
+ if build_shell == 'build':
|
|
363 |
+ command.append('--build')
|
|
364 |
+ command.extend([element_name, '--', 'true'])
|
|
365 |
+ result = cli.run(project=project, cwd=workspace_dir, args=command)
|
|
366 |
+ result.assert_success()
|
1 |
+from hashlib import sha256
|
|
2 |
+import os
|
|
3 |
+import pytest
|
|
4 |
+import random
|
|
5 |
+import tempfile
|
|
6 |
+from tests.testutils import cli
|
|
7 |
+ |
|
8 |
+from buildstream.storage._casbaseddirectory import CasBasedDirectory
|
|
9 |
+from buildstream.storage._filebaseddirectory import FileBasedDirectory
|
|
10 |
+from buildstream._artifactcache import ArtifactCache
|
|
11 |
+from buildstream._artifactcache.cascache import CASCache
|
|
12 |
+from buildstream import utils
|
|
13 |
+ |
|
14 |
+ |
|
15 |
+# These are comparitive tests that check that FileBasedDirectory and
|
|
16 |
+# CasBasedDirectory act identically.
|
|
17 |
+ |
|
18 |
+ |
|
19 |
+class FakeArtifactCache():
|
|
20 |
+ def __init__(self):
|
|
21 |
+ self.cas = None
|
|
22 |
+ |
|
23 |
+ |
|
24 |
+class FakeContext():
|
|
25 |
+ def __init__(self):
|
|
26 |
+ self.artifactdir = ''
|
|
27 |
+ self.artifactcache = FakeArtifactCache()
|
|
28 |
+ |
|
29 |
+ |
|
30 |
+# This is a set of example file system contents. It's a set of trees
|
|
31 |
+# which are either expected to be problematic or were found to be
|
|
32 |
+# problematic during random testing.
|
|
33 |
+ |
|
34 |
+# The test attempts to import each on top of each other to test
|
|
35 |
+# importing works consistently. Each tuple is defined as (<filename>,
|
|
36 |
+# <type>, <content>). Type can be 'F' (file), 'S' (symlink) or 'D'
|
|
37 |
+# (directory) with content being the contents for a file or the
|
|
38 |
+# destination for a symlink.
|
|
39 |
+root_filesets = [
|
|
40 |
+ [('a/b/c/textfile1', 'F', 'This is textfile 1\n')],
|
|
41 |
+ [('a/b/c/textfile1', 'F', 'This is the replacement textfile 1\n')],
|
|
42 |
+ [('a/b/d', 'D', '')],
|
|
43 |
+ [('a/b/c', 'S', '/a/b/d')],
|
|
44 |
+ [('a/b/d', 'S', '/a/b/c')],
|
|
45 |
+ [('a/b/d', 'D', ''), ('a/b/c', 'S', '/a/b/d')],
|
|
46 |
+ [('a/b/c', 'D', ''), ('a/b/d', 'S', '/a/b/c')],
|
|
47 |
+ [('a/b', 'F', 'This is textfile 1\n')],
|
|
48 |
+ [('a/b/c', 'F', 'This is textfile 1\n')],
|
|
49 |
+ [('a/b/c', 'D', '')]
|
|
50 |
+]
|
|
51 |
+ |
|
52 |
+empty_hash_ref = sha256().hexdigest()
|
|
53 |
+RANDOM_SEED = 69105
|
|
54 |
+NUM_RANDOM_TESTS = 10
|
|
55 |
+ |
|
56 |
+ |
|
57 |
+def generate_import_roots(rootno, directory):
|
|
58 |
+ rootname = "root{}".format(rootno)
|
|
59 |
+ rootdir = os.path.join(directory, "content", rootname)
|
|
60 |
+ if os.path.exists(rootdir):
|
|
61 |
+ return
|
|
62 |
+ for (path, typesymbol, content) in root_filesets[rootno - 1]:
|
|
63 |
+ if typesymbol == 'F':
|
|
64 |
+ (dirnames, filename) = os.path.split(path)
|
|
65 |
+ os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
|
|
66 |
+ with open(os.path.join(rootdir, dirnames, filename), "wt") as f:
|
|
67 |
+ f.write(content)
|
|
68 |
+ elif typesymbol == 'D':
|
|
69 |
+ os.makedirs(os.path.join(rootdir, path), exist_ok=True)
|
|
70 |
+ elif typesymbol == 'S':
|
|
71 |
+ (dirnames, filename) = os.path.split(path)
|
|
72 |
+ os.makedirs(os.path.join(rootdir, dirnames), exist_ok=True)
|
|
73 |
+ os.symlink(content, os.path.join(rootdir, path))
|
|
74 |
+ |
|
75 |
+ |
|
76 |
+def generate_random_root(rootno, directory):
|
|
77 |
+ # By seeding the random number generator, we ensure these tests
|
|
78 |
+ # will be repeatable, at least until Python changes the random
|
|
79 |
+ # number algorithm.
|
|
80 |
+ random.seed(RANDOM_SEED + rootno)
|
|
81 |
+ rootname = "root{}".format(rootno)
|
|
82 |
+ rootdir = os.path.join(directory, "content", rootname)
|
|
83 |
+ if os.path.exists(rootdir):
|
|
84 |
+ return
|
|
85 |
+ things = []
|
|
86 |
+ locations = ['.']
|
|
87 |
+ os.makedirs(rootdir)
|
|
88 |
+ for i in range(0, 100):
|
|
89 |
+ location = random.choice(locations)
|
|
90 |
+ thingname = "node{}".format(i)
|
|
91 |
+ thing = random.choice(['dir', 'link', 'file'])
|
|
92 |
+ target = os.path.join(rootdir, location, thingname)
|
|
93 |
+ if thing == 'dir':
|
|
94 |
+ os.makedirs(target)
|
|
95 |
+ locations.append(os.path.join(location, thingname))
|
|
96 |
+ elif thing == 'file':
|
|
97 |
+ with open(target, "wt") as f:
|
|
98 |
+ f.write("This is node {}\n".format(i))
|
|
99 |
+ elif thing == 'link':
|
|
100 |
+ symlink_type = random.choice(['absolute', 'relative', 'broken'])
|
|
101 |
+ if symlink_type == 'broken' or not things:
|
|
102 |
+ os.symlink("/broken", target)
|
|
103 |
+ elif symlink_type == 'absolute':
|
|
104 |
+ symlink_destination = random.choice(things)
|
|
105 |
+ os.symlink(symlink_destination, target)
|
|
106 |
+ else:
|
|
107 |
+ symlink_destination = random.choice(things)
|
|
108 |
+ relative_link = os.path.relpath(symlink_destination, start=location)
|
|
109 |
+ os.symlink(relative_link, target)
|
|
110 |
+ things.append(os.path.join(location, thingname))
|
|
111 |
+ |
|
112 |
+ |
|
113 |
+def file_contents(path):
|
|
114 |
+ with open(path, "r") as f:
|
|
115 |
+ result = f.read()
|
|
116 |
+ return result
|
|
117 |
+ |
|
118 |
+ |
|
119 |
+def file_contents_are(path, contents):
|
|
120 |
+ return file_contents(path) == contents
|
|
121 |
+ |
|
122 |
+ |
|
123 |
+def create_new_casdir(root_number, fake_context, tmpdir):
|
|
124 |
+ d = CasBasedDirectory(fake_context)
|
|
125 |
+ d.import_files(os.path.join(tmpdir, "content", "root{}".format(root_number)))
|
|
126 |
+ assert d.ref.hash != empty_hash_ref
|
|
127 |
+ return d
|
|
128 |
+ |
|
129 |
+ |
|
130 |
+def create_new_filedir(root_number, tmpdir):
|
|
131 |
+ root = os.path.join(tmpdir, "vdir")
|
|
132 |
+ os.makedirs(root)
|
|
133 |
+ d = FileBasedDirectory(root)
|
|
134 |
+ d.import_files(os.path.join(tmpdir, "content", "root{}".format(root_number)))
|
|
135 |
+ return d
|
|
136 |
+ |
|
137 |
+ |
|
138 |
+def combinations(integer_range):
|
|
139 |
+ for x in integer_range:
|
|
140 |
+ for y in integer_range:
|
|
141 |
+ yield (x, y)
|
|
142 |
+ |
|
143 |
+ |
|
144 |
+def resolve_symlinks(path, root):
|
|
145 |
+ """ A function to resolve symlinks inside 'path' components apart from the last one.
|
|
146 |
+ For example, resolve_symlinks('/a/b/c/d', '/a/b')
|
|
147 |
+ will return '/a/b/f/d' if /a/b/c is a symlink to /a/b/f. The final component of
|
|
148 |
+ 'path' is not resolved, because we typically want to inspect the symlink found
|
|
149 |
+ at that path, not its target.
|
|
150 |
+ |
|
151 |
+ """
|
|
152 |
+ components = path.split(os.path.sep)
|
|
153 |
+ location = root
|
|
154 |
+ for i in range(0, len(components) - 1):
|
|
155 |
+ location = os.path.join(location, components[i])
|
|
156 |
+ if os.path.islink(location):
|
|
157 |
+ # Resolve the link, add on all the remaining components
|
|
158 |
+ target = os.path.join(os.readlink(location))
|
|
159 |
+ tail = os.path.sep.join(components[i + 1:])
|
|
160 |
+ |
|
161 |
+ if target.startswith(os.path.sep):
|
|
162 |
+ # Absolute link - relative to root
|
|
163 |
+ location = os.path.join(root, target, tail)
|
|
164 |
+ else:
|
|
165 |
+ # Relative link - relative to symlink location
|
|
166 |
+ location = os.path.join(location, target)
|
|
167 |
+ return resolve_symlinks(location, root)
|
|
168 |
+ # If we got here, no symlinks were found. Add on the final component and return.
|
|
169 |
+ location = os.path.join(location, components[-1])
|
|
170 |
+ return location
|
|
171 |
+ |
|
172 |
+ |
|
173 |
+def directory_not_empty(path):
|
|
174 |
+ return os.listdir(path)
|
|
175 |
+ |
|
176 |
+ |
|
177 |
+def _import_test(tmpdir, original, overlay, generator_function, verify_contents=False):
|
|
178 |
+ fake_context = FakeContext()
|
|
179 |
+ fake_context.artifactcache.cas = CASCache(tmpdir)
|
|
180 |
+ # Create some fake content
|
|
181 |
+ generator_function(original, tmpdir)
|
|
182 |
+ if original != overlay:
|
|
183 |
+ generator_function(overlay, tmpdir)
|
|
184 |
+ |
|
185 |
+ d = create_new_casdir(original, fake_context, tmpdir)
|
|
186 |
+ |
|
187 |
+ duplicate_cas = create_new_casdir(original, fake_context, tmpdir)
|
|
188 |
+ |
|
189 |
+ assert duplicate_cas.ref.hash == d.ref.hash
|
|
190 |
+ |
|
191 |
+ d2 = create_new_casdir(overlay, fake_context, tmpdir)
|
|
192 |
+ d.import_files(d2)
|
|
193 |
+ export_dir = os.path.join(tmpdir, "output-{}-{}".format(original, overlay))
|
|
194 |
+ roundtrip_dir = os.path.join(tmpdir, "roundtrip-{}-{}".format(original, overlay))
|
|
195 |
+ d2.export_files(roundtrip_dir)
|
|
196 |
+ d.export_files(export_dir)
|
|
197 |
+ |
|
198 |
+ if verify_contents:
|
|
199 |
+ for item in root_filesets[overlay - 1]:
|
|
200 |
+ (path, typename, content) = item
|
|
201 |
+ realpath = resolve_symlinks(path, export_dir)
|
|
202 |
+ if typename == 'F':
|
|
203 |
+ if os.path.isdir(realpath) and directory_not_empty(realpath):
|
|
204 |
+ # The file should not have overwritten the directory in this case.
|
|
205 |
+ pass
|
|
206 |
+ else:
|
|
207 |
+ assert os.path.isfile(realpath), "{} did not exist in the combined virtual directory".format(path)
|
|
208 |
+ assert file_contents_are(realpath, content)
|
|
209 |
+ elif typename == 'S':
|
|
210 |
+ if os.path.isdir(realpath) and directory_not_empty(realpath):
|
|
211 |
+ # The symlink should not have overwritten the directory in this case.
|
|
212 |
+ pass
|
|
213 |
+ else:
|
|
214 |
+ assert os.path.islink(realpath)
|
|
215 |
+ assert os.readlink(realpath) == content
|
|
216 |
+ elif typename == 'D':
|
|
217 |
+ # We can't do any more tests than this because it
|
|
218 |
+ # depends on things present in the original. Blank
|
|
219 |
+ # directories here will be ignored and the original
|
|
220 |
+ # left in place.
|
|
221 |
+ assert os.path.lexists(realpath)
|
|
222 |
+ |
|
223 |
+ # Now do the same thing with filebaseddirectories and check the contents match
|
|
224 |
+ |
|
225 |
+ files = list(utils.list_relative_paths(roundtrip_dir))
|
|
226 |
+ duplicate_cas._import_files_from_directory(roundtrip_dir, files=files)
|
|
227 |
+ duplicate_cas._recalculate_recursing_down()
|
|
228 |
+ if duplicate_cas.parent:
|
|
229 |
+ duplicate_cas.parent._recalculate_recursing_up(duplicate_cas)
|
|
230 |
+ |
|
231 |
+ assert duplicate_cas.ref.hash == d.ref.hash
|
|
232 |
+ |
|
233 |
+ |
|
234 |
+# It's possible to parameterize on both original and overlay values,
|
|
235 |
+# but this leads to more tests being listed in the output than are
|
|
236 |
+# comfortable.
|
|
237 |
+@pytest.mark.parametrize("original", range(1, len(root_filesets) + 1))
|
|
238 |
+def test_fixed_cas_import(cli, tmpdir, original):
|
|
239 |
+ for overlay in range(1, len(root_filesets) + 1):
|
|
240 |
+ _import_test(str(tmpdir), original, overlay, generate_import_roots, verify_contents=True)
|
|
241 |
+ |
|
242 |
+ |
|
243 |
+@pytest.mark.parametrize("original", range(1, NUM_RANDOM_TESTS + 1))
|
|
244 |
+def test_random_cas_import(cli, tmpdir, original):
|
|
245 |
+ for overlay in range(1, NUM_RANDOM_TESTS + 1):
|
|
246 |
+ _import_test(str(tmpdir), original, overlay, generate_random_root, verify_contents=False)
|
|
247 |
+ |
|
248 |
+ |
|
249 |
+def _listing_test(tmpdir, root, generator_function):
|
|
250 |
+ fake_context = FakeContext()
|
|
251 |
+ fake_context.artifactcache.cas = CASCache(tmpdir)
|
|
252 |
+ # Create some fake content
|
|
253 |
+ generator_function(root, tmpdir)
|
|
254 |
+ |
|
255 |
+ d = create_new_filedir(root, tmpdir)
|
|
256 |
+ filelist = list(d.list_relative_paths())
|
|
257 |
+ |
|
258 |
+ d2 = create_new_casdir(root, fake_context, tmpdir)
|
|
259 |
+ filelist2 = list(d2.list_relative_paths())
|
|
260 |
+ |
|
261 |
+ assert filelist == filelist2
|
|
262 |
+ |
|
263 |
+ |
|
264 |
+@pytest.mark.parametrize("root", range(1, 11))
|
|
265 |
+def test_random_directory_listing(cli, tmpdir, root):
|
|
266 |
+ _listing_test(str(tmpdir), root, generate_random_root)
|
|
267 |
+ |
|
268 |
+ |
|
269 |
+@pytest.mark.parametrize("root", [1, 2, 3, 4, 5])
|
|
270 |
+def test_fixed_directory_listing(cli, tmpdir, root):
|
|
271 |
+ _listing_test(str(tmpdir), root, generate_import_roots)
|