[Notes] [Git][BuildStream/buildstream][jonathan/workspace-fragment-create] 11 commits: utils.py: Add a helper for searching upwards for files

[Jonathan Maw <gitlab mg gitlab com>]

Title: GitLab

Jonathan Maw pushed to branch jonathan/workspace-fragment-create at BuildStream / buildstream

Commits:

• 61b2635c
  by Jonathan Maw at 2018-12-06T17:04:45Z

  utils.py: Add a helper for searching upwards for files
  
  i.e. with a given directory and filename, check parent directories until
  either a directory with the filename is found, or you reach the root of
  the filesystem.
  
  This is a part of #222
  

• a047d12a
  by Jonathan Maw at 2018-12-06T17:04:45Z

  _workspaces.py: Add WorkspaceProject and WorkspaceProjectCache objects
  
  WorkspaceProject contains all the information needed to refer back to a
  project from its workspace (currently this is the project path and the
  element used to create this workspace)
  
  This is stored within a WorkspaceProjectCache object, which keeps
  WorkspaceProjects around so they don't need to be loaded from disk
  repeatedly.
  
  Workspaces has been extended to contain the WorkspaceProjectCache, and
  will use it when opening and closing workspaces.
  
  Context has been extended to contain a WorkspaceProjectCache, as there
  are times when we want to use it before a Workspaces can be initialised
  (looking up a WorkspaceProject to find the directory that the project is
  in)
  
  This is a part of #222
  

• 17f5c25d
  by Jonathan Maw at 2018-12-06T17:04:45Z

  _project.py: Rename _ensure_project_dir to _find_project_dir
  
  This is a part of #222
  

• 2eac2939
  by Jonathan Maw at 2018-12-06T17:04:45Z

  cli.py: Use utils' search upwards helper when searching for project.conf
  
  This is a part of #222
  

• c1202abf
  by Jonathan Maw at 2018-12-06T17:04:45Z

  _project.py: Find project from workspace if outside of a project
  
  This is a part of #222
  

• e868f400
  by Jonathan Maw at 2018-12-06T17:04:46Z

  _stream.py: Opened workspaces store the project that opened them
  
  This is a part of #222
  

• a64cc945
  by Jonathan Maw at 2018-12-06T17:04:46Z

  _stream.py: Re-create the .bstproject when resetting the workspace
  
  This is a part of #222
  

• b722785e
  by Jonathan Maw at 2018-12-06T17:04:46Z

  _workspaces.py: Do not include .bstproject.yaml in the cache key
  
  This is a part of #222
  

• 4446a7b2
  by Jonathan Maw at 2018-12-06T17:04:46Z

  cli: Interactively warn if the user is trying to close the workspace they're using to load the project
  
  This involves changes in:
  * _stream.py:
    * Add the helper Stream.workspace_is_required()
  * userconfig.yaml:
    * Add a default value for prompt.really-workspace-close-project-inaccessible
  * _context.py:
    * Load the prompt 'really-workspace-close-project-inaccessible' from
      user config.
  * cli.py:
    * If buildstream is invoked interactively, prompt the user to confirm
      that they want to close the workspace they're using to load this
      project.
  
  This is a part of #222
  

• 1d1602ad
  by Jonathan Maw at 2018-12-06T17:04:46Z

  tests: Test bst commands from an external workspace
  
  This is a part of #222
  

• ff8f1909
  by Jonathan Maw at 2018-12-06T17:04:46Z

  NEWS: Add an entry for being able to run commands from a workspace
  
  This is a part of #222
  

10 changed files:

• NEWS
• buildstream/_context.py
• buildstream/_frontend/cli.py
• buildstream/_project.py
• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/data/userconfig.yaml
• buildstream/utils.py
• tests/frontend/workspace.py
• tests/integration/shell.py

Changes:

NEWS

---

...	...	@@ -79,6 +79,10 @@ buildstream 1.3.1
79	79	plugin has now a tag tracking feature instead. This can be enabled
80	80	by setting 'track-tags'.
81	81
	82	+ o Opening a workspace now creates a .bstproject.yaml file that allows buildstream
	83	+ commands to be run from a workspace that is not inside a project.
	84	+
	85	+
82	86	=================
83	87	buildstream 1.1.5
84	88	=================

buildstream/_context.py

---

...	...	@@ -32,7 +32,7 @@ from ._message import Message, MessageType
32	32	from ._profile import Topics, profile_start, profile_end
33	33	from ._artifactcache import ArtifactCache
34	34	from ._artifactcache.cascache import CASCache
35		-from ._workspaces import Workspaces
	35	+from ._workspaces import Workspaces, WorkspaceProjectCache
36	36	from .plugin import _plugin_lookup
37	37
38	38
...	...	@@ -122,6 +122,10 @@ class Context():
122	122	# remove a workspace directory.
123	123	self.prompt_workspace_close_remove_dir = None
124	124
	125	+ # Boolean, whether we double-check with the user that they meant to
	126	+ # close the workspace when they're using it to access the project.
	127	+ self.prompt_workspace_close_project_inaccessible = None
	128	+
125	129	# Boolean, whether we double-check with the user that they meant to do
126	130	# a hard reset of a workspace, potentially losing changes.
127	131	self.prompt_workspace_reset_hard = None
...	...	@@ -140,6 +144,7 @@ class Context():
140	144	self._projects = []
141	145	self._project_overrides = {}
142	146	self._workspaces = None
	147	+ self._workspace_project_cache = WorkspaceProjectCache()
143	148	self._log_handle = None
144	149	self._log_filename = None
145	150	self._cascache = None
...	...	@@ -250,12 +255,15 @@ class Context():
250	255	defaults, Mapping, 'prompt')
251	256	_yaml.node_validate(prompt, [
252	257	'auto-init', 'really-workspace-close-remove-dir',
	258	+ 'really-workspace-close-project-inaccessible',
253	259	'really-workspace-reset-hard',
254	260	])
255	261	self.prompt_auto_init = _node_get_option_str(
256	262	prompt, 'auto-init', ['ask', 'no']) == 'ask'
257	263	self.prompt_workspace_close_remove_dir = _node_get_option_str(
258	264	prompt, 'really-workspace-close-remove-dir', ['ask', 'yes']) == 'ask'
	265	+ self.prompt_workspace_close_project_inaccessible = _node_get_option_str(
	266	+ prompt, 'really-workspace-close-project-inaccessible', ['ask', 'yes']) == 'ask'
259	267	self.prompt_workspace_reset_hard = _node_get_option_str(
260	268	prompt, 'really-workspace-reset-hard', ['ask', 'yes']) == 'ask'
261	269
...	...	@@ -285,7 +293,7 @@ class Context():
285	293	#
286	294	def add_project(self, project):
287	295	if not self._projects:
288		- self._workspaces = Workspaces(project)
	296	+ self._workspaces = Workspaces(project, self._workspace_project_cache)
289	297	self._projects.append(project)
290	298
291	299	# get_projects():
...	...	@@ -312,6 +320,16 @@ class Context():
312	320	def get_workspaces(self):
313	321	return self._workspaces
314	322
	323	+ # get_workspace_project_cache():
	324	+ #
	325	+ # Return the WorkspaceProjectCache object used for this BuildStream invocation
	326	+ #
	327	+ # Returns:
	328	+ # (WorkspaceProjectCache): The WorkspaceProjectCache object
	329	+ #
	330	+ def get_workspace_project_cache(self):
	331	+ return self._workspace_project_cache
	332	+
315	333	# get_overrides():
316	334	#
317	335	# Fetch the override dictionary for the active project. This returns

buildstream/_frontend/cli.py

---

...	...	@@ -59,18 +59,9 @@ def complete_target(args, incomplete):
59	59	:return: all the possible user-specified completions for the param
60	60	"""
61	61
	62	+ from .. import utils
62	63	project_conf = 'project.conf'
63	64
64		- def ensure_project_dir(directory):
65		- directory = os.path.abspath(directory)
66		- while not os.path.isfile(os.path.join(directory, project_conf)):
67		- parent_dir = os.path.dirname(directory)
68		- if directory == parent_dir:
69		- break
70		- directory = parent_dir
71		-
72		- return directory
73		-
74	65	# First resolve the directory, in case there is an
75	66	# active --directory/-C option
76	67	#
...	...	@@ -89,7 +80,7 @@ def complete_target(args, incomplete):
89	80	else:
90	81	# Check if this directory or any of its parent directories
91	82	# contain a project config file
92		- base_directory = ensure_project_dir(base_directory)
	83	+ base_directory = utils._search_upward_for_file(base_directory, project_conf)
93	84
94	85	# Now parse the project.conf just to find the element path,
95	86	# this is unfortunately a bit heavy.
...	...	@@ -772,11 +763,18 @@ def workspace_close(app, remove_dir, all_, elements):
772	763
773	764	elements = app.stream.redirect_element_names(elements)
774	765
775		- # Check that the workspaces in question exist
	766	+ # Check that the workspaces in question exist, and that it's safe to
	767	+ # remove them.
776	768	nonexisting = []
777	769	for element_name in elements:
778	770	if not app.stream.workspace_exists(element_name):
779	771	nonexisting.append(element_name)
	772	+ if (app.stream.workspace_is_required(element_name) and app.interactive and
	773	+ app.context.prompt_workspace_close_project_inaccessible):
	774	+ click.echo("Removing '{}' will prevent you from running buildstream commands".format(element_name))
	775	+ if not click.confirm('Are you sure you want to close this workspace?'):
	776	+ click.echo('Aborting', err=True)
	777	+ sys.exit(-1)
780	778	if nonexisting:
781	779	raise AppError("Workspace does not exist", detail="\n".join(nonexisting))
782	780

buildstream/_project.py

---

...	...	@@ -95,8 +95,10 @@ class Project():
95	95	# The project name
96	96	self.name = None
97	97
98		- # The project directory
99		- self.directory = self._ensure_project_dir(directory)
	98	+ self._context = context # The invocation Context, a private member
	99	+
	100	+ # The project directory, and whether the project was found from an external workspace
	101	+ self.directory, self._required_workspace_element = self._find_project_dir(directory)
100	102
101	103	# Absolute path to where elements are loaded from within the project
102	104	self.element_path = None
...	...	@@ -117,7 +119,6 @@ class Project():
117	119	#
118	120	# Private Members
119	121	#
120		- self._context = context # The invocation Context
121	122
122	123	self._default_mirror = default_mirror # The name of the preferred mirror.
123	124
...	...	@@ -371,6 +372,14 @@ class Project():
371	372
372	373	self._load_second_pass()
373	374
	375	+ # required_workspace_element()
	376	+ #
	377	+ # Returns the element whose workspace is required to load this project,
	378	+ # if any.
	379	+ #
	380	+ def required_workspace_element(self):
	381	+ return self._required_workspace_element
	382	+
374	383	# cleanup()
375	384	#
376	385	# Cleans up resources used loading elements
...	...	@@ -650,7 +659,7 @@ class Project():
650	659	# Source url aliases
651	660	output._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={})
652	661
653		- # _ensure_project_dir()
	662	+ # _find_project_dir()
654	663	#
655	664	# Returns path of the project directory, if a configuration file is found
656	665	# in given directory or any of its parent directories.
...	...	@@ -661,18 +670,26 @@ class Project():
661	670	# Raises:
662	671	# LoadError if project.conf is not found
663	672	#
664		- def _ensure_project_dir(self, directory):
665		- directory = os.path.abspath(directory)
666		- while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)):
667		- parent_dir = os.path.dirname(directory)
668		- if directory == parent_dir:
	673	+ # Returns:
	674	+ # (str) - the directory that contains the project, and
	675	+ # (str) - the name of the element required to find the project, or an empty string
	676	+ #
	677	+ def _find_project_dir(self, directory):
	678	+ workspace_element = ""
	679	+ project_directory = utils._search_upward_for_file(directory, _PROJECT_CONF_FILE)
	680	+ if not project_directory:
	681	+ workspace_project_cache = self._context.get_workspace_project_cache()
	682	+ workspace_project = workspace_project_cache.get(directory)
	683	+ if workspace_project:
	684	+ project_directory = workspace_project.get_default_project_path()
	685	+ workspace_element = workspace_project.get_default_element()
	686	+ else:
669	687	raise LoadError(
670	688	LoadErrorReason.MISSING_PROJECT_CONF,
671	689	'{} not found in current directory or any of its parent directories'
672	690	.format(_PROJECT_CONF_FILE))
673		- directory = parent_dir
674	691
675		- return directory
	692	+ return project_directory, workspace_element
676	693
677	694	def _load_plugin_factories(self, config, output):
678	695	plugin_source_origins = [] # Origins of custom sources

buildstream/_stream.py

---

...	...	@@ -581,15 +581,7 @@ class Stream():
581	581	todo_elements = "\nDid not try to create workspaces for " + todo_elements
582	582	raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
583	583
584		- workspaces.create_workspace(target._get_full_name(), directory)
585		-
586		- if not no_checkout:
587		- with target.timed_activity("Staging sources to {}".format(directory)):
588		- target._open_workspace()
589		-
590		- # Saving the workspace once it is set up means that if the next workspace fails to be created before
591		- # the configuration gets saved. The successfully created workspace still gets saved.
592		- workspaces.save_config()
	584	+ workspaces.create_workspace(target, directory, not no_checkout)
593	585	self._message(MessageType.INFO, "Created a workspace for element: {}"
594	586	.format(target._get_full_name()))
595	587
...	...	@@ -672,10 +664,7 @@ class Stream():
672	664	.format(workspace_path, e)) from e
673	665
674	666	workspaces.delete_workspace(element._get_full_name())
675		- workspaces.create_workspace(element._get_full_name(), workspace_path)
676		-
677		- with element.timed_activity("Staging sources to {}".format(workspace_path)):
678		- element._open_workspace()
	667	+ workspaces.create_workspace(element, workspace_path, True)
679	668
680	669	self._message(MessageType.INFO,
681	670	"Reset workspace for {} at: {}".format(element.name,
...	...	@@ -707,6 +696,20 @@ class Stream():
707	696
708	697	return False
709	698
	699	+ # workspace_is_required()
	700	+ #
	701	+ # Checks whether the workspace belonging to element_name is required to
	702	+ # load the project
	703	+ #
	704	+ # Args:
	705	+ # element_name (str): The element whose workspace may be required
	706	+ #
	707	+ # Returns:
	708	+ # (bool): True if the workspace is required
	709	+ def workspace_is_required(self, element_name):
	710	+ required_elm = self._project.required_workspace_element()
	711	+ return required_elm == element_name
	712	+
710	713	# workspace_list
711	714	#
712	715	# Serializes the workspaces and dumps them in YAML to stdout.

buildstream/_workspaces.py

---

...	...	@@ -25,6 +25,219 @@ from ._exceptions import LoadError, LoadErrorReason
25	25
26	26
27	27	BST_WORKSPACE_FORMAT_VERSION = 3
	28	+BST_WORKSPACE_PROJECT_FORMAT_VERSION = 1
	29	+WORKSPACE_PROJECT_FILE = ".bstproject.yaml"
	30	+
	31	+
	32	+# WorkspaceProject()
	33	+#
	34	+# An object to contain various helper functions and data required for
	35	+# referring from a workspace back to buildstream.
	36	+#
	37	+# Args:
	38	+# directory (str): The directory that the workspace exists in.
	39	+#
	40	+class WorkspaceProject():
	41	+ def __init__(self, directory):
	42	+ self._projects = []
	43	+ self._directory = directory
	44	+
	45	+ # get_default_project_path()
	46	+ #
	47	+ # Retrieves the default path to a project.
	48	+ #
	49	+ # Returns:
	50	+ # (str): The path to a project
	51	+ #
	52	+ def get_default_project_path(self):
	53	+ return self._projects[0]['project-path']
	54	+
	55	+ # get_default_element()
	56	+ #
	57	+ # Retrieves the name of the element that owns this workspace.
	58	+ #
	59	+ # Returns:
	60	+ # (str): The name of an element
	61	+ #
	62	+ def get_default_element(self):
	63	+ return self._projects[0]['element-name']
	64	+
	65	+ # to_dict()
	66	+ #
	67	+ # Turn the members data into a dict for serialization purposes
	68	+ #
	69	+ # Returns:
	70	+ # (dict): A dict representation of the WorkspaceProject
	71	+ #
	72	+ def to_dict(self):
	73	+ ret = {
	74	+ 'projects': self._projects,
	75	+ 'format-version': BST_WORKSPACE_PROJECT_FORMAT_VERSION,
	76	+ }
	77	+ return ret
	78	+
	79	+ # from_dict()
	80	+ #
	81	+ # Loads a new WorkspaceProject from a simple dictionary
	82	+ #
	83	+ # Args:
	84	+ # directory (str): The directory that the workspace exists in
	85	+ # dictionary (dict): The dict to generate a WorkspaceProject from
	86	+ #
	87	+ # Returns:
	88	+ # (WorkspaceProject): A newly instantiated WorkspaceProject
	89	+ #
	90	+ @classmethod
	91	+ def from_dict(cls, directory, dictionary):
	92	+ # Only know how to handle one format-version at the moment.
	93	+ format_version = int(dictionary['format-version'])
	94	+ assert format_version == BST_WORKSPACE_PROJECT_FORMAT_VERSION, \
	95	+ "Format version {} not found in {}".format(BST_WORKSPACE_PROJECT_FORMAT_VERSION, dictionary)
	96	+
	97	+ workspace_project = cls(directory)
	98	+ for item in dictionary['projects']:
	99	+ workspace_project.add_project(item['project-path'], item['element-name'])
	100	+
	101	+ return workspace_project
	102	+
	103	+ # load()
	104	+ #
	105	+ # Loads the WorkspaceProject for a given directory. This directory may be a
	106	+ # subdirectory of the workspace's directory.
	107	+ #
	108	+ # Args:
	109	+ # directory (str): The directory
	110	+ # Returns:
	111	+ # (WorkspaceProject): The created WorkspaceProject, if in a workspace, or
	112	+ # (NoneType): None, if the directory is not inside a workspace.
	113	+ #
	114	+ @classmethod
	115	+ def load(cls, directory):
	116	+ project_dir = cls.search_for_dir(directory)
	117	+ if project_dir:
	118	+ workspace_file = os.path.join(project_dir, WORKSPACE_PROJECT_FILE)
	119	+ data_dict = _yaml.load(workspace_file)
	120	+ return cls.from_dict(project_dir, data_dict)
	121	+ else:
	122	+ return None
	123	+
	124	+ # write()
	125	+ #
	126	+ # Writes the WorkspaceProject to disk
	127	+ #
	128	+ def write(self):
	129	+ os.makedirs(self._directory, exist_ok=True)
	130	+ _yaml.dump(self.to_dict(), self.get_filename())
	131	+
	132	+ # search_for_dir()
	133	+ #
	134	+ # Returns the directory that contains the workspace local project file,
	135	+ # searching upwards from search_dir.
	136	+ #
	137	+ @staticmethod
	138	+ def search_for_dir(search_dir):
	139	+ return utils._search_upward_for_file(search_dir, WORKSPACE_PROJECT_FILE)
	140	+
	141	+ # get_filename()
	142	+ #
	143	+ # Returns the full path to the workspace local project file
	144	+ #
	145	+ def get_filename(self):
	146	+ return os.path.join(self._directory, WORKSPACE_PROJECT_FILE)
	147	+
	148	+ # add_project()
	149	+ #
	150	+ # Adds an entry containing the project's path and element's name.
	151	+ #
	152	+ # Args:
	153	+ # project_path (str): The path to the project that opened the workspace.
	154	+ # element_name (str): The name of the element that the workspace belongs to.
	155	+ #
	156	+ def add_project(self, project_path, element_name):
	157	+ assert (project_path and element_name)
	158	+ self._projects.append({'project-path': project_path, 'element-name': element_name})
	159	+
	160	+
	161	+# WorkspaceProjectCache()
	162	+#
	163	+# A class to manage workspace project data for multiple workspaces.
	164	+#
	165	+class WorkspaceProjectCache():
	166	+ def __init__(self):
	167	+ self._projects = {} # Mapping of a workspace directory to its WorkspaceProject
	168	+
	169	+ # get()
	170	+ #
	171	+ # Returns a WorkspaceProject for a given directory, retrieving from the cache if
	172	+ # present, and searching the filesystem for the file and loading it if not.
	173	+ #
	174	+ # Args:
	175	+ # directory (str): The directory to search for a WorkspaceProject.
	176	+ #
	177	+ # Returns:
	178	+ # (WorkspaceProject): The WorkspaceProject that was found for that directory.
	179	+ # or (NoneType): None, if no WorkspaceProject can be found.
	180	+ #
	181	+ def get(self, directory):
	182	+ try:
	183	+ workspace_project = self._projects[directory]
	184	+ except KeyError:
	185	+ found_dir = WorkspaceProject.search_for_dir(directory)
	186	+ if found_dir:
	187	+ try:
	188	+ workspace_project = self._projects[found_dir]
	189	+ except KeyError:
	190	+ workspace_project = WorkspaceProject.load(found_dir)
	191	+ self._projects[found_dir] = workspace_project
	192	+ else:
	193	+ workspace_project = None
	194	+
	195	+ return workspace_project
	196	+
	197	+ # add()
	198	+ #
	199	+ # Adds the project path and element name to the WorkspaceProject that exists
	200	+ # for that directory
	201	+ #
	202	+ # Args:
	203	+ # directory (str): The directory to search for a WorkspaceProject.
	204	+ # project_path (str): The path to the project that refers to this workspace
	205	+ # element_name (str): The element in the project that was refers to this workspace
	206	+ #
	207	+ # Returns:
	208	+ # (WorkspaceProject): The WorkspaceProject that was found for that directory.
	209	+ #
	210	+ def add(self, directory, project_path, element_name):
	211	+ workspace_project = self.get(directory)
	212	+ if not workspace_project:
	213	+ workspace_project = WorkspaceProject(directory)
	214	+ self._projects[directory] = workspace_project
	215	+
	216	+ workspace_project.add_project(project_path, element_name)
	217	+ return workspace_project
	218	+
	219	+ # remove()
	220	+ #
	221	+ # Removes the project path and element name from the WorkspaceProject that exists
	222	+ # for that directory.
	223	+ #
	224	+ # NOTE: This currently just deletes the file, but with support for multiple
	225	+ # projects opening the same workspace, this will involve decreasing the count
	226	+ # and deleting the file if there are no more projects.
	227	+ #
	228	+ # Args:
	229	+ # directory (str): The directory to search for a WorkspaceProject.
	230	+ #
	231	+ def remove(self, directory):
	232	+ workspace_project = self.get(directory)
	233	+ if not workspace_project:
	234	+ raise LoadError(LoadErrorReason.MISSING_FILE,
	235	+ "Failed to find a {} file to remove".format(WORKSPACE_PROJECT_FILE))
	236	+ path = workspace_project.get_filename()
	237	+ try:
	238	+ os.unlink(path)
	239	+ except FileNotFoundError:
	240	+ pass
28	241
29	242
30	243	# Workspace()
...	...	@@ -174,10 +387,15 @@ class Workspace():
174	387	if recalculate or self._key is None:
175	388	fullpath = self.get_absolute_path()
176	389
	390	+ excluded_files = (WORKSPACE_PROJECT_FILE,)
	391	+
177	392	# Get a list of tuples of the the project relative paths and fullpaths
178	393	if os.path.isdir(fullpath):
179	394	filelist = utils.list_relative_paths(fullpath)
180		- filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
	395	+ filelist = [
	396	+ (relpath, os.path.join(fullpath, relpath)) for relpath in filelist
	397	+ if relpath not in excluded_files
	398	+ ]
181	399	else:
182	400	filelist = [(self.get_absolute_path(), fullpath)]
183	401
...	...	@@ -199,12 +417,14 @@ class Workspace():
199	417	#
200	418	# Args:
201	419	# toplevel_project (Project): Top project used to resolve paths.
	420	+# workspace_project_cache (WorkspaceProjectCache): The cache of WorkspaceProjects
202	421	#
203	422	class Workspaces():
204		- def __init__(self, toplevel_project):
	423	+ def __init__(self, toplevel_project, workspace_project_cache):
205	424	self._toplevel_project = toplevel_project
206	425	self._bst_directory = os.path.join(toplevel_project.directory, ".bst")
207	426	self._workspaces = self._load_config()
	427	+ self._workspace_project_cache = workspace_project_cache
208	428
209	429	# list()
210	430	#
...	...	@@ -219,19 +439,36 @@ class Workspaces():
219	439
220	440	# create_workspace()
221	441	#
222		- # Create a workspace in the given path for the given element.
	442	+ # Create a workspace in the given path for the given element, and potentially
	443	+ # checks-out the target into it.
223	444	#
224	445	# Args:
225		- # element_name (str) - The element name to create a workspace for
	446	+ # target (Element) - The element to create a workspace for
226	447	# path (str) - The path in which the workspace should be kept
	448	+ # checkout (bool): Whether to check-out the element's sources into the directory
227	449	#
228		- def create_workspace(self, element_name, path):
229		- if path.startswith(self._toplevel_project.directory):
230		- path = os.path.relpath(path, self._toplevel_project.directory)
	450	+ def create_workspace(self, target, path, checkout):
	451	+ element_name = target._get_full_name()
	452	+ project_dir = self._toplevel_project.directory
	453	+ if path.startswith(project_dir):
	454	+ workspace_path = os.path.relpath(path, project_dir)
	455	+ else:
	456	+ workspace_path = path
231	457
232		- self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
	458	+ self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path)
233	459
234		- return self._workspaces[element_name]
	460	+ if checkout:
	461	+ with target.timed_activity("Staging sources to {}".format(path)):
	462	+ target._open_workspace()
	463	+
	464	+ workspace_project = self._workspace_project_cache.add(path, project_dir, element_name)
	465	+ project_file_path = workspace_project.get_filename()
	466	+
	467	+ if os.path.exists(project_file_path):
	468	+ target.warn("{} was staged from this element's sources".format(WORKSPACE_PROJECT_FILE))
	469	+ workspace_project.write()
	470	+
	471	+ self.save_config()
235	472
236	473	# get_workspace()
237	474	#
...	...	@@ -280,8 +517,19 @@ class Workspaces():
280	517	# element_name (str) - The element name whose workspace to delete
281	518	#
282	519	def delete_workspace(self, element_name):
	520	+ workspace = self.get_workspace(element_name)
283	521	del self._workspaces[element_name]
284	522
	523	+ # Remove from the cache if it exists
	524	+ try:
	525	+ self._workspace_project_cache.remove(workspace.get_absolute_path())
	526	+ except LoadError as e:
	527	+ # We might be closing a workspace with a deleted directory
	528	+ if e.reason == LoadErrorReason.MISSING_FILE:
	529	+ pass
	530	+ else:
	531	+ raise
	532	+
285	533	# save_config()
286	534	#
287	535	# Dump the current workspace element to the project configuration

buildstream/data/userconfig.yaml

---

...	...	@@ -128,6 +128,14 @@ prompt:
128	128	#
129	129	really-workspace-close-remove-dir: ask
130	130
	131	+ # Whether to really proceed with 'bst workspace close' when doing so would
	132	+ # stop them from running bst commands in this workspace.
	133	+ #
	134	+ # ask - Ask the user if they are sure.
	135	+ # yes - Always close, without asking.
	136	+ #
	137	+ really-workspace-close-project-inaccessible: ask
	138	+
131	139	# Whether to really proceed with 'bst workspace reset' doing a hard reset of
132	140	# a workspace, potentially losing changes.
133	141	#

buildstream/utils.py

---

...	...	@@ -1259,3 +1259,17 @@ def _message_digest(message_buffer):
1259	1259	digest.hash = sha.hexdigest()
1260	1260	digest.size_bytes = len(message_buffer)
1261	1261	return digest
	1262	+
	1263	+
	1264	+# Returns the first directory to contain filename, or an empty string if
	1265	+# none found
	1266	+#
	1267	+def _search_upward_for_file(directory, filename):
	1268	+ directory = os.path.abspath(directory)
	1269	+ while not os.path.isfile(os.path.join(directory, filename)):
	1270	+ parent_dir = os.path.dirname(directory)
	1271	+ if directory == parent_dir:
	1272	+ return ""
	1273	+ directory = parent_dir
	1274	+
	1275	+ return directory

tests/frontend/workspace.py

---

...	...	@@ -31,6 +31,7 @@ import shutil
31	31	import subprocess
32	32	from ruamel.yaml.comments import CommentedSet
33	33	from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
	34	+from tests.testutils import create_artifact_share
34	35
35	36	from buildstream import _yaml
36	37	from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
...	...	@@ -615,9 +616,12 @@ def test_list(cli, tmpdir, datafiles):
615	616	@pytest.mark.datafiles(DATA_DIR)
616	617	@pytest.mark.parametrize("kind", repo_kinds)
617	618	@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
618		-def test_build(cli, tmpdir, datafiles, kind, strict):
	619	+@pytest.mark.parametrize("call_from", [("project"), ("workspace")])
	620	+def test_build(cli, tmpdir_factory, datafiles, kind, strict, call_from):
	621	+ tmpdir = tmpdir_factory.mktemp('')
619	622	element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
620	623	checkout = os.path.join(str(tmpdir), 'checkout')
	624	+ args_pre = ['-C', workspace] if call_from == "workspace" else []
621	625
622	626	# Modify workspace
623	627	shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
...	...	@@ -640,15 +644,14 @@ def test_build(cli, tmpdir, datafiles, kind, strict):
640	644	# Build modified workspace
641	645	assert cli.get_element_state(project, element_name) == 'buildable'
642	646	assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
643		- result = cli.run(project=project, args=['build', element_name])
	647	+ result = cli.run(project=project, args=args_pre + ['build', element_name])
644	648	result.assert_success()
645	649	assert cli.get_element_state(project, element_name) == 'cached'
646	650	assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
647	651
648	652	# Checkout the result
649		- result = cli.run(project=project, args=[
650		- 'checkout', element_name, checkout
651		- ])
	653	+ result = cli.run(project=project,
	654	+ args=args_pre + ['checkout', element_name, checkout])
652	655	result.assert_success()
653	656
654	657	# Check that the pony.conf from the modified workspace exists
...	...	@@ -1055,3 +1058,131 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles):
1055	1058	result = cli.run(project=project, args=["build", element_name])
1056	1059	assert "BUG" not in result.stderr
1057	1060	assert cli.get_element_state(project, element_name) != "cached"
	1061	+
	1062	+
	1063	+@pytest.mark.datafiles(DATA_DIR)
	1064	+def test_external_fetch(cli, datafiles, tmpdir_factory):
	1065	+ # Fetching from a workspace outside a project doesn't fail horribly
	1066	+ tmpdir = tmpdir_factory.mktemp('')
	1067	+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
	1068	+
	1069	+ result = cli.run(project=project, args=['-C', workspace, 'fetch', element_name])
	1070	+ result.assert_success()
	1071	+
	1072	+ # We already fetched it by opening the workspace, but we're also checking
	1073	+ # `bst show` works here
	1074	+ assert cli.get_element_state(project, element_name) == 'buildable'
	1075	+
	1076	+
	1077	+@pytest.mark.datafiles(DATA_DIR)
	1078	+def test_external_push_pull(cli, datafiles, tmpdir_factory):
	1079	+ # Pushing and pulling to/from an artifact cache works from an external workspace
	1080	+ tmpdir = tmpdir_factory.mktemp('')
	1081	+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
	1082	+
	1083	+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
	1084	+ result = cli.run(project=project, args=['-C', workspace, 'build', element_name])
	1085	+ result.assert_success()
	1086	+
	1087	+ cli.configure({
	1088	+ 'artifacts': {'url': share.repo, 'push': True}
	1089	+ })
	1090	+
	1091	+ result = cli.run(project=project, args=['-C', workspace, 'push', element_name])
	1092	+ result.assert_success()
	1093	+
	1094	+ result = cli.run(project=project, args=['-C', workspace, 'pull', '--deps', 'all', element_name])
	1095	+ result.assert_success()
	1096	+
	1097	+
	1098	+@pytest.mark.datafiles(DATA_DIR)
	1099	+def test_external_track(cli, datafiles, tmpdir_factory):
	1100	+ # Tracking does not get horribly confused
	1101	+ tmpdir = tmpdir_factory.mktemp('')
	1102	+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", True)
	1103	+
	1104	+ # The workspace is necessarily already tracked, so we only care that
	1105	+ # there's no weird errors.
	1106	+ result = cli.run(project=project, args=['-C', workspace, 'track', element_name])
	1107	+ result.assert_success()
	1108	+
	1109	+
	1110	+@pytest.mark.datafiles(DATA_DIR)
	1111	+def test_external_open_other(cli, datafiles, tmpdir_factory):
	1112	+ # >From inside an external workspace, open another workspace
	1113	+ tmpdir1 = tmpdir_factory.mktemp('')
	1114	+ tmpdir2 = tmpdir_factory.mktemp('')
	1115	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1116	+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
	1117	+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
	1118	+
	1119	+ # Closing the other element first, because I'm too lazy to create an
	1120	+ # element without opening it
	1121	+ result = cli.run(project=project, args=['workspace', 'close', beta_element])
	1122	+ result.assert_success()
	1123	+
	1124	+ result = cli.run(project=project, args=[
	1125	+ '-C', alpha_workspace, 'workspace', 'open', '--force', '--directory', beta_workspace, beta_element
	1126	+ ])
	1127	+ result.assert_success()
	1128	+
	1129	+
	1130	+@pytest.mark.datafiles(DATA_DIR)
	1131	+def test_external_close_other(cli, datafiles, tmpdir_factory):
	1132	+ # >From inside an external workspace, close the other workspace
	1133	+ tmpdir1 = tmpdir_factory.mktemp('')
	1134	+ tmpdir2 = tmpdir_factory.mktemp('')
	1135	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1136	+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
	1137	+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
	1138	+
	1139	+ result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element])
	1140	+ result.assert_success()
	1141	+
	1142	+
	1143	+@pytest.mark.datafiles(DATA_DIR)
	1144	+def test_external_close_self(cli, datafiles, tmpdir_factory):
	1145	+ # >From inside an external workspace, close it
	1146	+ tmpdir1 = tmpdir_factory.mktemp('')
	1147	+ tmpdir2 = tmpdir_factory.mktemp('')
	1148	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1149	+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
	1150	+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
	1151	+
	1152	+ result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', alpha_element])
	1153	+ result.assert_success()
	1154	+
	1155	+
	1156	+@pytest.mark.datafiles(DATA_DIR)
	1157	+def test_external_reset_other(cli, datafiles, tmpdir_factory):
	1158	+ tmpdir1 = tmpdir_factory.mktemp('')
	1159	+ tmpdir2 = tmpdir_factory.mktemp('')
	1160	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1161	+ alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha")
	1162	+ beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta")
	1163	+
	1164	+ result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'reset', beta_element])
	1165	+ result.assert_success()
	1166	+
	1167	+
	1168	+@pytest.mark.datafiles(DATA_DIR)
	1169	+def test_external_reset_self(cli, datafiles, tmpdir):
	1170	+ element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
	1171	+
	1172	+ # Command succeeds
	1173	+ result = cli.run(project=project, args=['-C', workspace, 'workspace', 'reset', element])
	1174	+ result.assert_success()
	1175	+
	1176	+ # Successive commands still work (i.e. .bstproject.yaml hasn't been deleted)
	1177	+ result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
	1178	+ result.assert_success()
	1179	+
	1180	+
	1181	+@pytest.mark.datafiles(DATA_DIR)
	1182	+def test_external_list(cli, datafiles, tmpdir_factory):
	1183	+ tmpdir = tmpdir_factory.mktemp('')
	1184	+ # Making use of the assumption that it's the same project in both invocations of open_workspace
	1185	+ element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False)
	1186	+
	1187	+ result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list'])
	1188	+ result.assert_success()

tests/integration/shell.py

---

...	...	@@ -353,3 +353,29 @@ def test_integration_devices(cli, tmpdir, datafiles):
353	353
354	354	result = execute_shell(cli, project, ["true"], element=element_name)
355	355	assert result.exit_code == 0
	356	+
	357	+
	358	+# Test that a shell can be opened from an external workspace
	359	+@pytest.mark.datafiles(DATA_DIR)
	360	+@pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
	361	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	362	+def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell):
	363	+ tmpdir = tmpdir_factory.mktemp("")
	364	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	365	+ element_name = 'autotools/amhello.bst'
	366	+ workspace_dir = os.path.join(str(tmpdir), 'workspace')
	367	+
	368	+ result = cli.run(project=project, args=[
	369	+ 'workspace', 'open', '--directory', workspace_dir, element_name
	370	+ ])
	371	+ result.assert_success()
	372	+
	373	+ result = cli.run(project=project, args=['-C', workspace_dir, 'build', element_name])
	374	+ result.assert_success()
	375	+
	376	+ command = ['shell']
	377	+ if build_shell == 'build':
	378	+ command.append('--build')
	379	+ command.extend([element_name, '--', 'true'])
	380	+ result = cli.run(project=project, cwd=workspace_dir, args=command)
	381	+ result.assert_success()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.