[Notes] [Git][BuildStream/buildstream][jonathan/workspace-fragment-create] 6 commits: _yaml.py: Do not insert into cache if retrieved from cache

[Jonathan Maw <gitlab mg gitlab com>]

Title: GitLab

Jonathan Maw pushed to branch jonathan/workspace-fragment-create at BuildStream / buildstream

Commits:

• 8c41b5b8
  by Daniel Silverstone at 2018-10-26T09:02:30Z

  _yaml.py: Do not insert into cache if retrieved from cache
  
  We were unconditionally inserting into the yaml cache anything we had
  retrieved from the yaml cache.  This had the side-effect of being expensive
  on re-runs of commands where yaml hadn't changed.
  
  Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>
  

• 494d7018
  by Jonathan Maw at 2018-10-26T09:26:36Z

  Merge branch 'danielsilverstone-ct/do-not-reinsert-yaml' into 'master'
  
  _yaml.py: Do not insert into cache if retrieved from cache
  
  See merge request BuildStream/buildstream!902

• 21a6029f
  by Jonathan Maw at 2018-10-26T09:49:26Z

  _workspaces.py: Add a WorkspaceLocal object
  
  WorkspaceLocal contains all the information needed to generate data
  that's stored within the workspace. Currently this is just a list of
  project directories.
  

• 8be2b8bd
  by Jonathan Maw at 2018-10-26T09:49:26Z

  _stream.py: Opened workspaces store the project that opened them
  

• 69edf3bd
  by Jonathan Maw at 2018-10-26T09:49:26Z

  _stream.py: workspace_close deletes the stored project reference
  

• 0106b847
  by Jonathan Maw at 2018-10-26T09:49:26Z

  tests: Workspace tests check existence of .bstproject.yaml
  

4 changed files:

• buildstream/_stream.py
• buildstream/_workspaces.py
• buildstream/_yaml.py
• tests/frontend/workspace.py

Changes:

buildstream/_stream.py

---

...	...	@@ -509,6 +509,9 @@ class Stream():
509	509	with target.timed_activity("Staging sources to {}".format(directory)):
510	510	target._open_workspace()
511	511
	512	+ workspace_local = workspaces.create_workspace_local(directory)
	513	+ workspace_local.write()
	514	+
512	515	workspaces.save_config()
513	516	self._message(MessageType.INFO, "Saved workspace configuration")
514	517
...	...	@@ -533,6 +536,10 @@ class Stream():
533	536	except OSError as e:
534	537	raise StreamError("Could not remove '{}': {}"
535	538	.format(workspace.get_absolute_path(), e)) from e
	539	+ else:
	540	+ # TODO: At some point, closing a workspace only deletes the file if no projects are using it.
	541	+ workspace_local = workspaces.create_workspace_local(workspace.get_absolute_path())
	542	+ workspace_local.delete()
536	543
537	544	# Delete the workspace and save the configuration
538	545	workspaces.delete_workspace(element_name)

buildstream/_workspaces.py

---

...	...	@@ -25,6 +25,82 @@ from ._exceptions import LoadError, LoadErrorReason
25	25
26	26
27	27	BST_WORKSPACE_FORMAT_VERSION = 3
	28	+BST_WORKSPACE_LOCAL_FORMAT_VERSION = 1
	29	+WORKSPACE_LOCAL_FILE = ".bstproject.yaml"
	30	+
	31	+
	32	+# WorkspaceLocal()
	33	+#
	34	+# An object to contain various helper functions and data required for
	35	+# referring from a workspace back to buildstream.
	36	+#
	37	+# Args:
	38	+# directory (str): The directory that the workspace exists in
	39	+# project_paths (list of str): The project paths used to refer back
	40	+# to buildstream projects.
	41	+class WorkspaceLocal():
	42	+ def __init__(self, directory, project_paths):
	43	+ self._project_paths = project_paths
	44	+ self._directory = directory
	45	+
	46	+ # get_default_path()
	47	+ #
	48	+ # Retrieves the default path to a project.
	49	+ #
	50	+ # Returns:
	51	+ # (str): The path to a project
	52	+ def get_default_path(self):
	53	+ return self._project_paths[0]
	54	+
	55	+ # to_dict()
	56	+ #
	57	+ # Turn the members data into a dict for serialization purposes
	58	+ #
	59	+ # Returns:
	60	+ # (dict): A dict representation of the WorkspaceLocal
	61	+ #
	62	+ def to_dict(self):
	63	+ ret = {
	64	+ 'projects': self._project_paths,
	65	+ 'format-version': BST_WORKSPACE_LOCAL_FORMAT_VERSION,
	66	+ }
	67	+ return ret
	68	+
	69	+ # from_dict()
	70	+ #
	71	+ # Loads a new WorkspaceLocal from a simple dictionary
	72	+ #
	73	+ # Args:
	74	+ # directory (str): The directory that the workspace exists in
	75	+ # (dict) dictionary: The dict to generate a WorkspaceLocal from
	76	+ #
	77	+ # Returns:
	78	+ # (WorkspaceLocal): A newly instantiated WorkspaceLocal
	79	+ @classmethod
	80	+ def from_dict(cls, directory, dictionary):
	81	+ # Only know how to handle one format-version at the moment.
	82	+ assert dictionary['format-version'] == BST_WORKSPACE_LOCAL_FORMAT_VERSION
	83	+
	84	+ return cls(directory, dictionary['projects'])
	85	+
	86	+ # write()
	87	+ #
	88	+ # Writes the WorkspaceLocal to disk
	89	+ def write(self):
	90	+ os.makedirs(self._directory, exist_ok=True)
	91	+ _yaml.dump(self.to_dict(), self._get_filename())
	92	+
	93	+ # delete()
	94	+ #
	95	+ # Deletes the WorkspaceLocal from disk, if it exists.
	96	+ def delete(self):
	97	+ try:
	98	+ os.unlink(self._get_filename())
	99	+ except FileNotFoundError:
	100	+ pass
	101	+
	102	+ def _get_filename(self):
	103	+ return os.path.join(self._directory, WORKSPACE_LOCAL_FILE)
28	104
29	105
30	106	# Workspace()
...	...	@@ -302,6 +378,21 @@ class Workspaces():
302	378	_yaml.dump(_yaml.node_sanitize(config),
303	379	self._get_filename())
304	380
	381	+ # create_workspace_local()
	382	+ #
	383	+ # Create a WorkspaceLocal in the given path.
	384	+ #
	385	+ # Args:
	386	+ # path (str): The path to the workspace
	387	+ #
	388	+ # Returns:
	389	+ # (WorkspaceLocal): The created WorkspaceLocal
	390	+
	391	+ def create_workspace_local(self, directory):
	392	+ # TODO: Read existing WorkspaceLocal if it exists. Doesn't matter yet.
	393	+ project_dir = self._toplevel_project.directory
	394	+ return WorkspaceLocal(directory, [project_dir])
	395	+
305	396	# _load_config()
306	397	#
307	398	# Loads and parses the workspace configuration

buildstream/_yaml.py

---

...	...	@@ -204,9 +204,8 @@ def load(filename, shortname=None, copy_tree=False, *, project=None, yaml_cache=
204	204
205	205	if not data:
206	206	data = load_data(contents, file, copy_tree=copy_tree)
207		-
208		- if yaml_cache:
209		- yaml_cache.put_from_key(project, filename, key, data)
	207	+ if yaml_cache:
	208	+ yaml_cache.put_from_key(project, filename, key, data)
210	209
211	210	return data
212	211	except FileNotFoundError as e:

tests/frontend/workspace.py

---

...	...	@@ -93,6 +93,13 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir
93	93
94	94	result.assert_success()
95	95
	96	+ # Assert that a .bstproject.yaml file has been created
	97	+ # and contains the path to the project
	98	+ bstproject_path = os.path.join(workspace_dir, '.bstproject.yaml')
	99	+ assert os.path.exists(bstproject_path)
	100	+ with open(bstproject_path) as f:
	101	+ assert project_path in f.read()
	102	+
96	103	# Assert that we are now buildable because the source is
97	104	# now cached.
98	105	assert cli.get_element_state(project_path, element_name) == 'buildable'
...	...	@@ -148,6 +155,10 @@ def test_open_force(cli, tmpdir, datafiles, kind):
148	155	# Assert the workspace dir still exists
149	156	assert os.path.exists(workspace)
150	157
	158	+ # Assert the bstproject doesn't exist
	159	+ bstproject_path = os.path.join(workspace, '.bstproject.yaml')
	160	+ assert not os.path.exists(bstproject_path)
	161	+
151	162	# Now open the workspace again with --force, this should happily succeed
152	163	result = cli.run(project=project, args=[
153	164	'workspace', 'open', '--force', element_name, workspace

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.