[Notes] [Git][BuildStream/buildstream][lachlan/pickle-yaml-test-list-composite] 4 commits: yamlcache/tests: Make yamlcache.open pass a dir in instead of divining its own default

[Lachlan <gitlab mg gitlab com>]

Title: GitLab

Lachlan pushed to branch lachlan/pickle-yaml-test-list-composite at BuildStream / buildstream

Commits:

• 030935ff
  by Jonathan Maw at 2018-10-05T16:39:11Z

  yamlcache/tests: Make yamlcache.open pass a dir in instead of divining its own default
  

• ae5b554b
  by Lachlan Mackenzie at 2018-10-05T16:39:11Z

  Add loader for yaml test files cache option
  

• fa7d0e88
  by Lachlan Mackenzie at 2018-10-05T16:39:11Z

  Add yaml cache testing to yaml list composition test
  

• f9e39bdc
  by Lachlan Mackenzie at 2018-10-05T16:39:23Z

  Add yaml cache test to yaml list composition twice
  

4 changed files:

• buildstream/_loader/loader.py
• buildstream/_yamlcache.py
• tests/frontend/yamlcache.py
• tests/yaml/yaml.py

Changes:

buildstream/_loader/loader.py

---

...	...	@@ -113,7 +113,13 @@ class Loader():
113	113	profile_start(Topics.LOAD_PROJECT, target)
114	114	junction, name, loader = self._parse_name(target, rewritable, ticker,
115	115	fetch_subprojects=fetch_subprojects)
116		- with YamlCache.open(self._context) as yaml_cache:
	116	+
	117	+ # XXX This will need to be changed to the context's top-level project if this method
	118	+ # is ever used for subprojects
	119	+ top_dir = self.project.directory
	120	+
	121	+ cache_file = YamlCache.get_cache_file(top_dir)
	122	+ with YamlCache.open(self._context, cache_file) as yaml_cache:
117	123	loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
118	124	deps.append(Dependency(name, junction=junction))
119	125	profile_end(Topics.LOAD_PROJECT, target)

buildstream/_yamlcache.py

---

...	...	@@ -54,8 +54,10 @@ class YamlCache():
54	54	self._context = context
55	55
56	56	# Writes the yaml cache to the specified path.
57		- def write(self):
58		- path = self._get_cache_file(self._context)
	57	+ #
	58	+ # Args:
	59	+ # path (str): The path to the cache file.
	60	+ def write(self, path):
59	61	parent_dir = os.path.dirname(path)
60	62	os.makedirs(parent_dir, exist_ok=True)
61	63	with open(path, "wb") as f:
...	...	@@ -139,14 +141,14 @@ class YamlCache():
139	141	#
140	142	# Args:
141	143	# context (Context): The context.
	144	+ # cachefile (str): The path to the cache file.
142	145	#
143	146	# Returns:
144	147	# (YamlCache): A YamlCache.
145	148	@staticmethod
146	149	@contextmanager
147		- def open(context):
	150	+ def open(context, cachefile):
148	151	# Try to load from disk first
149		- cachefile = YamlCache._get_cache_file(context)
150	152	cache = None
151	153	if os.path.exists(cachefile):
152	154	try:
...	...	@@ -155,12 +157,13 @@ class YamlCache():
155	157	except pickle.UnpicklingError as e:
156	158	sys.stderr.write("Failed to load YamlCache, {}\n".format(e))
157	159
	160	+ # Failed to load from disk, create a new one
158	161	if not cache:
159	162	cache = YamlCache(context)
160	163
161	164	yield cache
162	165
163		- cache.write()
	166	+ cache.write(cachefile)
164	167
165	168	# Calculates a key for putting into the cache.
166	169	@staticmethod
...	...	@@ -170,13 +173,7 @@ class YamlCache():
170	173
171	174	# Retrieves a path to the yaml cache file.
172	175	@staticmethod
173		- def _get_cache_file(context):
174		- try:
175		- toplevel_project = context.get_toplevel_project()
176		- top_dir = toplevel_project.directory
177		- except IndexError:
178		- # Context has no projects, fall back to current directory
179		- top_dir = os.getcwd()
	176	+ def get_cache_file(top_dir):
180	177	return os.path.join(top_dir, ".bst", YAML_CACHE_FILENAME)
181	178
182	179

tests/frontend/yamlcache.py

---

...	...	@@ -52,7 +52,8 @@ def generate_project(tmpdir, ref_storage, with_junction, name="test"):
52	52	def with_yamlcache(project_dir):
53	53	context = Context()
54	54	project = Project(project_dir, context)
55		- with YamlCache.open(context) as yamlcache:
	55	+ cache_file = YamlCache.get_cache_file(project_dir)
	56	+ with YamlCache.open(context, cache_file) as yamlcache:
56	57	yield yamlcache, project
57	58
58	59
...	...	@@ -62,12 +63,12 @@ def yamlcache_key(yamlcache, in_file, copy_tree=False):
62	63	return key
63	64
64	65
65		-def modified_file(input_file):
	66	+def modified_file(input_file, tmpdir):
66	67	with open(input_file) as f:
67	68	data = f.read()
68	69	assert 'variables' not in data
69	70	data += '\nvariables: {modified: True}\n'
70		- _, temppath = tempfile.mkstemp(text=True)
	71	+ _, temppath = tempfile.mkstemp(dir=tmpdir, text=True)
71	72	with open(temppath, 'w') as f:
72	73	f.write(data)
73	74
...	...	@@ -96,7 +97,7 @@ def test_yamlcache_used(cli, tmpdir, ref_storage, with_junction, move_project):
96	97	# *Absolutely* horrible cache corruption to check it's being used
97	98	# Modifying the data from the cache is fraught with danger,
98	99	# so instead I'll load a modified version of the original file
99		- temppath = modified_file(element_path)
	100	+ temppath = modified_file(element_path, str(tmpdir))
100	101	contents = _yaml.load(temppath, copy_tree=False, project=prj)
101	102	key = yamlcache_key(yc, element_path)
102	103	yc.put(prj, element_path, key, contents)

tests/yaml/yaml.py

---

1	1	import os
2	2	import pytest
	3	+import tempfile
3	4	from collections import Mapping
4	5
5	6	from buildstream import _yaml
6	7	from buildstream._exceptions import LoadError, LoadErrorReason
	8	+from buildstream._context import Context
	9	+from buildstream._yamlcache import YamlCache
7	10
8	11	DATA_DIR = os.path.join(
9	12	os.path.dirname(os.path.realpath(__file__)),
...	...	@@ -150,6 +153,22 @@ def test_composite_preserve_originals(datafiles):
150	153	assert(_yaml.node_get(orig_extra, str, 'old') == 'new')
151	154
152	155
	156	+def load_yaml_file(filename, *, cache_path, shortname=None, from_cache='raw'):
	157	+
	158	+ temppath = tempfile.mkstemp(dir=str(cache_path), text=True)
	159	+ context = Context()
	160	+
	161	+ with YamlCache.open(context, str(temppath)) as yc:
	162	+ if from_cache == 'raw':
	163	+ return _yaml.load(filename, shortname)
	164	+ elif from_cache == 'cached':
	165	+ _yaml.load(filename, shortname)
	166	+
	167	+ return _yaml.load(filename, shortname, yaml_cache=yc)
	168	+ else:
	169	+ assert False
	170	+
	171	+
153	172	# Tests for list composition
154	173	#
155	174	# Each test composits a filename on top of basics.yaml, and tests
...	...	@@ -165,6 +184,7 @@ def test_composite_preserve_originals(datafiles):
165	184	# prov_col: The expected provenance column of "mood"
166	185	#
167	186	@pytest.mark.datafiles(os.path.join(DATA_DIR))
	187	+@pytest.mark.parametrize('caching', [('raw'), ('cached')])
168	188	@pytest.mark.parametrize("filename,index,length,mood,prov_file,prov_line,prov_col", [
169	189
170	190	# Test results of compositing with the (<) prepend directive
...	...	@@ -195,14 +215,15 @@ def test_composite_preserve_originals(datafiles):
195	215	('implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8),
196	216	('implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8),
197	217	])
198		-def test_list_composition(datafiles, filename,
	218	+def test_list_composition(datafiles, filename, tmpdir,
199	219	index, length, mood,
200		- prov_file, prov_line, prov_col):
201		- base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
202		- overlay = os.path.join(datafiles.dirname, datafiles.basename, filename)
	220	+ prov_file, prov_line, prov_col, caching):
	221	+ base_file = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
	222	+ overlay_file = os.path.join(datafiles.dirname, datafiles.basename, filename)
	223	+
	224	+ base = load_yaml_file(base_file, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
	225	+ overlay = load_yaml_file(overlay_file, cache_path=tmpdir, shortname=filename, from_cache=caching)
203	226
204		- base = _yaml.load(base, shortname='basics.yaml')
205		- overlay = _yaml.load(overlay, shortname=filename)
206	227	_yaml.composite_dict(base, overlay)
207	228
208	229	children = _yaml.node_get(base, list, 'children')
...	...	@@ -254,6 +275,7 @@ def test_list_deletion(datafiles):
254	275	# prov_col: The expected provenance column of "mood"
255	276	#
256	277	@pytest.mark.datafiles(os.path.join(DATA_DIR))
	278	+@pytest.mark.parametrize('caching', [('raw'), ('cached')])
257	279	@pytest.mark.parametrize("filename1,filename2,index,length,mood,prov_file,prov_line,prov_col", [
258	280
259	281	# Test results of compositing literal list with (>) and then (<)
...	...	@@ -310,9 +332,9 @@ def test_list_deletion(datafiles):
310	332	('listoverwrite.yaml', 'listprepend.yaml', 2, 4, 'overwrite1', 'listoverwrite.yaml', 5, 10),
311	333	('listoverwrite.yaml', 'listprepend.yaml', 3, 4, 'overwrite2', 'listoverwrite.yaml', 7, 10),
312	334	])
313		-def test_list_composition_twice(datafiles, filename1, filename2,
	335	+def test_list_composition_twice(datafiles, tmpdir, filename1, filename2,
314	336	index, length, mood,
315		- prov_file, prov_line, prov_col):
	337	+ prov_file, prov_line, prov_col, caching):
316	338	file_base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
317	339	file1 = os.path.join(datafiles.dirname, datafiles.basename, filename1)
318	340	file2 = os.path.join(datafiles.dirname, datafiles.basename, filename2)
...	...	@@ -320,9 +342,9 @@ def test_list_composition_twice(datafiles, filename1, filename2,
320	342	#####################
321	343	# Round 1 - Fight !
322	344	#####################
323		- base = _yaml.load(file_base, shortname='basics.yaml')
324		- overlay1 = _yaml.load(file1, shortname=filename1)
325		- overlay2 = _yaml.load(file2, shortname=filename2)
	345	+ base = load_yaml_file(file_base, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
	346	+ overlay1 = load_yaml_file(file1, cache_path=tmpdir, shortname=filename1, from_cache=caching)
	347	+ overlay2 = load_yaml_file(file2, cache_path=tmpdir, shortname=filename2, from_cache=caching)
326	348
327	349	_yaml.composite_dict(base, overlay1)
328	350	_yaml.composite_dict(base, overlay2)
...	...	@@ -337,9 +359,9 @@ def test_list_composition_twice(datafiles, filename1, filename2,
337	359	#####################
338	360	# Round 2 - Fight !
339	361	#####################
340		- base = _yaml.load(file_base, shortname='basics.yaml')
341		- overlay1 = _yaml.load(file1, shortname=filename1)
342		- overlay2 = _yaml.load(file2, shortname=filename2)
	362	+ base = load_yaml_file(file_base, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
	363	+ overlay1 = load_yaml_file(file1, cache_path=tmpdir, shortname=filename1, from_cache=caching)
	364	+ overlay2 = load_yaml_file(file2, cache_path=tmpdir, shortname=filename2, from_cache=caching)
343	365
344	366	_yaml.composite_dict(overlay1, overlay2)
345	367	_yaml.composite_dict(base, overlay1)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.