[Notes] [Git][BuildStream/buildstream][raoul/870-root-cache-dir] rootcachedir: add new dir option that's default to other dirs

[Raoul Hidalgo Charman <gitlab mg gitlab com>]

Title: GitLab

Raoul Hidalgo Charman pushed to branch raoul/870-root-cache-dir at BuildStream / buildstream

Commits:

• 673fa0b7
  by Raoul Hidalgo Charman at 2019-01-23T16:11:02Z

  rootcachedir: add new dir option that's default to other dirs
  
  Deprecates artifactdir and builddir.
  
  Fixes #870
  

16 changed files:

• buildstream/_context.py
• buildstream/data/userconfig.yaml
• conftest.py
• doc/bst2html.py
• doc/sessions/running-commands.run
• tests/artifactcache/expiry.py
• tests/artifactcache/junctions.py
• tests/artifactcache/pull.py
• tests/artifactcache/push.py
• tests/frontend/pull.py
• tests/integration/build-tree.py
• tests/integration/cachedfail.py
• tests/integration/messages.py
• tests/integration/pullbuildtrees.py
• tests/integration/source-determinism.py
• tests/testutils/runcli.py

Changes:

buildstream/_context.py

---

...	...	@@ -58,12 +58,21 @@ class Context():
58	58	# Filename indicating which configuration file was used, or None for the defaults
59	59	self.config_origin = None
60	60
	61	+ # The directory under which other directories are based
	62	+ self.rootcachedir = None
	63	+
61	64	# The directory where various sources are stored
62	65	self.sourcedir = None
63	66
64	67	# The directory where build sandboxes will be created
65	68	self.builddir = None
66	69
	70	+ # The directory for CAS
	71	+ self.casdir = None
	72	+
	73	+ # The directory for temporary files
	74	+ self.tmpdir = None
	75	+
67	76	# Default root location for workspaces
68	77	self.workspacedir = None
69	78
...	...	@@ -188,13 +197,30 @@ class Context():
188	197	user_config = _yaml.load(config)
189	198	_yaml.composite(defaults, user_config)
190	199
	200	+ # Give deprecation warnings
	201	+ if defaults.get('builddir'):
	202	+ print("builddir is deprecated, use rootcachedir")
	203	+ else:
	204	+ defaults['builddir'] = os.path.join(defaults['rootcachedir'], 'build')
	205	+
	206	+ if defaults.get('artifactdir'):
	207	+ print("artifactdir is deprecated, use rootcachedir")
	208	+ else:
	209	+ defaults['artifactdir'] = os.path.join(defaults['rootcachedir'], 'artifacts')
	210	+
191	211	_yaml.node_validate(defaults, [
192		- 'sourcedir', 'builddir', 'artifactdir', 'logdir',
	212	+ 'rootcachedir', 'sourcedir', 'builddir', 'artifactdir', 'logdir',
193	213	'scheduler', 'artifacts', 'logging', 'projects',
194		- 'cache', 'prompt', 'workspacedir', 'remote-execution'
	214	+ 'cache', 'prompt', 'workspacedir', 'remote-execution',
195	215	])
196	216
197		- for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
	217	+ # add directories not set by users
	218	+ defaults['tmpdir'] = os.path.join(defaults['rootcachedir'], 'tmp')
	219	+ defaults['casdir'] = os.path.join(defaults['rootcachedir'], 'cas')
	220	+
	221	+ for directory in ['rootcachedir', 'sourcedir', 'builddir',
	222	+ 'artifactdir', 'logdir', 'workspacedir', 'casdir',
	223	+ 'tmpdir']:
198	224	# Allow the ~ tilde expansion and any environment variables in
199	225	# path specification in the config files.
200	226	#
...	...	@@ -654,7 +680,7 @@ class Context():
654	680
655	681	def get_cascache(self):
656	682	if self._cascache is None:
657		- self._cascache = CASCache(self.artifactdir)
	683	+ self._cascache = CASCache(self.rootcachedir)
658	684	return self._cascache
659	685
660	686	# guess_element()

buildstream/data/userconfig.yaml

---

...	...	@@ -13,11 +13,8 @@
13	13	# Location to store sources
14	14	sourcedir: ${XDG_CACHE_HOME}/buildstream/sources
15	15
16		-# Location to perform builds
17		-builddir: ${XDG_CACHE_HOME}/buildstream/build
18		-
19		-# Location to store local binary artifacts
20		-artifactdir: ${XDG_CACHE_HOME}/buildstream/artifacts
	16	+# Root location for other directories in the cache
	17	+rootcachedir: ${XDG_CACHE_HOME}/buildstream
21	18
22	19	# Location to store build logs
23	20	logdir: ${XDG_CACHE_HOME}/buildstream/logs

conftest.py

---

...	...	@@ -53,15 +53,15 @@ def pytest_runtest_setup(item):
53	53	class IntegrationCache():
54	54
55	55	def __init__(self, cache):
56		- cache = os.path.abspath(cache)
	56	+ self.root = os.path.abspath(cache)
57	57
58	58	# Use the same sources every time
59		- self.sources = os.path.join(cache, 'sources')
	59	+ self.sources = os.path.join(self.root, 'sources')
60	60
61	61	# Create a temp directory for the duration of the test for
62	62	# the artifacts directory
63	63	try:
64		- self.artifacts = tempfile.mkdtemp(dir=cache, prefix='artifacts-')
	64	+ self.artifacts = tempfile.mkdtemp(dir=self.root, prefix='artifacts-')
65	65	except OSError as e:
66	66	raise AssertionError("Unable to create test directory !") from e
67	67
...	...	@@ -86,6 +86,10 @@ def integration_cache(request):
86	86	shutil.rmtree(cache.artifacts)
87	87	except FileNotFoundError:
88	88	pass
	89	+ try:
	90	+ shutil.rmtree(os.path.join(cache.root, 'cas'))
	91	+ except FileNotFoundError:
	92	+ pass
89	93
90	94
91	95	#################################################

doc/bst2html.py

---

...	...	@@ -194,10 +194,9 @@ def workdir(source_cache=None):
194	194
195	195	bst_config_file = os.path.join(tempdir, 'buildstream.conf')
196	196	config = {
	197	+ 'rootcachedir': tempdir,
197	198	'sourcedir': source_cache,
198		- 'artifactdir': os.path.join(tempdir, 'artifacts'),
199	199	'logdir': os.path.join(tempdir, 'logs'),
200		- 'builddir': os.path.join(tempdir, 'build'),
201	200	}
202	201	_yaml.dump(config, bst_config_file)
203	202
...	...	@@ -411,12 +410,10 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
411	410	# Encode and save the output if that was asked for
412	411	output = _yaml.node_get(command, str, 'output', default_value=None)
413	412	if output is not None:
414		-
415	413	# Convert / Generate a nice <div>
416	414	converted = generate_html(command_out, directory, config_file,
417	415	source_cache, tempdir, palette,
418	416	command_str, command_fake_output is not None)
419		-
420	417	# Save it
421	418	filename = os.path.join(desc_dir, output)
422	419	filename = os.path.realpath(filename)

doc/sessions/running-commands.run

---

...	...	@@ -2,7 +2,7 @@
2	2	commands:
3	3	# Make it fetch first
4	4	- directory: ../examples/running-commands
5		- command: fetch hello.bst
	5	+ command: source fetch hello.bst
6	6
7	7	# Capture a show output
8	8	- directory: ../examples/running-commands

tests/artifactcache/expiry.py

---

...	...	@@ -403,7 +403,7 @@ def test_extract_expiry(cli, datafiles, tmpdir):
403	403	# Now we should have a directory for the cached target2.bst, which
404	404	# replaced target.bst in the cache, we should not have a directory
405	405	# for the target.bst
406		- refsdir = os.path.join(project, 'cache', 'artifacts', 'cas', 'refs', 'heads')
	406	+ refsdir = os.path.join(project, 'cache', 'cas', 'refs', 'heads')
407	407	refsdirtest = os.path.join(refsdir, 'test')
408	408	refsdirtarget = os.path.join(refsdirtest, 'target')
409	409	refsdirtarget2 = os.path.join(refsdirtest, 'target2')

tests/artifactcache/junctions.py

---

...	...	@@ -68,8 +68,8 @@ def test_push_pull(cli, tmpdir, datafiles):
68	68	# Now we've pushed, delete the user's local artifact cache
69	69	# directory and try to redownload it from the share
70	70	#
71		- artifacts = os.path.join(cli.directory, 'artifacts')
72		- shutil.rmtree(artifacts)
	71	+ cas = os.path.join(cli.directory, 'cas')
	72	+ shutil.rmtree(cas)
73	73
74	74	# Assert that nothing is cached locally anymore
75	75	state = cli.get_element_state(project, 'target.bst')

tests/artifactcache/pull.py

---

...	...	@@ -56,7 +56,7 @@ def test_pull(cli, tmpdir, datafiles):
56	56	# Set up an artifact cache.
57	57	with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
58	58	# Configure artifact share
59		- artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
	59	+ cache_dir = os.path.join(str(tmpdir), 'cache')
60	60	user_config_file = str(tmpdir.join('buildstream.conf'))
61	61	user_config = {
62	62	'scheduler': {
...	...	@@ -65,7 +65,8 @@ def test_pull(cli, tmpdir, datafiles):
65	65	'artifacts': {
66	66	'url': share.repo,
67	67	'push': True,
68		- }
	68	+ },
	69	+ 'rootcachedir': cache_dir
69	70	}
70	71
71	72	# Write down the user configuration file
...	...	@@ -92,7 +93,6 @@ def test_pull(cli, tmpdir, datafiles):
92	93	# Fake minimal context
93	94	context = Context()
94	95	context.load(config=user_config_file)
95		- context.artifactdir = os.path.join(str(tmpdir), 'cache', 'artifacts')
96	96	context.set_message_handler(message_handler)
97	97
98	98	# Load the project and CAS cache
...	...	@@ -102,7 +102,10 @@ def test_pull(cli, tmpdir, datafiles):
102	102
103	103	# Assert that the element's artifact is **not** cached
104	104	element = project.load_elements(['target.bst'])[0]
	105	+ print(element)
105	106	element_key = cli.get_element_key(project_dir, 'target.bst')
	107	+ print(context.casdir)
	108	+ print(cas.get_artifact_fullname(element, element_key))
106	109	assert not cas.contains(element, element_key)
107	110
108	111	queue = multiprocessing.Queue()
...	...	@@ -110,7 +113,7 @@ def test_pull(cli, tmpdir, datafiles):
110	113	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
111	114	process = multiprocessing.Process(target=_queue_wrapper,
112	115	args=(_test_pull, queue, user_config_file, project_dir,
113		- artifact_dir, 'target.bst', element_key))
	116	+ cache_dir, 'target.bst', element_key))
114	117
115	118	try:
116	119	# Keep SIGINT blocked in the child process
...	...	@@ -127,12 +130,14 @@ def test_pull(cli, tmpdir, datafiles):
127	130	assert cas.contains(element, element_key)
128	131
129	132
130		-def _test_pull(user_config_file, project_dir, artifact_dir,
	133	+def _test_pull(user_config_file, project_dir, cache_dir,
131	134	element_name, element_key, queue):
132	135	# Fake minimal context
133	136	context = Context()
134	137	context.load(config=user_config_file)
135		- context.artifactdir = artifact_dir
	138	+ context.rootcachedir = cache_dir
	139	+ context.casdir = os.path.join(cache_dir, 'cas')
	140	+ context.tmpdir = os.path.join(cache_dir, 'tmp')
136	141	context.set_message_handler(message_handler)
137	142
138	143	# Load the project manually
...	...	@@ -165,7 +170,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
165	170	# Set up an artifact cache.
166	171	with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
167	172	# Configure artifact share
168		- artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
	173	+ rootcache_dir = os.path.join(str(tmpdir), 'cache')
169	174	user_config_file = str(tmpdir.join('buildstream.conf'))
170	175	user_config = {
171	176	'scheduler': {
...	...	@@ -174,7 +179,8 @@ def test_pull_tree(cli, tmpdir, datafiles):
174	179	'artifacts': {
175	180	'url': share.repo,
176	181	'push': True,
177		- }
	182	+ },
	183	+ 'rootcachedir': rootcache_dir
178	184	}
179	185
180	186	# Write down the user configuration file
...	...	@@ -195,7 +201,6 @@ def test_pull_tree(cli, tmpdir, datafiles):
195	201	# Fake minimal context
196	202	context = Context()
197	203	context.load(config=user_config_file)
198		- context.artifactdir = os.path.join(str(tmpdir), 'cache', 'artifacts')
199	204	context.set_message_handler(message_handler)
200	205
201	206	# Load the project and CAS cache
...	...	@@ -218,7 +223,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
218	223	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
219	224	process = multiprocessing.Process(target=_queue_wrapper,
220	225	args=(_test_push_tree, queue, user_config_file, project_dir,
221		- artifact_dir, artifact_digest))
	226	+ artifact_digest))
222	227
223	228	try:
224	229	# Keep SIGINT blocked in the child process
...	...	@@ -246,7 +251,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
246	251	# Use subprocess to avoid creation of gRPC threads in main BuildStream process
247	252	process = multiprocessing.Process(target=_queue_wrapper,
248	253	args=(_test_pull_tree, queue, user_config_file, project_dir,
249		- artifact_dir, tree_digest))
	254	+ tree_digest))
250	255
251	256	try:
252	257	# Keep SIGINT blocked in the child process
...	...	@@ -268,11 +273,10 @@ def test_pull_tree(cli, tmpdir, datafiles):
268	273	assert os.path.exists(cas.objpath(directory_digest))
269	274
270	275
271		-def _test_push_tree(user_config_file, project_dir, artifact_dir, artifact_digest, queue):
	276	+def _test_push_tree(user_config_file, project_dir, artifact_digest, queue):
272	277	# Fake minimal context
273	278	context = Context()
274	279	context.load(config=user_config_file)
275		- context.artifactdir = artifact_dir
276	280	context.set_message_handler(message_handler)
277	281
278	282	# Load the project manually
...	...	@@ -304,11 +308,10 @@ def _test_push_tree(user_config_file, project_dir, artifact_dir, artifact_digest
304	308	queue.put("No remote configured")
305	309
306	310
307		-def _test_pull_tree(user_config_file, project_dir, artifact_dir, artifact_digest, queue):
	311	+def _test_pull_tree(user_config_file, project_dir, artifact_digest, queue):
308	312	# Fake minimal context
309	313	context = Context()
310	314	context.load(config=user_config_file)
311		- context.artifactdir = artifact_dir
312	315	context.set_message_handler(message_handler)
313	316
314	317	# Load the project manually

tests/artifactcache/push.py

---

...	...	@@ -51,7 +51,7 @@ def test_push(cli, tmpdir, datafiles):
51	51	# Set up an artifact cache.
52	52	with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
53	53	# Configure artifact share
54		- artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
	54	+ rootcache_dir = os.path.join(str(tmpdir), 'cache')
55	55	user_config_file = str(tmpdir.join('buildstream.conf'))
56	56	user_config = {
57	57	'scheduler': {
...	...	@@ -60,7 +60,8 @@ def test_push(cli, tmpdir, datafiles):
60	60	'artifacts': {
61	61	'url': share.repo,
62	62	'push': True,
63		- }
	63	+ },
	64	+ 'rootcachedir': rootcache_dir
64	65	}
65	66
66	67	# Write down the user configuration file
...	...	@@ -69,7 +70,6 @@ def test_push(cli, tmpdir, datafiles):
69	70	# Fake minimal context
70	71	context = Context()
71	72	context.load(config=user_config_file)
72		- context.artifactdir = artifact_dir
73	73	context.set_message_handler(message_handler)
74	74
75	75	# Load the project manually
...	...	@@ -89,7 +89,7 @@ def test_push(cli, tmpdir, datafiles):
89	89	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
90	90	process = multiprocessing.Process(target=_queue_wrapper,
91	91	args=(_test_push, queue, user_config_file, project_dir,
92		- artifact_dir, 'target.bst', element_key))
	92	+ 'target.bst', element_key))
93	93
94	94	try:
95	95	# Keep SIGINT blocked in the child process
...	...	@@ -106,12 +106,10 @@ def test_push(cli, tmpdir, datafiles):
106	106	assert share.has_artifact('test', 'target.bst', element_key)
107	107
108	108
109		-def _test_push(user_config_file, project_dir, artifact_dir,
110		- element_name, element_key, queue):
	109	+def _test_push(user_config_file, project_dir, element_name, element_key, queue):
111	110	# Fake minimal context
112	111	context = Context()
113	112	context.load(config=user_config_file)
114		- context.artifactdir = artifact_dir
115	113	context.set_message_handler(message_handler)
116	114
117	115	# Load the project manually
...	...	@@ -152,7 +150,7 @@ def test_push_directory(cli, tmpdir, datafiles):
152	150	# Set up an artifact cache.
153	151	with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
154	152	# Configure artifact share
155		- artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
	153	+ rootcache_dir = os.path.join(str(tmpdir), 'cache')
156	154	user_config_file = str(tmpdir.join('buildstream.conf'))
157	155	user_config = {
158	156	'scheduler': {
...	...	@@ -161,7 +159,8 @@ def test_push_directory(cli, tmpdir, datafiles):
161	159	'artifacts': {
162	160	'url': share.repo,
163	161	'push': True,
164		- }
	162	+ },
	163	+ 'rootcachedir': rootcache_dir
165	164	}
166	165
167	166	# Write down the user configuration file
...	...	@@ -170,7 +169,6 @@ def test_push_directory(cli, tmpdir, datafiles):
170	169	# Fake minimal context
171	170	context = Context()
172	171	context.load(config=user_config_file)
173		- context.artifactdir = os.path.join(str(tmpdir), 'cache', 'artifacts')
174	172	context.set_message_handler(message_handler)
175	173
176	174	# Load the project and CAS cache
...	...	@@ -182,6 +180,7 @@ def test_push_directory(cli, tmpdir, datafiles):
182	180	# Assert that the element's artifact is cached
183	181	element = project.load_elements(['target.bst'])[0]
184	182	element_key = cli.get_element_key(project_dir, 'target.bst')
	183	+ print(context.casdir)
185	184	assert artifactcache.contains(element, element_key)
186	185
187	186	# Manually setup the CAS remote
...	...	@@ -198,7 +197,7 @@ def test_push_directory(cli, tmpdir, datafiles):
198	197	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
199	198	process = multiprocessing.Process(target=_queue_wrapper,
200	199	args=(_test_push_directory, queue, user_config_file,
201		- project_dir, artifact_dir, artifact_digest))
	200	+ project_dir, artifact_digest))
202	201
203	202	try:
204	203	# Keep SIGINT blocked in the child process
...	...	@@ -216,11 +215,10 @@ def test_push_directory(cli, tmpdir, datafiles):
216	215	assert share.has_object(artifact_digest)
217	216
218	217
219		-def _test_push_directory(user_config_file, project_dir, artifact_dir, artifact_digest, queue):
	218	+def _test_push_directory(user_config_file, project_dir, artifact_digest, queue):
220	219	# Fake minimal context
221	220	context = Context()
222	221	context.load(config=user_config_file)
223		- context.artifactdir = artifact_dir
224	222	context.set_message_handler(message_handler)
225	223
226	224	# Load the project manually
...	...	@@ -254,6 +252,7 @@ def test_push_message(cli, tmpdir, datafiles):
254	252	with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
255	253	# Configure artifact share
256	254	artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
	255	+ rootcache_dir = os.path.join(str(tmpdir), 'cache')
257	256	user_config_file = str(tmpdir.join('buildstream.conf'))
258	257	user_config = {
259	258	'scheduler': {
...	...	@@ -262,7 +261,8 @@ def test_push_message(cli, tmpdir, datafiles):
262	261	'artifacts': {
263	262	'url': share.repo,
264	263	'push': True,
265		- }
	264	+ },
	265	+ 'rootcachedir': rootcache_dir
266	266	}
267	267
268	268	# Write down the user configuration file
...	...	@@ -273,7 +273,7 @@ def test_push_message(cli, tmpdir, datafiles):
273	273	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
274	274	process = multiprocessing.Process(target=_queue_wrapper,
275	275	args=(_test_push_message, queue, user_config_file,
276		- project_dir, artifact_dir))
	276	+ project_dir))
277	277
278	278	try:
279	279	# Keep SIGINT blocked in the child process
...	...	@@ -292,11 +292,10 @@ def test_push_message(cli, tmpdir, datafiles):
292	292	assert share.has_object(message_digest)
293	293
294	294
295		-def _test_push_message(user_config_file, project_dir, artifact_dir, queue):
	295	+def _test_push_message(user_config_file, project_dir, queue):
296	296	# Fake minimal context
297	297	context = Context()
298	298	context.load(config=user_config_file)
299		- context.artifactdir = artifact_dir
300	299	context.set_message_handler(message_handler)
301	300
302	301	# Load the project manually

tests/frontend/pull.py

---

...	...	@@ -62,8 +62,8 @@ def test_push_pull_all(cli, tmpdir, datafiles):
62	62	# Now we've pushed, delete the user's local artifact cache
63	63	# directory and try to redownload it from the share
64	64	#
65		- artifacts = os.path.join(cli.directory, 'artifacts')
66		- shutil.rmtree(artifacts)
	65	+ cas = os.path.join(cli.directory, 'cas')
	66	+ shutil.rmtree(cas)
67	67
68	68	# Assert that nothing is cached locally anymore
69	69	for element_name in all_elements:
...	...	@@ -104,8 +104,8 @@ def test_pull_secondary_cache(cli, tmpdir, datafiles):
104	104	assert_shared(cli, share2, project, 'target.bst')
105	105
106	106	# Delete the user's local artifact cache.
107		- artifacts = os.path.join(cli.directory, 'artifacts')
108		- shutil.rmtree(artifacts)
	107	+ cas = os.path.join(cli.directory, 'cas')
	108	+ shutil.rmtree(cas)
109	109
110	110	# Assert that the element is not cached anymore.
111	111	assert cli.get_element_state(project, 'target.bst') != 'cached'
...	...	@@ -158,8 +158,8 @@ def test_push_pull_specific_remote(cli, tmpdir, datafiles):
158	158	# Now we've pushed, delete the user's local artifact cache
159	159	# directory and try to redownload it from the good_share.
160	160	#
161		- artifacts = os.path.join(cli.directory, 'artifacts')
162		- shutil.rmtree(artifacts)
	161	+ cas = os.path.join(cli.directory, 'cas')
	162	+ shutil.rmtree(cas)
163	163
164	164	result = cli.run(project=project, args=['artifact', 'pull', 'target.bst', '--remote',
165	165	good_share.repo])
...	...	@@ -199,8 +199,8 @@ def test_push_pull_non_strict(cli, tmpdir, datafiles):
199	199	# Now we've pushed, delete the user's local artifact cache
200	200	# directory and try to redownload it from the share
201	201	#
202		- artifacts = os.path.join(cli.directory, 'artifacts')
203		- shutil.rmtree(artifacts)
	202	+ cas = os.path.join(cli.directory, 'cas')
	203	+ shutil.rmtree(cas)
204	204
205	205	# Assert that nothing is cached locally anymore
206	206	for element_name in all_elements:
...	...	@@ -249,8 +249,8 @@ def test_push_pull_track_non_strict(cli, tmpdir, datafiles):
249	249	# Now we've pushed, delete the user's local artifact cache
250	250	# directory and try to redownload it from the share
251	251	#
252		- artifacts = os.path.join(cli.directory, 'artifacts')
253		- shutil.rmtree(artifacts)
	252	+ cas = os.path.join(cli.directory, 'cas')
	253	+ shutil.rmtree(cas)
254	254
255	255	# Assert that nothing is cached locally anymore
256	256	for element_name in all_elements:
...	...	@@ -285,7 +285,7 @@ def test_push_pull_cross_junction(cli, tmpdir, datafiles):
285	285	result.assert_success()
286	286	assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached'
287	287
288		- cache_dir = os.path.join(project, 'cache', 'artifacts')
	288	+ cache_dir = os.path.join(project, 'cache', 'cas')
289	289	shutil.rmtree(cache_dir)
290	290
291	291	assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'buildable'
...	...	@@ -320,8 +320,8 @@ def test_pull_missing_blob(cli, tmpdir, datafiles):
320	320	# Now we've pushed, delete the user's local artifact cache
321	321	# directory and try to redownload it from the share
322	322	#
323		- artifacts = os.path.join(cli.directory, 'artifacts')
324		- shutil.rmtree(artifacts)
	323	+ cas = os.path.join(cli.directory, 'cas')
	324	+ shutil.rmtree(cas)
325	325
326	326	# Assert that nothing is cached locally anymore
327	327	for element_name in all_elements:

tests/integration/build-tree.py

---

...	...	@@ -158,10 +158,8 @@ def test_buildtree_options(cli, tmpdir, datafiles):
158	158	assert cli.get_element_state(project, element_name) == 'cached'
159	159
160	160	# Discard the cache
161		- cli.configure({
162		- 'artifacts': {'url': share.repo, 'push': True},
163		- 'artifactdir': os.path.join(cli.directory, 'artifacts2')
164		- })
	161	+ shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'artifacts')))
	162	+ shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas')))
165	163	assert cli.get_element_state(project, element_name) != 'cached'
166	164
167	165	# Pull from cache, but do not include buildtrees.

tests/integration/cachedfail.py

---

...	...	@@ -160,7 +160,6 @@ def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
160	160
161	161	# This element should have failed
162	162	assert cli.get_element_state(project, 'element.bst') == 'failed'
163		- # This element should have been pushed to the remote
164	163	assert share.has_artifact('test', 'element.bst', cli.get_element_key(project, 'element.bst'))
165	164
166	165

tests/integration/messages.py

---

...	...	@@ -40,7 +40,7 @@ DATA_DIR = os.path.join(
40	40	@pytest.mark.integration
41	41	@pytest.mark.datafiles(DATA_DIR)
42	42	@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
43		-def test_disable_message_lines(cli, tmpdir, datafiles):
	43	+def test_disable_message_lines(cli, tmpdir, datafiles, integration_cache):
44	44	project = os.path.join(datafiles.dirname, datafiles.basename)
45	45	element_path = os.path.join(project, 'elements')
46	46	element_name = 'message.bst'
...	...	@@ -66,7 +66,7 @@ def test_disable_message_lines(cli, tmpdir, datafiles):
66	66	assert 'echo "Silly message"' in result.stderr
67	67
68	68	# Let's now build it again, but with --message-lines 0
69		- cli.remove_artifact_from_cache(project, element_name)
	69	+ cli.remove_artifact_from_cache(project, element_name, cache_dir=integration_cache.root)
70	70	result = cli.run(project=project, args=["--message-lines", "0",
71	71	"build", element_name])
72	72	result.assert_success()
...	...	@@ -76,7 +76,7 @@ def test_disable_message_lines(cli, tmpdir, datafiles):
76	76	@pytest.mark.integration
77	77	@pytest.mark.datafiles(DATA_DIR)
78	78	@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
79		-def test_disable_error_lines(cli, tmpdir, datafiles):
	79	+def test_disable_error_lines(cli, tmpdir, datafiles, integration_cache):
80	80	project = os.path.join(datafiles.dirname, datafiles.basename)
81	81	element_path = os.path.join(project, 'elements')
82	82	element_name = 'message.bst'
...	...	@@ -103,7 +103,7 @@ def test_disable_error_lines(cli, tmpdir, datafiles):
103	103	assert "This is a syntax error" in result.stderr
104	104
105	105	# Let's now build it again, but with --error-lines 0
106		- cli.remove_artifact_from_cache(project, element_name)
	106	+ cli.remove_artifact_from_cache(project, element_name, cache_dir=integration_cache.root)
107	107	result = cli.run(project=project, args=["--error-lines", "0",
108	108	"build", element_name])
109	109	result.assert_main_error(ErrorDomain.STREAM, None)

tests/integration/pullbuildtrees.py

---

...	...	@@ -19,9 +19,10 @@ DATA_DIR = os.path.join(
19	19	# cleared as just forcefully removing the refpath leaves dangling objects.
20	20	def default_state(cli, tmpdir, share):
21	21	shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
	22	+ shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
22	23	cli.configure({
23	24	'artifacts': {'url': share.repo, 'push': False},
24		- 'artifactdir': os.path.join(str(tmpdir), 'artifacts'),
	25	+ 'rootcachedir': str(tmpdir),
25	26	'cache': {'pull-buildtrees': False},
26	27	})
27	28
...	...	@@ -42,7 +43,7 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
42	43	create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3:
43	44	cli2.configure({
44	45	'artifacts': {'url': share1.repo, 'push': True},
45		- 'artifactdir': os.path.join(str(tmpdir), 'artifacts')
	46	+ 'rootcachedir': str(tmpdir),
46	47	})
47	48
48	49	# Build autotools element, checked pushed, delete local
...	...	@@ -79,7 +80,7 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
79	80	assert os.path.isdir(buildtreedir)
80	81	default_state(cli2, tmpdir, share1)
81	82
82		- # Pull artifact with pullbuildtrees set in user config, then assert
	83	+ # Pull artifact with pullbuildtrees set in user onfig, then assert
83	84	# that pulling with the same user config doesn't creates a pull job,
84	85	# or when buildtrees cli flag is set.
85	86	cli2.configure({'cache': {'pull-buildtrees': True}})

tests/integration/source-determinism.py

---

...	...	@@ -94,9 +94,7 @@ def test_deterministic_source_umask(cli, tmpdir, datafiles, kind, integration_ca
94	94	return f.read()
95	95	finally:
96	96	os.umask(old_umask)
97		- cache_dir = integration_cache.artifacts
98		- cli.remove_artifact_from_cache(project, element_name,
99		- cache_dir=cache_dir)
	97	+ cli.remove_artifact_from_cache(project, element_name, cache_dir=integration_cache.root)
100	98
101	99	assert get_value_for_umask(0o022) == get_value_for_umask(0o077)
102	100
...	...	@@ -156,8 +154,6 @@ def test_deterministic_source_local(cli, tmpdir, datafiles, integration_cache):
156	154	with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
157	155	return f.read()
158	156	finally:
159		- cache_dir = integration_cache.artifacts
160		- cli.remove_artifact_from_cache(project, element_name,
161		- cache_dir=cache_dir)
	157	+ cli.remove_artifact_from_cache(project, element_name, cache_dir=integration_cache.root)
162	158
163	159	assert get_value_for_mask(0o7777) == get_value_for_mask(0o0700)

tests/testutils/runcli.py

---

...	...	@@ -247,15 +247,13 @@ class Cli():
247	247	*, cache_dir=None):
248	248	# Read configuration to figure out where artifacts are stored
249	249	if not cache_dir:
250		- default = os.path.join(project, 'cache', 'artifacts')
251		-
252		- if self.config is not None:
253		- cache_dir = self.config.get('artifactdir', default)
254		- else:
255		- cache_dir = default
	250	+ cache_dir = os.path.join(project, 'cache')
256	251
257	252	cache_dir = os.path.join(cache_dir, 'cas', 'refs', 'heads')
258	253
	254	+ # replace forward slashes
	255	+ element_name = element_name.replace('/', '-')
	256	+
259	257	cache_dir = os.path.splitext(os.path.join(cache_dir, 'test', element_name))[0]
260	258	shutil.rmtree(cache_dir)
261	259
...	...	@@ -553,11 +551,21 @@ def cli_integration(tmpdir, integration_cache):
553	551	# We want to cache sources for integration tests more permanently,
554	552	# to avoid downloading the huge base-sdk repeatedly
555	553	fixture.configure({
	554	+ 'rootcachedir': integration_cache.root,
556	555	'sourcedir': integration_cache.sources,
557		- 'artifactdir': integration_cache.artifacts
558	556	})
559	557
560		- return fixture
	558	+ yield fixture
	559	+
	560	+ # remove following folders if necessary
	561	+ try:
	562	+ shutil.rmtree(os.path.join(integration_cache.root, 'build'))
	563	+ except FileNotFoundError:
	564	+ pass
	565	+ try:
	566	+ shutil.rmtree(os.path.join(integration_cache.root, 'tmp'))
	567	+ except FileNotFoundError:
	568	+ pass
561	569
562	570
563	571	@contextmanager
...	...	@@ -597,10 +605,8 @@ def configured(directory, config=None):
597	605
598	606	if not config.get('sourcedir', False):
599	607	config['sourcedir'] = os.path.join(directory, 'sources')
600		- if not config.get('builddir', False):
601		- config['builddir'] = os.path.join(directory, 'build')
602		- if not config.get('artifactdir', False):
603		- config['artifactdir'] = os.path.join(directory, 'artifacts')
	608	+ if not config.get('rootcachedir', False):
	609	+ config['rootcachedir'] = directory
604	610	if not config.get('logdir', False):
605	611	config['logdir'] = os.path.join(directory, 'logs')
606	612

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.