[Notes] [Git][BuildStream/buildstream][raoul/870-root-cache-dir] CASCache: Move cache check methods here

[Raoul Hidalgo Charman <gitlab mg gitlab com>]

Title: GitLab

Raoul Hidalgo Charman pushed to branch raoul/870-root-cache-dir at BuildStream / buildstream

Commits:

• c194907e
  by Raoul Hidalgo Charman at 2019-01-31T10:32:35Z

  CASCache: Move cache check methods here
  
  Part of #870
  

16 changed files:

• buildstream/_artifactcache.py
• buildstream/_cas/__init__.py
• buildstream/_cas/cascache.py
• buildstream/_cas/casserver.py
• buildstream/_context.py
• buildstream/_frontend/cli.py
• buildstream/_frontend/status.py
• buildstream/_frontend/widget.py
• buildstream/_scheduler/jobs/cachesizejob.py
• buildstream/_scheduler/jobs/cleanupjob.py
• buildstream/_scheduler/jobs/job.py
• buildstream/utils.py
• tests/artifactcache/cache_size.py
• tests/artifactcache/expiry.py
• tests/testutils/artifactshare.py
• tests/testutils/runcli.py

Changes:

buildstream/_artifactcache.py

---

...	...	@@ -46,39 +46,6 @@ class ArtifactCacheSpec(CASRemoteSpec):
46	46	pass
47	47
48	48
49		-# ArtifactCacheUsage
50		-#
51		-# A simple object to report the current artifact cache
52		-# usage details.
53		-#
54		-# Note that this uses the user configured cache quota
55		-# rather than the internal quota with protective headroom
56		-# removed, to provide a more sensible value to display to
57		-# the user.
58		-#
59		-# Args:
60		-# artifacts (ArtifactCache): The artifact cache to get the status of
61		-#
62		-class ArtifactCacheUsage():
63		-
64		- def __init__(self, artifacts):
65		- context = artifacts.context
66		- self.quota_config = context.config_cache_quota # Configured quota
67		- self.quota_size = artifacts._cache_quota_original # Resolved cache quota in bytes
68		- self.used_size = artifacts.get_cache_size() # Size used by artifacts in bytes
69		- self.used_percent = 0 # Percentage of the quota used
70		- if self.quota_size is not None:
71		- self.used_percent = int(self.used_size * 100 / self.quota_size)
72		-
73		- # Formattable into a human readable string
74		- #
75		- def __str__(self):
76		- return "{} / {} ({}%)" \
77		- .format(utils._pretty_size(self.used_size, dec_places=1),
78		- self.quota_config,
79		- self.used_percent)
80		-
81		-
82	49	# An ArtifactCache manages artifacts.
83	50	#
84	51	# Args:
...	...	@@ -100,6 +67,7 @@ class ArtifactCache():
100	67	self._cache_quota_original = None # The cache quota as specified by the user, in bytes
101	68	self._cache_quota_headroom = None # The headroom in bytes before reaching the quota or full disk
102	69	self._cache_lower_threshold = None # The target cache size for a cleanup
	70	+
103	71	self._remotes_setup = False # Check to prevent double-setup of remotes
104	72
105	73	# Per-project list of _CASRemote instances.
...	...	@@ -266,7 +234,7 @@ class ArtifactCache():
266	234	space_saved = 0
267	235
268	236	# Start off with an announcement with as much info as possible
269		- volume_size, volume_avail = self._get_cache_volume_size()
	237	+ volume_size, volume_avail = self.cas._get_cache_volume_size()
270	238	self._message(MessageType.STATUS, "Starting cache cleanup",
271	239	detail=("Elements required by the current build plan: {}\n" +
272	240	"User specified quota: {} ({})\n" +
...	...	@@ -275,7 +243,7 @@ class ArtifactCache():
275	243	.format(len(self._required_elements),
276	244	context.config_cache_quota,
277	245	utils._pretty_size(self._cache_quota_original, dec_places=2),
278		- utils._pretty_size(self.get_cache_size(), dec_places=2),
	246	+ utils._pretty_size(self.cas.get_cache_size(), dec_places=2),
279	247	utils._pretty_size(volume_size, dec_places=2),
280	248	utils._pretty_size(volume_avail, dec_places=2)))
281	249
...	...	@@ -292,9 +260,10 @@ class ArtifactCache():
292	260	])
293	261
294	262	# Do a real computation of the cache size once, just in case
295		- self.compute_cache_size()
	263	+ usage = self.cas.compute_cache_size()
	264	+ self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
296	265
297		- while self.get_cache_size() >= self._cache_lower_threshold:
	266	+ while self.cas.get_cache_size() >= self._cache_lower_threshold:
298	267	try:
299	268	to_remove = artifacts.pop(0)
300	269	except IndexError:
...	...	@@ -311,7 +280,7 @@ class ArtifactCache():
311	280	"Please increase the cache-quota in {} and/or make more disk space."
312	281	.format(removed_ref_count,
313	282	utils._pretty_size(space_saved, dec_places=2),
314		- utils._pretty_size(self.get_cache_size(), dec_places=2),
	283	+ utils._pretty_size(self.cas.get_cache_size(), dec_places=2),
315	284	len(self._required_elements),
316	285	(context.config_origin or default_conf)))
317	286
...	...	@@ -337,7 +306,7 @@ class ArtifactCache():
337	306	to_remove))
338	307
339	308	# Remove the size from the removed size
340		- self.set_cache_size(self._cache_size - size)
	309	+ self.cas.set_cache_size(self.cas._cache_size - size)
341	310
342	311	# User callback
343	312	#
...	...	@@ -353,29 +322,9 @@ class ArtifactCache():
353	322	"Cache usage is now: {}")
354	323	.format(removed_ref_count,
355	324	utils._pretty_size(space_saved, dec_places=2),
356		- utils._pretty_size(self.get_cache_size(), dec_places=2)))
	325	+ utils._pretty_size(self.cas.get_cache_size(), dec_places=2)))
357	326
358		- return self.get_cache_size()
359		-
360		- # compute_cache_size()
361		- #
362		- # Computes the real artifact cache size by calling
363		- # the abstract calculate_cache_size() method.
364		- #
365		- # Returns:
366		- # (int): The size of the artifact cache.
367		- #
368		- def compute_cache_size(self):
369		- old_cache_size = self._cache_size
370		- new_cache_size = self.cas.calculate_cache_size()
371		-
372		- if old_cache_size != new_cache_size:
373		- self._cache_size = new_cache_size
374		-
375		- usage = ArtifactCacheUsage(self)
376		- self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
377		-
378		- return self._cache_size
	327	+ return self.cas.get_cache_size()
379	328
380	329	# add_artifact_size()
381	330	#
...	...	@@ -386,51 +335,10 @@ class ArtifactCache():
386	335	# artifact_size (int): The size to add.
387	336	#
388	337	def add_artifact_size(self, artifact_size):
389		- cache_size = self.get_cache_size()
	338	+ cache_size = self.cas.get_cache_size()
390	339	cache_size += artifact_size
391	340
392		- self.set_cache_size(cache_size)
393		-
394		- # get_cache_size()
395		- #
396		- # Fetches the cached size of the cache, this is sometimes
397		- # an estimate and periodically adjusted to the real size
398		- # when a cache size calculation job runs.
399		- #
400		- # When it is an estimate, the value is either correct, or
401		- # it is greater than the actual cache size.
402		- #
403		- # Returns:
404		- # (int) An approximation of the artifact cache size, in bytes.
405		- #
406		- def get_cache_size(self):
407		-
408		- # If we don't currently have an estimate, figure out the real cache size.
409		- if self._cache_size is None:
410		- stored_size = self._read_cache_size()
411		- if stored_size is not None:
412		- self._cache_size = stored_size
413		- else:
414		- self.compute_cache_size()
415		-
416		- return self._cache_size
417		-
418		- # set_cache_size()
419		- #
420		- # Forcefully set the overall cache size.
421		- #
422		- # This is used to update the size in the main process after
423		- # having calculated in a cleanup or a cache size calculation job.
424		- #
425		- # Args:
426		- # cache_size (int): The size to set.
427		- #
428		- def set_cache_size(self, cache_size):
429		-
430		- assert cache_size is not None
431		-
432		- self._cache_size = cache_size
433		- self._write_cache_size(self._cache_size)
	341	+ self.cas.set_cache_size(cache_size)
434	342
435	343	# full()
436	344	#
...	...	@@ -443,7 +351,7 @@ class ArtifactCache():
443	351	#
444	352	def full(self):
445	353
446		- if self.get_cache_size() > self._cache_quota:
	354	+ if self.cas.get_cache_size() > self.cas._cache_quota:
447	355	return True
448	356
449	357	_, volume_avail = self._get_cache_volume_size()
...	...	@@ -896,44 +804,6 @@ class ArtifactCache():
896	804	with self.context.timed_activity("Initializing remote caches", silent_nested=True):
897	805	self.initialize_remotes(on_failure=remote_failed)
898	806
899		- # _write_cache_size()
900		- #
901		- # Writes the given size of the artifact to the cache's size file
902		- #
903		- # Args:
904		- # size (int): The size of the artifact cache to record
905		- #
906		- def _write_cache_size(self, size):
907		- assert isinstance(size, int)
908		- size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
909		- with utils.save_file_atomic(size_file_path, "w") as f:
910		- f.write(str(size))
911		-
912		- # _read_cache_size()
913		- #
914		- # Reads and returns the size of the artifact cache that's stored in the
915		- # cache's size file
916		- #
917		- # Returns:
918		- # (int): The size of the artifact cache, as recorded in the file
919		- #
920		- def _read_cache_size(self):
921		- size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
922		-
923		- if not os.path.exists(size_file_path):
924		- return None
925		-
926		- with open(size_file_path, "r") as f:
927		- size = f.read()
928		-
929		- try:
930		- num_size = int(size)
931		- except ValueError as e:
932		- raise ArtifactError("Size '{}' parsed from '{}' was not an integer".format(
933		- size, size_file_path)) from e
934		-
935		- return num_size
936		-
937	807	# _calculate_cache_quota()
938	808	#
939	809	# Calculates and sets the cache quota and lower threshold based on the
...	...	@@ -959,9 +829,8 @@ class ArtifactCache():
959	829	"{}\nPlease specify the value in bytes or as a % of full disk space.\n"
960	830	"\nValid values are, for example: 800M 10G 1T 50%\n"
961	831	.format(str(e))) from e
962		-
963		- total_size, available_space = self._get_cache_volume_size()
964		- cache_size = self.get_cache_size()
	832	+ total_size, available_space = self.cas._get_cache_volume_size()
	833	+ cache_size = self.cas.get_cache_size()
965	834
966	835	# Ensure system has enough storage for the cache_quota
967	836	#
...	...	@@ -1016,22 +885,6 @@ class ArtifactCache():
1016	885	self._cache_quota = cache_quota - self._cache_quota_headroom
1017	886	self._cache_lower_threshold = self._cache_quota / 2
1018	887
1019		- # _get_cache_volume_size()
1020		- #
1021		- # Get the available space and total space for the volume on
1022		- # which the artifact cache is located.
1023		- #
1024		- # Returns:
1025		- # (int): The total number of bytes on the volume
1026		- # (int): The number of available bytes on the volume
1027		- #
1028		- # NOTE: We use this stub to allow the test cases
1029		- # to override what an artifact cache thinks
1030		- # about it's disk size and available bytes.
1031		- #
1032		- def _get_cache_volume_size(self):
1033		- return utils._get_volume_size(self.context.artifactdir)
1034		-
1035	888
1036	889	# _configured_remote_artifact_cache_specs():
1037	890	#

buildstream/_cas/__init__.py

---

...	...	@@ -17,5 +17,5 @@
17	17	# Authors:
18	18	# Tristan Van Berkom <tristan vanberkom codethink co uk>
19	19
20		-from .cascache import CASCache
	20	+from .cascache import CASCache, CASCacheUsage
21	21	from .casremote import CASRemote, CASRemoteSpec

buildstream/_cas/cascache.py

---

...	...	@@ -36,20 +36,61 @@ from .._exceptions import CASCacheError
36	36	from .casremote import BlobNotFound, _CASBatchRead, _CASBatchUpdate
37	37
38	38
	39	+CACHE_SIZE_FILE = "cache_size"
	40	+
	41	+
	42	+# CASCacheUsage
	43	+#
	44	+# A simple object to report the current CAS cache usage details.
	45	+#
	46	+# Note that this uses the user configured cache quota
	47	+# rather than the internal quota with protective headroom
	48	+# removed, to provide a more sensible value to display to
	49	+# the user.
	50	+#
	51	+# Args:
	52	+# cas (CASCache): The CAS cache to get the status of
	53	+#
	54	+class CASCacheUsage():
	55	+
	56	+ def __init__(self, cas):
	57	+ self.quota_config = cas._config_cache_quota # Configured quota
	58	+ self.quota_size = cas._cache_quota_original # Resolved cache quota in bytes
	59	+ self.used_size = cas.get_cache_size() # Size used by artifacts in bytes
	60	+ self.used_percent = 0 # Percentage of the quota used
	61	+ if self.quota_size is not None:
	62	+ self.used_percent = int(self.used_size * 100 / self.quota_size)
	63	+
	64	+ # Formattable into a human readable string
	65	+ #
	66	+ def __str__(self):
	67	+ return "{} / {} ({}%)" \
	68	+ .format(utils._pretty_size(self.used_size, dec_places=1),
	69	+ self.quota_config,
	70	+ self.used_percent)
	71	+
	72	+
39	73	# A CASCache manages a CAS repository as specified in the Remote Execution API.
40	74	#
41	75	# Args:
42	76	# path (str): The root directory for the CAS repository
	77	+# cache_quota (int): User configured cache quota
43	78	#
44	79	class CASCache():
45	80
46		- def __init__(self, path):
	81	+ def __init__(self, path, cache_quota):
47	82	self.casdir = os.path.join(path, 'cas')
48	83	self.tmpdir = os.path.join(path, 'tmp')
49	84	os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True)
50	85	os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
51	86	os.makedirs(self.tmpdir, exist_ok=True)
52	87
	88	+ self._config_cache_quota = cache_quota
	89	+ self._cache_size = None # The current cache size, sometimes it's an estimate
	90	+ self._cache_quota = None # The cache quota
	91	+ self._cache_quota_original = None # The cache quota as specified by the user, in bytes
	92	+ self._cache_lower_threshold = None # The target cache size for a cleanup
	93	+
53	94	# preflight():
54	95	#
55	96	# Preflight check.
...	...	@@ -587,6 +628,65 @@ class CASCache():
587	628	reachable = set()
588	629	self._reachable_refs_dir(reachable, tree, update_mtime=True)
589	630
	631	+ # compute_cache_size()
	632	+ #
	633	+ # Computes the real artifact cache size by calling
	634	+ # the abstract calculate_cache_size() method.
	635	+ #
	636	+ # Returns:
	637	+ # (int): The size of the artifact cache.
	638	+ #
	639	+ def compute_cache_size(self):
	640	+ old_cache_size = self._cache_size
	641	+ new_cache_size = self.calculate_cache_size()
	642	+
	643	+ if old_cache_size != new_cache_size:
	644	+ self._cache_size = new_cache_size
	645	+
	646	+ return self._cache_size
	647	+
	648	+ # get_cache_size()
	649	+ #
	650	+ # Fetches the cached size of the cache, this is sometimes
	651	+ # an estimate and periodically adjusted to the real size
	652	+ # when a cache size calculation job runs.
	653	+ #
	654	+ # When it is an estimate, the value is either correct, or
	655	+ # it is greater than the actual cache size.
	656	+ #
	657	+ # Returns:
	658	+ # (int) An approximation of the artifact cache size, in bytes.
	659	+ #
	660	+ def get_cache_size(self):
	661	+
	662	+ # If we don't currently have an estimate, figure out the real cache size.
	663	+ if self._cache_size is None:
	664	+ stored_size = self._read_cache_size()
	665	+ if stored_size is not None:
	666	+ self._cache_size = stored_size
	667	+ else:
	668	+ self._cache_size = self.compute_cache_size()
	669	+ # self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
	670	+
	671	+ return self._cache_size
	672	+
	673	+ # set_cache_size()
	674	+ #
	675	+ # Forcefully set the overall cache size.
	676	+ #
	677	+ # This is used to update the size in the main process after
	678	+ # having calculated in a cleanup or a cache size calculation job.
	679	+ #
	680	+ # Args:
	681	+ # cache_size (int): The size to set.
	682	+ #
	683	+ def set_cache_size(self, cache_size):
	684	+
	685	+ assert cache_size is not None
	686	+
	687	+ self._cache_size = cache_size
	688	+ self._write_cache_size(self._cache_size)
	689	+
590	690	################################################
591	691	# Local Private Methods #
592	692	################################################
...	...	@@ -1015,6 +1115,60 @@ class CASCache():
1015	1115	# Send final batch
1016	1116	batch.send()
1017	1117
	1118	+ # _read_cache_size()
	1119	+ #
	1120	+ # Reads and returns the size of the artifact cache that's stored in the
	1121	+ # cache's size file
	1122	+ #
	1123	+ # Returns:
	1124	+ # (int): The size of the artifact cache, as recorded in the file
	1125	+ #
	1126	+ def _read_cache_size(self):
	1127	+ size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
	1128	+
	1129	+ if not os.path.exists(size_file_path):
	1130	+ return None
	1131	+
	1132	+ with open(size_file_path, "r") as f:
	1133	+ size = f.read()
	1134	+
	1135	+ try:
	1136	+ num_size = int(size)
	1137	+ except ValueError as e:
	1138	+ raise CASCacheError("Size '{}' parsed from '{}' was not an integer".format(
	1139	+ size, size_file_path)) from e
	1140	+
	1141	+ return num_size
	1142	+
	1143	+ # _write_cache_size()
	1144	+ #
	1145	+ # Writes the given size of the artifact to the cache's size file
	1146	+ #
	1147	+ # Args:
	1148	+ # size (int): The size of the artifact cache to record
	1149	+ #
	1150	+ def _write_cache_size(self, size):
	1151	+ assert isinstance(size, int)
	1152	+ size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
	1153	+ with utils.save_file_atomic(size_file_path, "w") as f:
	1154	+ f.write(str(size))
	1155	+
	1156	+ # _get_cache_volume_size()
	1157	+ #
	1158	+ # Get the available space and total space for the volume on
	1159	+ # which the artifact cache is located.
	1160	+ #
	1161	+ # Returns:
	1162	+ # (int): The total number of bytes on the volume
	1163	+ # (int): The number of available bytes on the volume
	1164	+ #
	1165	+ # NOTE: We use this stub to allow the test cases
	1166	+ # to override what an artifact cache thinks
	1167	+ # about it's disk size and available bytes.
	1168	+ #
	1169	+ def _get_cache_volume_size(self):
	1170	+ return utils._get_volume_size(self.casdir)
	1171	+
1018	1172
1019	1173	def _grouper(iterable, n):
1020	1174	while True:

buildstream/_cas/casserver.py

---

...	...	@@ -61,7 +61,7 @@ class ArtifactTooLargeException(Exception):
61	61	def create_server(repo, *, enable_push,
62	62	max_head_size=int(10e9),
63	63	min_head_size=int(2e9)):
64		- cas = CASCache(os.path.abspath(repo))
	64	+ cas = CASCache(os.path.abspath(repo), max_head_size)
65	65
66	66	# Use max_workers default from Python 3.5+
67	67	max_workers = (os.cpu_count() or 1) * 5

buildstream/_context.py

---

...	...	@@ -30,8 +30,8 @@ from . import _yaml
30	30	from ._exceptions import LoadError, LoadErrorReason, BstError
31	31	from ._message import Message, MessageType
32	32	from ._profile import Topics, profile_start, profile_end
33		-from ._artifactcache import ArtifactCache, ArtifactCacheUsage
34		-from ._cas import CASCache
	33	+from ._artifactcache import ArtifactCache
	34	+from ._cas import CASCache, CASCacheUsage
35	35	from ._workspaces import Workspaces, WorkspaceProjectCache
36	36	from .plugin import _plugin_lookup
37	37	from .sandbox import SandboxRemote
...	...	@@ -315,15 +315,15 @@ class Context():
315	315
316	316	return self._artifactcache
317	317
318		- # get_artifact_cache_usage()
	318	+ # get_cache_usage()
319	319	#
320	320	# Fetches the current usage of the artifact cache
321	321	#
322	322	# Returns:
323		- # (ArtifactCacheUsage): The current status
	323	+ # (CASCacheUsage): The current status
324	324	#
325		- def get_artifact_cache_usage(self):
326		- return ArtifactCacheUsage(self.artifactcache)
	325	+ def get_cache_usage(self):
	326	+ return CASCacheUsage(self.get_cascache())
327	327
328	328	# add_project():
329	329	#
...	...	@@ -690,7 +690,7 @@ class Context():
690	690
691	691	def get_cascache(self):
692	692	if self._cascache is None:
693		- self._cascache = CASCache(self.rootcachedir)
	693	+ self._cascache = CASCache(self.rootcachedir, self.config_cache_quota)
694	694	return self._cascache
695	695
696	696

buildstream/_frontend/cli.py

---

...	...	@@ -11,7 +11,6 @@ from .._exceptions import BstError, LoadError, AppError
11	11	from .._versions import BST_FORMAT_VERSION
12	12	from .complete import main_bashcomplete, complete_path, CompleteUnhandled
13	13
14		-
15	14	##################################################################
16	15	# Override of click's main entry point #
17	16	##################################################################

buildstream/_frontend/status.py

---

...	...	@@ -404,7 +404,7 @@ class _StatusHeader():
404	404	#
405	405	# ~~~~~~ cache: 69% ~~~~~~
406	406	#
407		- usage = self._context.get_artifact_cache_usage()
	407	+ usage = self._context.get_cache_usage()
408	408	usage_percent = '{}%'.format(usage.used_percent)
409	409
410	410	size = 21

buildstream/_frontend/widget.py

---

...	...	@@ -452,7 +452,7 @@ class LogLine(Widget):
452	452	values["Session Start"] = starttime.strftime('%A, %d-%m-%Y at %H:%M:%S')
453	453	values["Project"] = "{} ({})".format(project.name, project.directory)
454	454	values["Targets"] = ", ".join([t.name for t in stream.targets])
455		- values["Cache Usage"] = "{}".format(context.get_artifact_cache_usage())
	455	+ values["Cache Usage"] = "{}".format(context.get_cache_usage())
456	456	text += self._format_values(values)
457	457
458	458	# User configurations

buildstream/_scheduler/jobs/cachesizejob.py

---

...	...	@@ -25,14 +25,14 @@ class CacheSizeJob(Job):
25	25	self._complete_cb = complete_cb
26	26
27	27	context = self._scheduler.context
28		- self._artifacts = context.artifactcache
	28	+ self._cas = context.get_cascache()
29	29
30	30	def child_process(self):
31		- return self._artifacts.compute_cache_size()
	31	+ return self._cas.compute_cache_size()
32	32
33	33	def parent_complete(self, status, result):
34	34	if status == JobStatus.OK:
35		- self._artifacts.set_cache_size(result)
	35	+ self._cas.set_cache_size(result)
36	36
37	37	if self._complete_cb:
38	38	self._complete_cb(status, result)

buildstream/_scheduler/jobs/cleanupjob.py

---

...	...	@@ -25,27 +25,27 @@ class CleanupJob(Job):
25	25	self._complete_cb = complete_cb
26	26
27	27	context = self._scheduler.context
	28	+ self._cas = context.get_cascache()
28	29	self._artifacts = context.artifactcache
29	30
30	31	def child_process(self):
31	32	def progress():
32	33	self.send_message('update-cache-size',
33		- self._artifacts.get_cache_size())
	34	+ self._cas.get_cache_size())
34	35	return self._artifacts.clean(progress)
35	36
36	37	def handle_message(self, message_type, message):
37		-
38	38	# Update the cache size in the main process as we go,
39	39	# this provides better feedback in the UI.
40	40	if message_type == 'update-cache-size':
41		- self._artifacts.set_cache_size(message)
	41	+ self._cas.set_cache_size(message)
42	42	return True
43	43
44	44	return False
45	45
46	46	def parent_complete(self, status, result):
47	47	if status == JobStatus.OK:
48		- self._artifacts.set_cache_size(result)
	48	+ self._cas.set_cache_size(result)
49	49
50	50	if self._complete_cb:
51	51	self._complete_cb(status, result)

buildstream/_scheduler/jobs/job.py

---

...	...	@@ -117,6 +117,8 @@ class Job():
117	117	self._logfile = logfile
118	118	self._task_id = None
119	119
	120	+ print("job init")
	121	+
120	122	# spawn()
121	123	#
122	124	# Spawns the job.
...	...	@@ -381,7 +383,6 @@ class Job():
381	383	# queue (multiprocessing.Queue): The message queue for IPC
382	384	#
383	385	def _child_action(self, queue):
384		-
385	386	# This avoids some SIGTSTP signals from grandchildren
386	387	# getting propagated up to the master process
387	388	os.setsid()

buildstream/utils.py

---

...	...	@@ -43,6 +43,7 @@ from . import _signals
43	43	from ._exceptions import BstError, ErrorDomain
44	44	from ._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
45	45
	46	+
46	47	# The magic number for timestamps: 2011-11-11 11:11:11
47	48	_magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11])
48	49

tests/artifactcache/cache_size.py

---

...	...	@@ -50,14 +50,15 @@ def test_cache_size_write(cli, tmpdir):
50	50
51	51	# Artifact cache must be in a known place
52	52	artifactdir = os.path.join(project_dir, "artifacts")
53		- cli.configure({"artifactdir": artifactdir})
	53	+ casdir = os.path.join(project_dir, "cas")
	54	+ cli.configure({"rootcachedir": project_dir})
54	55
55	56	# Build, to populate the cache
56	57	res = cli.run(project=project_dir, args=["build", "test.bst"])
57	58	res.assert_success()
58	59
59	60	# Inspect the artifact cache
60		- sizefile = os.path.join(artifactdir, CACHE_SIZE_FILE)
	61	+ sizefile = os.path.join(casdir, CACHE_SIZE_FILE)
61	62	assert os.path.isfile(sizefile)
62	63	with open(sizefile, "r") as f:
63	64	size_data = f.read()
...	...	@@ -80,7 +81,7 @@ def test_quota_over_1024T(cli, tmpdir):
80	81	_yaml.dump({'name': 'main'}, str(project.join("project.conf")))
81	82
82	83	volume_space_patch = mock.patch(
83		- "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
	84	+ "buildstream._cas.CASCache._get_cache_volume_size",
84	85	autospec=True,
85	86	return_value=(1025 * TiB, 1025 * TiB)
86	87	)

tests/artifactcache/expiry.py

---

...	...	@@ -372,13 +372,13 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
372	372	total_space = 10000
373	373
374	374	volume_space_patch = mock.patch(
375		- "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
	375	+ "buildstream._cas.CASCache._get_cache_volume_size",
376	376	autospec=True,
377	377	return_value=(total_space, free_space),
378	378	)
379	379
380	380	cache_size_patch = mock.patch(
381		- "buildstream._artifactcache.ArtifactCache.get_cache_size",
	381	+ "buildstream._cas.CASCache.get_cache_size",
382	382	autospec=True,
383	383	return_value=0,
384	384	)

tests/testutils/artifactshare.py

---

...	...	@@ -49,7 +49,7 @@ class ArtifactShare():
49	49
50	50	os.makedirs(self.repodir)
51	51
52		- self.cas = CASCache(self.repodir)
	52	+ self.cas = CASCache(self.repodir, max_head_size)
53	53
54	54	self.total_space = total_space
55	55	self.free_space = free_space

tests/testutils/runcli.py

---

...	...	@@ -336,6 +336,7 @@ class Cli():
336	336	exception = None
337	337	exit_code = 0
338	338
	339	+
339	340	# Temporarily redirect sys.stdin to /dev/null to ensure that
340	341	# Popen doesn't attempt to read pytest's dummy stdin.
341	342	old_stdin = sys.stdin
...	...	@@ -345,6 +346,7 @@ class Cli():
345	346	capture = MultiCapture(out=True, err=True, in_=False, Capture=capture_kind)
346	347	capture.start_capturing()
347	348
	349	+
348	350	try:
349	351	cli.main(args=args or (), prog_name=cli.name, **extra)
350	352	except SystemExit as e:

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.