[Notes] [Git][BuildStream/buildstream][raoul/870-root-cache-dir] CASCache: Move cache check methods here

[Raoul Hidalgo Charman <gitlab mg gitlab com>]

Title: GitLab

Raoul Hidalgo Charman pushed to branch raoul/870-root-cache-dir at BuildStream / buildstream

Commits:

• 93c6d5c7
  by Raoul Hidalgo Charman at 2019-02-04T16:14:02Z

  CASCache: Move cache check methods here
  
  A lot of code that checks cache quota has been touched.
  
  Part of #870
  

14 changed files:

• buildstream/_artifactcache.py
• buildstream/_cas/__init__.py
• buildstream/_cas/cascache.py
• buildstream/_context.py
• buildstream/_frontend/cli.py
• buildstream/_frontend/status.py
• buildstream/_frontend/widget.py
• buildstream/_scheduler/jobs/cachesizejob.py
• buildstream/_scheduler/jobs/cleanupjob.py
• buildstream/_scheduler/jobs/job.py
• buildstream/utils.py
• tests/artifactcache/cache_size.py
• tests/artifactcache/expiry.py
• tests/testutils/artifactshare.py

Changes:

buildstream/_artifactcache.py

---

...	...	@@ -23,7 +23,7 @@ import string
23	23	from collections.abc import Mapping
24	24
25	25	from .types import _KeyStrength
26		-from ._exceptions import ArtifactError, CASError, LoadError, LoadErrorReason
	26	+from ._exceptions import ArtifactError, CASError
27	27	from ._message import Message, MessageType
28	28	from . import utils
29	29	from . import _yaml
...	...	@@ -46,39 +46,6 @@ class ArtifactCacheSpec(CASRemoteSpec):
46	46	pass
47	47
48	48
49		-# ArtifactCacheUsage
50		-#
51		-# A simple object to report the current artifact cache
52		-# usage details.
53		-#
54		-# Note that this uses the user configured cache quota
55		-# rather than the internal quota with protective headroom
56		-# removed, to provide a more sensible value to display to
57		-# the user.
58		-#
59		-# Args:
60		-# artifacts (ArtifactCache): The artifact cache to get the status of
61		-#
62		-class ArtifactCacheUsage():
63		-
64		- def __init__(self, artifacts):
65		- context = artifacts.context
66		- self.quota_config = context.config_cache_quota # Configured quota
67		- self.quota_size = artifacts._cache_quota_original # Resolved cache quota in bytes
68		- self.used_size = artifacts.get_cache_size() # Size used by artifacts in bytes
69		- self.used_percent = 0 # Percentage of the quota used
70		- if self.quota_size is not None:
71		- self.used_percent = int(self.used_size * 100 / self.quota_size)
72		-
73		- # Formattable into a human readable string
74		- #
75		- def __str__(self):
76		- return "{} / {} ({}%)" \
77		- .format(utils._pretty_size(self.used_size, dec_places=1),
78		- self.quota_config,
79		- self.used_percent)
80		-
81		-
82	49	# An ArtifactCache manages artifacts.
83	50	#
84	51	# Args:
...	...	@@ -91,15 +58,23 @@ class ArtifactCache():
91	58
92	59	self.cas = context.get_cascache()
93	60
	61	+ # check to see if the cache quota exceeds space available
	62	+ if self.cas.cache_quota_exceeded:
	63	+ self._message(MessageType.WARN,
	64	+ "Your system does not have enough available " +
	65	+ "space to support the cache quota specified.",
	66	+ detail=("You have specified a quota of {quota} total disk space.\n" +
	67	+ "The filesystem containing {local_cache_path} only " +
	68	+ "has {available_size} available.")
	69	+ .format(quota=context.config_cache_quota,
	70	+ local_cache_path=context.rootcachedir,
	71	+ available_size=self.cas.available_space))
	72	+
94	73	self.global_remote_specs = []
95	74	self.project_remote_specs = {}
96	75
97	76	self._required_elements = set() # The elements required for this session
98		- self._cache_size = None # The current cache size, sometimes it's an estimate
99		- self._cache_quota = None # The cache quota
100		- self._cache_quota_original = None # The cache quota as specified by the user, in bytes
101		- self._cache_quota_headroom = None # The headroom in bytes before reaching the quota or full disk
102		- self._cache_lower_threshold = None # The target cache size for a cleanup
	77	+
103	78	self._remotes_setup = False # Check to prevent double-setup of remotes
104	79
105	80	# Per-project list of _CASRemote instances.
...	...	@@ -110,8 +85,6 @@ class ArtifactCache():
110	85
111	86	os.makedirs(self.extractdir, exist_ok=True)
112	87
113		- self._calculate_cache_quota()
114		-
115	88	# get_artifact_fullname()
116	89	#
117	90	# Generate a full name for an artifact, including the
...	...	@@ -266,7 +239,7 @@ class ArtifactCache():
266	239	space_saved = 0
267	240
268	241	# Start off with an announcement with as much info as possible
269		- volume_size, volume_avail = self._get_cache_volume_size()
	242	+ volume_size, volume_avail = self.cas._get_cache_volume_size()
270	243	self._message(MessageType.STATUS, "Starting cache cleanup",
271	244	detail=("Elements required by the current build plan: {}\n" +
272	245	"User specified quota: {} ({})\n" +
...	...	@@ -274,8 +247,8 @@ class ArtifactCache():
274	247	"Cache volume: {} total, {} available")
275	248	.format(len(self._required_elements),
276	249	context.config_cache_quota,
277		- utils._pretty_size(self._cache_quota_original, dec_places=2),
278		- utils._pretty_size(self.get_cache_size(), dec_places=2),
	250	+ utils._pretty_size(self.cas._cache_quota, dec_places=2),
	251	+ utils._pretty_size(self.cas.get_cache_size(), dec_places=2),
279	252	utils._pretty_size(volume_size, dec_places=2),
280	253	utils._pretty_size(volume_avail, dec_places=2)))
281	254
...	...	@@ -292,9 +265,10 @@ class ArtifactCache():
292	265	])
293	266
294	267	# Do a real computation of the cache size once, just in case
295		- self.compute_cache_size()
	268	+ usage = self.cas.compute_cache_size()
	269	+ self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
296	270
297		- while self.get_cache_size() >= self._cache_lower_threshold:
	271	+ while self.cas.get_cache_size() >= self.cas._cache_lower_threshold:
298	272	try:
299	273	to_remove = artifacts.pop(0)
300	274	except IndexError:
...	...	@@ -311,7 +285,7 @@ class ArtifactCache():
311	285	"Please increase the cache-quota in {} and/or make more disk space."
312	286	.format(removed_ref_count,
313	287	utils._pretty_size(space_saved, dec_places=2),
314		- utils._pretty_size(self.get_cache_size(), dec_places=2),
	288	+ utils._pretty_size(self.cas.get_cache_size(), dec_places=2),
315	289	len(self._required_elements),
316	290	(context.config_origin or default_conf)))
317	291
...	...	@@ -337,7 +311,7 @@ class ArtifactCache():
337	311	to_remove))
338	312
339	313	# Remove the size from the removed size
340		- self.set_cache_size(self._cache_size - size)
	314	+ self.cas.set_cache_size(self.cas._cache_size - size)
341	315
342	316	# User callback
343	317	#
...	...	@@ -353,29 +327,12 @@ class ArtifactCache():
353	327	"Cache usage is now: {}")
354	328	.format(removed_ref_count,
355	329	utils._pretty_size(space_saved, dec_places=2),
356		- utils._pretty_size(self.get_cache_size(), dec_places=2)))
357		-
358		- return self.get_cache_size()
359		-
360		- # compute_cache_size()
361		- #
362		- # Computes the real artifact cache size by calling
363		- # the abstract calculate_cache_size() method.
364		- #
365		- # Returns:
366		- # (int): The size of the artifact cache.
367		- #
368		- def compute_cache_size(self):
369		- old_cache_size = self._cache_size
370		- new_cache_size = self.cas.calculate_cache_size()
	330	+ utils._pretty_size(self.cas.get_cache_size(), dec_places=2)))
371	331
372		- if old_cache_size != new_cache_size:
373		- self._cache_size = new_cache_size
	332	+ return self.cas.get_cache_size()
374	333
375		- usage = ArtifactCacheUsage(self)
376		- self._message(MessageType.STATUS, "Cache usage recomputed: {}".format(usage))
377		-
378		- return self._cache_size
	334	+ def full(self):
	335	+ return self.cas.full()
379	336
380	337	# add_artifact_size()
381	338	#
...	...	@@ -386,71 +343,10 @@ class ArtifactCache():
386	343	# artifact_size (int): The size to add.
387	344	#
388	345	def add_artifact_size(self, artifact_size):
389		- cache_size = self.get_cache_size()
	346	+ cache_size = self.cas.get_cache_size()
390	347	cache_size += artifact_size
391	348
392		- self.set_cache_size(cache_size)
393		-
394		- # get_cache_size()
395		- #
396		- # Fetches the cached size of the cache, this is sometimes
397		- # an estimate and periodically adjusted to the real size
398		- # when a cache size calculation job runs.
399		- #
400		- # When it is an estimate, the value is either correct, or
401		- # it is greater than the actual cache size.
402		- #
403		- # Returns:
404		- # (int) An approximation of the artifact cache size, in bytes.
405		- #
406		- def get_cache_size(self):
407		-
408		- # If we don't currently have an estimate, figure out the real cache size.
409		- if self._cache_size is None:
410		- stored_size = self._read_cache_size()
411		- if stored_size is not None:
412		- self._cache_size = stored_size
413		- else:
414		- self.compute_cache_size()
415		-
416		- return self._cache_size
417		-
418		- # set_cache_size()
419		- #
420		- # Forcefully set the overall cache size.
421		- #
422		- # This is used to update the size in the main process after
423		- # having calculated in a cleanup or a cache size calculation job.
424		- #
425		- # Args:
426		- # cache_size (int): The size to set.
427		- #
428		- def set_cache_size(self, cache_size):
429		-
430		- assert cache_size is not None
431		-
432		- self._cache_size = cache_size
433		- self._write_cache_size(self._cache_size)
434		-
435		- # full()
436		- #
437		- # Checks if the artifact cache is full, either
438		- # because the user configured quota has been exceeded
439		- # or because the underlying disk is almost full.
440		- #
441		- # Returns:
442		- # (bool): True if the artifact cache is full
443		- #
444		- def full(self):
445		-
446		- if self.get_cache_size() > self._cache_quota:
447		- return True
448		-
449		- _, volume_avail = self._get_cache_volume_size()
450		- if volume_avail < self._cache_quota_headroom:
451		- return True
452		-
453		- return False
	349	+ self.cas.set_cache_size(cache_size)
454	350
455	351	# preflight():
456	352	#
...	...	@@ -896,142 +792,6 @@ class ArtifactCache():
896	792	with self.context.timed_activity("Initializing remote caches", silent_nested=True):
897	793	self.initialize_remotes(on_failure=remote_failed)
898	794
899		- # _write_cache_size()
900		- #
901		- # Writes the given size of the artifact to the cache's size file
902		- #
903		- # Args:
904		- # size (int): The size of the artifact cache to record
905		- #
906		- def _write_cache_size(self, size):
907		- assert isinstance(size, int)
908		- size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
909		- with utils.save_file_atomic(size_file_path, "w") as f:
910		- f.write(str(size))
911		-
912		- # _read_cache_size()
913		- #
914		- # Reads and returns the size of the artifact cache that's stored in the
915		- # cache's size file
916		- #
917		- # Returns:
918		- # (int): The size of the artifact cache, as recorded in the file
919		- #
920		- def _read_cache_size(self):
921		- size_file_path = os.path.join(self.context.artifactdir, CACHE_SIZE_FILE)
922		-
923		- if not os.path.exists(size_file_path):
924		- return None
925		-
926		- with open(size_file_path, "r") as f:
927		- size = f.read()
928		-
929		- try:
930		- num_size = int(size)
931		- except ValueError as e:
932		- raise ArtifactError("Size '{}' parsed from '{}' was not an integer".format(
933		- size, size_file_path)) from e
934		-
935		- return num_size
936		-
937		- # _calculate_cache_quota()
938		- #
939		- # Calculates and sets the cache quota and lower threshold based on the
940		- # quota set in Context.
941		- # It checks that the quota is both a valid _expression_, and that there is
942		- # enough disk space to satisfy that quota
943		- #
944		- def _calculate_cache_quota(self):
945		- # Headroom intended to give BuildStream a bit of leeway.
946		- # This acts as the minimum size of cache_quota and also
947		- # is taken from the user requested cache_quota.
948		- #
949		- if 'BST_TEST_SUITE' in os.environ:
950		- self._cache_quota_headroom = 0
951		- else:
952		- self._cache_quota_headroom = 2e9
953		-
954		- try:
955		- cache_quota = utils._parse_size(self.context.config_cache_quota,
956		- self.context.artifactdir)
957		- except utils.UtilError as e:
958		- raise LoadError(LoadErrorReason.INVALID_DATA,
959		- "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
960		- "\nValid values are, for example: 800M 10G 1T 50%\n"
961		- .format(str(e))) from e
962		-
963		- total_size, available_space = self._get_cache_volume_size()
964		- cache_size = self.get_cache_size()
965		-
966		- # Ensure system has enough storage for the cache_quota
967		- #
968		- # If cache_quota is none, set it to the maximum it could possibly be.
969		- #
970		- # Also check that cache_quota is at least as large as our headroom.
971		- #
972		- if cache_quota is None: # Infinity, set to max system storage
973		- cache_quota = cache_size + available_space
974		- if cache_quota < self._cache_quota_headroom: # Check minimum
975		- raise LoadError(LoadErrorReason.INVALID_DATA,
976		- "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
977		- "BuildStream requires a minimum cache quota of 2G.")
978		- elif cache_quota > total_size:
979		- # A quota greater than the total disk size is certianly an error
980		- raise ArtifactError("Your system does not have enough available " +
981		- "space to support the cache quota specified.",
982		- detail=("You have specified a quota of {quota} total disk space.\n" +
983		- "The filesystem containing {local_cache_path} only " +
984		- "has {total_size} total disk space.")
985		- .format(
986		- quota=self.context.config_cache_quota,
987		- local_cache_path=self.context.artifactdir,
988		- total_size=utils._pretty_size(total_size)),
989		- reason='insufficient-storage-for-quota')
990		- elif cache_quota > cache_size + available_space:
991		- # The quota does not fit in the available space, this is a warning
992		- if '%' in self.context.config_cache_quota:
993		- available = (available_space / total_size) * 100
994		- available = '{}% of total disk space'.format(round(available, 1))
995		- else:
996		- available = utils._pretty_size(available_space)
997		-
998		- self._message(MessageType.WARN,
999		- "Your system does not have enough available " +
1000		- "space to support the cache quota specified.",
1001		- detail=("You have specified a quota of {quota} total disk space.\n" +
1002		- "The filesystem containing {local_cache_path} only " +
1003		- "has {available_size} available.")
1004		- .format(quota=self.context.config_cache_quota,
1005		- local_cache_path=self.context.artifactdir,
1006		- available_size=available))
1007		-
1008		- # Place a slight headroom (2e9 (2GB) on the cache_quota) into
1009		- # cache_quota to try and avoid exceptions.
1010		- #
1011		- # Of course, we might still end up running out during a build
1012		- # if we end up writing more than 2G, but hey, this stuff is
1013		- # already really fuzzy.
1014		- #
1015		- self._cache_quota_original = cache_quota
1016		- self._cache_quota = cache_quota - self._cache_quota_headroom
1017		- self._cache_lower_threshold = self._cache_quota / 2
1018		-
1019		- # _get_cache_volume_size()
1020		- #
1021		- # Get the available space and total space for the volume on
1022		- # which the artifact cache is located.
1023		- #
1024		- # Returns:
1025		- # (int): The total number of bytes on the volume
1026		- # (int): The number of available bytes on the volume
1027		- #
1028		- # NOTE: We use this stub to allow the test cases
1029		- # to override what an artifact cache thinks
1030		- # about it's disk size and available bytes.
1031		- #
1032		- def _get_cache_volume_size(self):
1033		- return utils._get_volume_size(self.context.artifactdir)
1034		-
1035	795
1036	796	# _configured_remote_artifact_cache_specs():
1037	797	#

buildstream/_cas/__init__.py

---

...	...	@@ -17,5 +17,5 @@
17	17	# Authors:
18	18	# Tristan Van Berkom <tristan vanberkom codethink co uk>
19	19
20		-from .cascache import CASCache
	20	+from .cascache import CASCache, CASCacheUsage
21	21	from .casremote import CASRemote, CASRemoteSpec

buildstream/_cas/cascache.py

---

...	...	@@ -31,25 +31,73 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
31	31	from .._protos.buildstream.v2 import buildstream_pb2
32	32
33	33	from .. import utils
34		-from .._exceptions import CASCacheError
	34	+from .._exceptions import CASCacheError, LoadError, LoadErrorReason
35	35
36	36	from .casremote import BlobNotFound, _CASBatchRead, _CASBatchUpdate
37	37
38	38
	39	+CACHE_SIZE_FILE = "cache_size"
	40	+
	41	+
	42	+# CASCacheUsage
	43	+#
	44	+# A simple object to report the current CAS cache usage details.
	45	+#
	46	+# Note that this uses the user configured cache quota
	47	+# rather than the internal quota with protective headroom
	48	+# removed, to provide a more sensible value to display to
	49	+# the user.
	50	+#
	51	+# Args:
	52	+# cas (CASCache): The CAS cache to get the status of
	53	+#
	54	+class CASCacheUsage():
	55	+
	56	+ def __init__(self, cas):
	57	+ self.quota_config = cas._config_cache_quota # Configured quota
	58	+ self.quota_size = cas._cache_quota_original # Resolved cache quota in bytes
	59	+ self.used_size = cas.get_cache_size() # Size used by artifacts in bytes
	60	+ self.used_percent = 0 # Percentage of the quota used
	61	+ if self.quota_size is not None:
	62	+ self.used_percent = int(self.used_size * 100 / self.quota_size)
	63	+
	64	+ # Formattable into a human readable string
	65	+ #
	66	+ def __str__(self):
	67	+ return "{} / {} ({}%)" \
	68	+ .format(utils._pretty_size(self.used_size, dec_places=1),
	69	+ self.quota_config,
	70	+ self.used_percent)
	71	+
	72	+
39	73	# A CASCache manages a CAS repository as specified in the Remote Execution API.
40	74	#
41	75	# Args:
42	76	# path (str): The root directory for the CAS repository
	77	+# cache_quota (int): User configured cache quota
43	78	#
44	79	class CASCache():
45	80
46		- def __init__(self, path):
	81	+ def __init__(self, path, cache_quota=None):
47	82	self.casdir = os.path.join(path, 'cas')
48	83	self.tmpdir = os.path.join(path, 'tmp')
49	84	os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True)
50	85	os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
51	86	os.makedirs(self.tmpdir, exist_ok=True)
52	87
	88	+ self._config_cache_quota = cache_quota
	89	+ self._cache_size = None # The current cache size, sometimes it's an estimate
	90	+ self._cache_quota = None # The cache quota
	91	+ self._cache_quota_original = None # The cache quota as specified by the user, in bytes
	92	+ self._cache_quota_headroom = None
	93	+ self._cache_lower_threshold = None # The target cache size for a cleanup
	94	+
	95	+ # Flag used for when cache quota is larger than available space
	96	+ self.cache_quota_exceeded = False
	97	+ self.available_space = None
	98	+
	99	+ self._calculate_cache_quota()
	100	+
53	101	# preflight():
54	102	#
55	103	# Preflight check.
...	...	@@ -587,6 +635,84 @@ class CASCache():
587	635	reachable = set()
588	636	self._reachable_refs_dir(reachable, tree, update_mtime=True)
589	637
	638	+ # compute_cache_size()
	639	+ #
	640	+ # Computes the real artifact cache size by calling
	641	+ # the abstract calculate_cache_size() method.
	642	+ #
	643	+ # Returns:
	644	+ # (int): The size of the artifact cache.
	645	+ #
	646	+ def compute_cache_size(self):
	647	+ old_cache_size = self._cache_size
	648	+ new_cache_size = self.calculate_cache_size()
	649	+
	650	+ if old_cache_size != new_cache_size:
	651	+ self._cache_size = new_cache_size
	652	+
	653	+ return self._cache_size
	654	+
	655	+ # get_cache_size()
	656	+ #
	657	+ # Fetches the cached size of the cache, this is sometimes
	658	+ # an estimate and periodically adjusted to the real size
	659	+ # when a cache size calculation job runs.
	660	+ #
	661	+ # When it is an estimate, the value is either correct, or
	662	+ # it is greater than the actual cache size.
	663	+ #
	664	+ # Returns:
	665	+ # (int) An approximation of the artifact cache size, in bytes.
	666	+ #
	667	+ def get_cache_size(self):
	668	+
	669	+ # If we don't currently have an estimate, figure out the real cache size.
	670	+ if self._cache_size is None:
	671	+ stored_size = self._read_cache_size()
	672	+ if stored_size is not None:
	673	+ self._cache_size = stored_size
	674	+ else:
	675	+ self._cache_size = self.compute_cache_size()
	676	+
	677	+ return self._cache_size
	678	+
	679	+ # set_cache_size()
	680	+ #
	681	+ # Forcefully set the overall cache size.
	682	+ #
	683	+ # This is used to update the size in the main process after
	684	+ # having calculated in a cleanup or a cache size calculation job.
	685	+ #
	686	+ # Args:
	687	+ # cache_size (int): The size to set.
	688	+ #
	689	+ def set_cache_size(self, cache_size):
	690	+
	691	+ assert cache_size is not None
	692	+
	693	+ self._cache_size = cache_size
	694	+ self._write_cache_size(self._cache_size)
	695	+
	696	+ # full()
	697	+ #
	698	+ # Checks if the artifact cache is full, either
	699	+ # because the user configured quota has been exceeded
	700	+ # or because the underlying disk is almost full.
	701	+ #
	702	+ # Returns:
	703	+ # (bool): True if the artifact cache is full
	704	+ #
	705	+ def full(self):
	706	+
	707	+ if self.get_cache_size() > self._cache_quota:
	708	+ return True
	709	+
	710	+ _, volume_avail = self._get_cache_volume_size()
	711	+ if volume_avail < self._cache_quota_headroom:
	712	+ return True
	713	+
	714	+ return False
	715	+
590	716	################################################
591	717	# Local Private Methods #
592	718	################################################
...	...	@@ -1015,6 +1141,122 @@ class CASCache():
1015	1141	# Send final batch
1016	1142	batch.send()
1017	1143
	1144	+ # _read_cache_size()
	1145	+ #
	1146	+ # Reads and returns the size of the artifact cache that's stored in the
	1147	+ # cache's size file
	1148	+ #
	1149	+ # Returns:
	1150	+ # (int): The size of the artifact cache, as recorded in the file
	1151	+ #
	1152	+ def _read_cache_size(self):
	1153	+ size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
	1154	+
	1155	+ if not os.path.exists(size_file_path):
	1156	+ return None
	1157	+
	1158	+ with open(size_file_path, "r") as f:
	1159	+ size = f.read()
	1160	+
	1161	+ try:
	1162	+ num_size = int(size)
	1163	+ except ValueError as e:
	1164	+ raise CASCacheError("Size '{}' parsed from '{}' was not an integer".format(
	1165	+ size, size_file_path)) from e
	1166	+
	1167	+ return num_size
	1168	+
	1169	+ # _write_cache_size()
	1170	+ #
	1171	+ # Writes the given size of the artifact to the cache's size file
	1172	+ #
	1173	+ # Args:
	1174	+ # size (int): The size of the artifact cache to record
	1175	+ #
	1176	+ def _write_cache_size(self, size):
	1177	+ assert isinstance(size, int)
	1178	+ size_file_path = os.path.join(self.casdir, CACHE_SIZE_FILE)
	1179	+ with utils.save_file_atomic(size_file_path, "w") as f:
	1180	+ f.write(str(size))
	1181	+
	1182	+ # _get_cache_volume_size()
	1183	+ #
	1184	+ # Get the available space and total space for the volume on
	1185	+ # which the artifact cache is located.
	1186	+ #
	1187	+ # Returns:
	1188	+ # (int): The total number of bytes on the volume
	1189	+ # (int): The number of available bytes on the volume
	1190	+ #
	1191	+ # NOTE: We use this stub to allow the test cases
	1192	+ # to override what an artifact cache thinks
	1193	+ # about it's disk size and available bytes.
	1194	+ #
	1195	+ def _get_cache_volume_size(self):
	1196	+ return utils._get_volume_size(self.casdir)
	1197	+
	1198	+ # _calculate_cache_quota()
	1199	+ #
	1200	+ # Calculates and sets the cache quota and lower threshold based on the
	1201	+ # quota set in Context.
	1202	+ # It checks that the quota is both a valid _expression_, and that there is
	1203	+ # enough disk space to satisfy that quota
	1204	+ #
	1205	+ def _calculate_cache_quota(self):
	1206	+ # Headroom intended to give BuildStream a bit of leeway.
	1207	+ # This acts as the minimum size of cache_quota and also
	1208	+ # is taken from the user requested cache_quota.
	1209	+ #
	1210	+ if 'BST_TEST_SUITE' in os.environ:
	1211	+ self._cache_quota_headroom = 0
	1212	+ else:
	1213	+ self._cache_quota_headroom = 2e9
	1214	+
	1215	+ total_size, available_space = self._get_cache_volume_size()
	1216	+ cache_size = self.get_cache_size()
	1217	+ self.available_space = available_space
	1218	+
	1219	+ # Ensure system has enough storage for the cache_quota
	1220	+ #
	1221	+ # If cache_quota is none, set it to the maximum it could possibly be.
	1222	+ #
	1223	+ # Also check that cache_quota is at least as large as our headroom.
	1224	+ #
	1225	+ cache_quota = self._config_cache_quota
	1226	+ if cache_quota is None: # Infinity, set to max system storage
	1227	+ cache_quota = cache_size + available_space
	1228	+ if cache_quota < self._cache_quota_headroom: # Check minimum
	1229	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	1230	+ "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
	1231	+ "BuildStream requires a minimum cache quota of 2G.")
	1232	+ elif cache_quota > total_size:
	1233	+ # A quota greater than the total disk size is certianly an error
	1234	+ raise CASCacheError("Your system does not have enough available " +
	1235	+ "space to support the cache quota specified.",
	1236	+ detail=("You have specified a quota of {quota} total disk space.\n" +
	1237	+ "The filesystem containing {local_cache_path} only " +
	1238	+ "has {total_size} total disk space.")
	1239	+ .format(
	1240	+ quota=self._config_cache_quota,
	1241	+ local_cache_path=self.casdir,
	1242	+ total_size=utils._pretty_size(total_size)),
	1243	+ reason='insufficient-storage-for-quota')
	1244	+ elif cache_quota > cache_size + available_space:
	1245	+ # FIXME: we want a message to be printed out here but message
	1246	+ # depends on context and cas server can't have context currently
	1247	+ self.cache_quota_exceeded = True
	1248	+
	1249	+ # Place a slight headroom (2e9 (2GB) on the cache_quota) into
	1250	+ # cache_quota to try and avoid exceptions.
	1251	+ #
	1252	+ # Of course, we might still end up running out during a build
	1253	+ # if we end up writing more than 2G, but hey, this stuff is
	1254	+ # already really fuzzy.
	1255	+ #
	1256	+ self._cache_quota_original = cache_quota
	1257	+ self._cache_quota = cache_quota - self._cache_quota_headroom
	1258	+ self._cache_lower_threshold = self._cache_quota / 2
	1259	+
1018	1260
1019	1261	def _grouper(iterable, n):
1020	1262	while True:

buildstream/_context.py

---

...	...	@@ -30,8 +30,8 @@ from . import _yaml
30	30	from ._exceptions import LoadError, LoadErrorReason, BstError
31	31	from ._message import Message, MessageType
32	32	from ._profile import Topics, profile_start, profile_end
33		-from ._artifactcache import ArtifactCache, ArtifactCacheUsage
34		-from ._cas import CASCache
	33	+from ._artifactcache import ArtifactCache
	34	+from ._cas import CASCache, CASCacheUsage
35	35	from ._workspaces import Workspaces, WorkspaceProjectCache
36	36	from .plugin import _plugin_lookup
37	37	from .sandbox import SandboxRemote
...	...	@@ -237,7 +237,16 @@ class Context():
237	237	cache = _yaml.node_get(defaults, Mapping, 'cache')
238	238	_yaml.node_validate(cache, ['quota', 'pull-buildtrees'])
239	239
240		- self.config_cache_quota = _yaml.node_get(cache, str, 'quota')
	240	+ config_cache_quota = _yaml.node_get(cache, str, 'quota')
	241	+ try:
	242	+ self.config_cache_quota = utils._parse_size(config_cache_quota,
	243	+ self.casdir)
	244	+ except utils.UtilError as e:
	245	+ print(config_cache_quota)
	246	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	247	+ "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
	248	+ "\nValid values are, for example: 800M 10G 1T 50%\n"
	249	+ .format(str(e))) from e
241	250
242	251	# Load artifact share configuration
243	252	self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
...	...	@@ -315,15 +324,15 @@ class Context():
315	324
316	325	return self._artifactcache
317	326
318		- # get_artifact_cache_usage()
	327	+ # get_cache_usage()
319	328	#
320	329	# Fetches the current usage of the artifact cache
321	330	#
322	331	# Returns:
323		- # (ArtifactCacheUsage): The current status
	332	+ # (CASCacheUsage): The current status
324	333	#
325		- def get_artifact_cache_usage(self):
326		- return ArtifactCacheUsage(self.artifactcache)
	334	+ def get_cache_usage(self):
	335	+ return CASCacheUsage(self.get_cascache())
327	336
328	337	# add_project():
329	338	#
...	...	@@ -690,7 +699,7 @@ class Context():
690	699
691	700	def get_cascache(self):
692	701	if self._cascache is None:
693		- self._cascache = CASCache(self.rootcachedir)
	702	+ self._cascache = CASCache(self.rootcachedir, self.config_cache_quota)
694	703	return self._cascache
695	704
696	705

buildstream/_frontend/cli.py

---

...	...	@@ -11,11 +11,11 @@ from .._exceptions import BstError, LoadError, AppError
11	11	from .._versions import BST_FORMAT_VERSION
12	12	from .complete import main_bashcomplete, complete_path, CompleteUnhandled
13	13
14		-
15	14	##################################################################
16	15	# Override of click's main entry point #
17	16	##################################################################
18	17
	18	+
19	19	# search_command()
20	20	#
21	21	# Helper function to get a command and context object

buildstream/_frontend/status.py

---

...	...	@@ -404,7 +404,7 @@ class _StatusHeader():
404	404	#
405	405	# ~~~~~~ cache: 69% ~~~~~~
406	406	#
407		- usage = self._context.get_artifact_cache_usage()
	407	+ usage = self._context.get_cache_usage()
408	408	usage_percent = '{}%'.format(usage.used_percent)
409	409
410	410	size = 21

buildstream/_frontend/widget.py

---

...	...	@@ -452,7 +452,7 @@ class LogLine(Widget):
452	452	values["Session Start"] = starttime.strftime('%A, %d-%m-%Y at %H:%M:%S')
453	453	values["Project"] = "{} ({})".format(project.name, project.directory)
454	454	values["Targets"] = ", ".join([t.name for t in stream.targets])
455		- values["Cache Usage"] = "{}".format(context.get_artifact_cache_usage())
	455	+ values["Cache Usage"] = "{}".format(context.get_cache_usage())
456	456	text += self._format_values(values)
457	457
458	458	# User configurations

buildstream/_scheduler/jobs/cachesizejob.py

---

...	...	@@ -25,14 +25,14 @@ class CacheSizeJob(Job):
25	25	self._complete_cb = complete_cb
26	26
27	27	context = self._scheduler.context
28		- self._artifacts = context.artifactcache
	28	+ self._cas = context.get_cascache()
29	29
30	30	def child_process(self):
31		- return self._artifacts.compute_cache_size()
	31	+ return self._cas.compute_cache_size()
32	32
33	33	def parent_complete(self, status, result):
34	34	if status == JobStatus.OK:
35		- self._artifacts.set_cache_size(result)
	35	+ self._cas.set_cache_size(result)
36	36
37	37	if self._complete_cb:
38	38	self._complete_cb(status, result)

buildstream/_scheduler/jobs/cleanupjob.py

---

...	...	@@ -25,27 +25,27 @@ class CleanupJob(Job):
25	25	self._complete_cb = complete_cb
26	26
27	27	context = self._scheduler.context
	28	+ self._cas = context.get_cascache()
28	29	self._artifacts = context.artifactcache
29	30
30	31	def child_process(self):
31	32	def progress():
32	33	self.send_message('update-cache-size',
33		- self._artifacts.get_cache_size())
	34	+ self._cas.get_cache_size())
34	35	return self._artifacts.clean(progress)
35	36
36	37	def handle_message(self, message_type, message):
37		-
38	38	# Update the cache size in the main process as we go,
39	39	# this provides better feedback in the UI.
40	40	if message_type == 'update-cache-size':
41		- self._artifacts.set_cache_size(message)
	41	+ self._cas.set_cache_size(message)
42	42	return True
43	43
44	44	return False
45	45
46	46	def parent_complete(self, status, result):
47	47	if status == JobStatus.OK:
48		- self._artifacts.set_cache_size(result)
	48	+ self._cas.set_cache_size(result)
49	49
50	50	if self._complete_cb:
51	51	self._complete_cb(status, result)

buildstream/_scheduler/jobs/job.py

---

...	...	@@ -381,7 +381,6 @@ class Job():
381	381	# queue (multiprocessing.Queue): The message queue for IPC
382	382	#
383	383	def _child_action(self, queue):
384		-
385	384	# This avoids some SIGTSTP signals from grandchildren
386	385	# getting propagated up to the master process
387	386	os.setsid()

buildstream/utils.py

---

...	...	@@ -43,6 +43,7 @@ from . import _signals
43	43	from ._exceptions import BstError, ErrorDomain
44	44	from ._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
45	45
	46	+
46	47	# The magic number for timestamps: 2011-11-11 11:11:11
47	48	_magic_timestamp = calendar.timegm([2011, 11, 11, 11, 11, 11])
48	49

tests/artifactcache/cache_size.py

---

...	...	@@ -50,14 +50,15 @@ def test_cache_size_write(cli, tmpdir):
50	50
51	51	# Artifact cache must be in a known place
52	52	artifactdir = os.path.join(project_dir, "artifacts")
53		- cli.configure({"artifactdir": artifactdir})
	53	+ casdir = os.path.join(project_dir, "cas")
	54	+ cli.configure({"rootcachedir": project_dir})
54	55
55	56	# Build, to populate the cache
56	57	res = cli.run(project=project_dir, args=["build", "test.bst"])
57	58	res.assert_success()
58	59
59	60	# Inspect the artifact cache
60		- sizefile = os.path.join(artifactdir, CACHE_SIZE_FILE)
	61	+ sizefile = os.path.join(casdir, CACHE_SIZE_FILE)
61	62	assert os.path.isfile(sizefile)
62	63	with open(sizefile, "r") as f:
63	64	size_data = f.read()
...	...	@@ -80,11 +81,11 @@ def test_quota_over_1024T(cli, tmpdir):
80	81	_yaml.dump({'name': 'main'}, str(project.join("project.conf")))
81	82
82	83	volume_space_patch = mock.patch(
83		- "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
	84	+ "buildstream._cas.CASCache._get_cache_volume_size",
84	85	autospec=True,
85	86	return_value=(1025 * TiB, 1025 * TiB)
86	87	)
87	88
88	89	with volume_space_patch:
89	90	result = cli.run(project, args=["build", "file.bst"])
90		- result.assert_main_error(ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota')
	91	+ result.assert_main_error(ErrorDomain.CAS, 'insufficient-storage-for-quota')

tests/artifactcache/expiry.py

---

...	...	@@ -340,11 +340,11 @@ def test_never_delete_required_track(cli, datafiles, tmpdir):
340	340	("200%", ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA),
341	341
342	342	# Not enough space on disk even if you cleaned up
343		- ("11K", ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota'),
	343	+ ("11K", ErrorDomain.CAS, 'insufficient-storage-for-quota'),
344	344
345	345	# Not enough space for these caches
346		- ("7K", 'warning', 'Your system does not have enough available'),
347		- ("70%", 'warning', 'Your system does not have enough available')
	346	+ ("7K", 'success', None),
	347	+ ("70%", 'success', None)
348	348	])
349	349	@pytest.mark.datafiles(DATA_DIR)
350	350	def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reason):
...	...	@@ -354,7 +354,7 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
354	354	cli.configure({
355	355	'cache': {
356	356	'quota': quota,
357		- }
	357	+ },
358	358	})
359	359
360	360	# We patch how we get space information
...	...	@@ -372,13 +372,13 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, err_domain, err_reas
372	372	total_space = 10000
373	373
374	374	volume_space_patch = mock.patch(
375		- "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
	375	+ "buildstream.utils._get_volume_size",
376	376	autospec=True,
377	377	return_value=(total_space, free_space),
378	378	)
379	379
380	380	cache_size_patch = mock.patch(
381		- "buildstream._artifactcache.ArtifactCache.get_cache_size",
	381	+ "buildstream._cas.CASCache.get_cache_size",
382	382	autospec=True,
383	383	return_value=0,
384	384	)

tests/testutils/artifactshare.py

---

...	...	@@ -46,7 +46,6 @@ class ArtifactShare():
46	46	# in tests as a remote artifact push/pull configuration
47	47	#
48	48	self.repodir = os.path.join(self.directory, 'repo')
49		-
50	49	os.makedirs(self.repodir)
51	50
52	51	self.cas = CASCache(self.repodir)
...	...	@@ -171,7 +170,9 @@ class ArtifactShare():
171	170
172	171	def _mock_statvfs(self, path):
173	172	repo_size = 0
174		- for root, _, files in os.walk(self.repodir):
	173	+ for root, dirs, files in os.walk(self.repodir):
	174	+ for dirname in dirs:
	175	+ repo_size += os.path.getsize(os.path.join(root, dirname))
175	176	for filename in files:
176	177	repo_size += os.path.getsize(os.path.join(root, filename))
177	178

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.