[Notes] [Git][BuildStream/buildstream][tiagogomes/issue-520] cascache: move tmp directory one level up

[Tiago Gomes <gitlab mg gitlab com>]

Title: GitLab

Tiago Gomes pushed to branch tiagogomes/issue-520 at BuildStream / buildstream

Commits:

• c68dcab8
  by Tiago Gomes at 2018-08-09T09:18:43Z

  cascache: move tmp directory one level up
  
  The CAS uses a temp directory while manipulating the cache, temporary
  files can be added and removed while adding artifacts to the cache here.
  
  Since calculation of the cache size happens in parallel to artifact
  cache additions, this causes race conditions in the size calculation
  job, as we end up calling `stat` on temporary files which are being
  removed in parallel.
  
  Handle this by moving the temporary directory out of the way, and avoid
  considering the tmp directory when calculating the cache size
  

3 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/_artifactcache/casserver.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -80,6 +80,8 @@ class ArtifactCache():
80	80	self.context = context
81	81	self.required_artifacts = set()
82	82	self.extractdir = os.path.join(context.artifactdir, 'extract')
	83	+ self.tmpdir = os.path.join(context.artifactdir, 'tmp')
	84	+
83	85	self.max_size = context.cache_quota
84	86	self.estimated_size = None
85	87
...	...	@@ -89,7 +91,8 @@ class ArtifactCache():
89	91	self._local = False
90	92	self.cache_size = None
91	93
92		- os.makedirs(context.artifactdir, exist_ok=True)
	94	+ os.makedirs(self.extractdir, exist_ok=True)
	95	+ os.makedirs(self.tmpdir, exist_ok=True)
93	96
94	97	################################################
95	98	# Methods implemented on the abstract class #

buildstream/_artifactcache/cascache.py

---

...	...	@@ -56,7 +56,8 @@ class CASCache(ArtifactCache):
56	56	super().__init__(context)
57	57
58	58	self.casdir = os.path.join(context.artifactdir, 'cas')
59		- os.makedirs(os.path.join(self.casdir, 'tmp'), exist_ok=True)
	59	+ os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True)
	60	+ os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
60	61
61	62	self._enable_push = enable_push
62	63
...	...	@@ -85,8 +86,6 @@ class CASCache(ArtifactCache):
85	86	# artifact has already been extracted
86	87	return dest
87	88
88		- os.makedirs(self.extractdir, exist_ok=True)
89		-
90	89	with tempfile.TemporaryDirectory(prefix='tmp', dir=self.extractdir) as tmpdir:
91	90	checkoutdir = os.path.join(tmpdir, ref)
92	91	self._checkout(checkoutdir, tree)
...	...	@@ -394,7 +393,7 @@ class CASCache(ArtifactCache):
394	393	try:
395	394	h = hashlib.sha256()
396	395	# Always write out new file to avoid corruption if input file is modified
397		- with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as out:
	396	+ with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
398	397	# Set mode bits to 0644
399	398	os.chmod(out.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
400	399
...	...	@@ -764,7 +763,7 @@ class CASCache(ArtifactCache):
764	763	# already in local cache
765	764	return
766	765
767		- with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as out:
	766	+ with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
768	767	self._fetch_blob(remote, tree, out)
769	768
770	769	directory = remote_execution_pb2.Directory()
...	...	@@ -778,7 +777,7 @@ class CASCache(ArtifactCache):
778	777	# already in local cache
779	778	continue
780	779
781		- with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as f:
	780	+ with tempfile.NamedTemporaryFile(dir=self.tmpdir) as f:
782	781	self._fetch_blob(remote, filenode.digest, f)
783	782
784	783	digest = self.add_object(path=f.name)

buildstream/_artifactcache/casserver.py

---

...	...	@@ -161,7 +161,7 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
161	161	offset = 0
162	162	finished = False
163	163	resource_name = None
164		- with tempfile.NamedTemporaryFile(dir=os.path.join(self.cas.casdir, 'tmp')) as out:
	164	+ with tempfile.NamedTemporaryFile(dir=self.cas.tmpdir) as out:
165	165	for request in request_iterator:
166	166	assert not finished
167	167	assert request.write_offset == offset

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.