[Notes] [Git][BuildStream/buildstream][tiagogomes/issue-520] cascache: move tmp directory one level up

[Tiago Gomes <gitlab mg gitlab com>]

Title: GitLab

Tiago Gomes pushed to branch tiagogomes/issue-520 at BuildStream / buildstream

Commits:

• 18be2529
  by Tiago Gomes at 2018-08-08T14:39:08Z

  cascache: move tmp directory one level up
  
  The tmp directory is filled when adding an artifact with temporary
  files. This causes problems on calculate_cache_size() because we could
  try to a attemp to do a stat() on a temporary file that meanwhile has
  been reaped.
  
  Handle this by not considering the tmp directory when calculating the
  cache size.
  

3 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/_artifactcache/casserver.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -80,6 +80,8 @@ class ArtifactCache():
80	80	self.context = context
81	81	self.required_artifacts = set()
82	82	self.extractdir = os.path.join(context.artifactdir, 'extract')
	83	+ self.tmpdir = os.path.join(context.artifactdir, 'tmp')
	84	+
83	85	self.max_size = context.cache_quota
84	86	self.estimated_size = None
85	87
...	...	@@ -89,7 +91,8 @@ class ArtifactCache():
89	91	self._local = False
90	92	self.cache_size = None
91	93
92		- os.makedirs(context.artifactdir, exist_ok=True)
	94	+ os.makedirs(self.extractdir, exist_ok=True)
	95	+ os.makedirs(self.tmpdir, exist_ok=True)
93	96
94	97	################################################
95	98	# Methods implemented on the abstract class #

buildstream/_artifactcache/cascache.py

---

...	...	@@ -56,7 +56,7 @@ class CASCache(ArtifactCache):
56	56	super().__init__(context)
57	57
58	58	self.casdir = os.path.join(context.artifactdir, 'cas')
59		- os.makedirs(os.path.join(self.casdir, 'tmp'), exist_ok=True)
	59	+ os.makedirs(self.casdir, exist_ok=True)
60	60
61	61	self._enable_push = enable_push
62	62
...	...	@@ -85,8 +85,6 @@ class CASCache(ArtifactCache):
85	85	# artifact has already been extracted
86	86	return dest
87	87
88		- os.makedirs(self.extractdir, exist_ok=True)
89		-
90	88	with tempfile.TemporaryDirectory(prefix='tmp', dir=self.extractdir) as tmpdir:
91	89	checkoutdir = os.path.join(tmpdir, ref)
92	90	self._checkout(checkoutdir, tree)
...	...	@@ -394,7 +392,7 @@ class CASCache(ArtifactCache):
394	392	try:
395	393	h = hashlib.sha256()
396	394	# Always write out new file to avoid corruption if input file is modified
397		- with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as out:
	395	+ with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
398	396	# Set mode bits to 0644
399	397	os.chmod(out.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
400	398
...	...	@@ -764,7 +762,7 @@ class CASCache(ArtifactCache):
764	762	# already in local cache
765	763	return
766	764
767		- with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as out:
	765	+ with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
768	766	self._fetch_blob(remote, tree, out)
769	767
770	768	directory = remote_execution_pb2.Directory()
...	...	@@ -778,7 +776,7 @@ class CASCache(ArtifactCache):
778	776	# already in local cache
779	777	continue
780	778
781		- with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as f:
	779	+ with tempfile.NamedTemporaryFile(dir=self.tmpdir) as f:
782	780	self._fetch_blob(remote, filenode.digest, f)
783	781
784	782	digest = self.add_object(path=f.name)

buildstream/_artifactcache/casserver.py

---

...	...	@@ -161,7 +161,7 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
161	161	offset = 0
162	162	finished = False
163	163	resource_name = None
164		- with tempfile.NamedTemporaryFile(dir=os.path.join(self.cas.casdir, 'tmp')) as out:
	164	+ with tempfile.NamedTemporaryFile(dir=self.cas.tmpdir) as out:
165	165	for request in request_iterator:
166	166	assert not finished
167	167	assert request.write_offset == offset

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.