[Notes] [Git][BuildStream/buildstream][tiagogomes/issue-520] cascache: move tmp directory one level up

[Tiago Gomes <gitlab mg gitlab com>]

Title: GitLab

Tiago Gomes pushed to branch tiagogomes/issue-520 at BuildStream / buildstream

Commits:

• b98f736c
  by Tiago Gomes at 2018-08-07T16:29:40Z

  cascache: move tmp directory one level up
  
  The tmp directory is filled when adding an artifact with temporary
  files. This causes problems on calculate_cache_size() because we could
  try to a attemp to do a stat() on a temporary file that meanwhile has
  been reaped.
  
  Handle this by not considering the tmp directory when calculating the
  cache size.
  

2 changed files:

• buildstream/_artifactcache/cascache.py
• buildstream/_artifactcache/casserver.py

Changes:

buildstream/_artifactcache/cascache.py

---

...	...	@@ -56,7 +56,10 @@ class CASCache(ArtifactCache):
56	56	super().__init__(context)
57	57
58	58	self.casdir = os.path.join(context.artifactdir, 'cas')
59		- os.makedirs(os.path.join(self.casdir, 'tmp'), exist_ok=True)
	59	+ os.makedirs(self.casdir, exist_ok=True)
	60	+
	61	+ self.tmpdir = os.path.join(context.artifactdir, 'tmp')
	62	+ os.makedirs(self.tmpdir, exist_ok=True)
60	63
61	64	self._enable_push = enable_push
62	65
...	...	@@ -394,7 +397,7 @@ class CASCache(ArtifactCache):
394	397	try:
395	398	h = hashlib.sha256()
396	399	# Always write out new file to avoid corruption if input file is modified
397		- with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as out:
	400	+ with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
398	401	# Set mode bits to 0644
399	402	os.chmod(out.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
400	403
...	...	@@ -764,7 +767,7 @@ class CASCache(ArtifactCache):
764	767	# already in local cache
765	768	return
766	769
767		- with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as out:
	770	+ with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
768	771	self._fetch_blob(remote, tree, out)
769	772
770	773	directory = remote_execution_pb2.Directory()
...	...	@@ -778,7 +781,7 @@ class CASCache(ArtifactCache):
778	781	# already in local cache
779	782	continue
780	783
781		- with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as f:
	784	+ with tempfile.NamedTemporaryFile(dir=self.tmpdir) as f:
782	785	self._fetch_blob(remote, filenode.digest, f)
783	786
784	787	digest = self.add_object(path=f.name)

buildstream/_artifactcache/casserver.py

---

...	...	@@ -161,7 +161,7 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
161	161	offset = 0
162	162	finished = False
163	163	resource_name = None
164		- with tempfile.NamedTemporaryFile(dir=os.path.join(self.cas.casdir, 'tmp')) as out:
	164	+ with tempfile.NamedTemporaryFile(dir=self.cas.tmpdir) as out:
165	165	for request in request_iterator:
166	166	assert not finished
167	167	assert request.write_offset == offset

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.