Tristan Van Berkom pushed to branch tristan/cas-cleanup-improve at BuildStream / buildstream
Commits:
-
356b1b25
by Tristan Van Berkom at 2019-01-18T19:25:16Z
-
27b42332
by Tristan Van Berkom at 2019-01-18T19:29:13Z
2 changed files:
Changes:
| ... | ... | @@ -22,7 +22,6 @@ import itertools |
| 22 | 22 |
import os
|
| 23 | 23 |
import stat
|
| 24 | 24 |
import errno
|
| 25 |
-import tempfile
|
|
| 26 | 25 |
import uuid
|
| 27 | 26 |
import contextlib
|
| 28 | 27 |
|
| ... | ... | @@ -130,7 +129,7 @@ class CASCache(): |
| 130 | 129 |
else:
|
| 131 | 130 |
return dest
|
| 132 | 131 |
|
| 133 |
- with tempfile.TemporaryDirectory(prefix='tmp', dir=self.tmpdir) as tmpdir:
|
|
| 132 |
+ with utils._tempdir(prefix='tmp', dir=self.tmpdir) as tmpdir:
|
|
| 134 | 133 |
checkoutdir = os.path.join(tmpdir, ref)
|
| 135 | 134 |
self._checkout(checkoutdir, tree)
|
| 136 | 135 |
|
| ... | ... | @@ -375,7 +374,7 @@ class CASCache(): |
| 375 | 374 |
for chunk in iter(lambda: tmp.read(4096), b""):
|
| 376 | 375 |
h.update(chunk)
|
| 377 | 376 |
else:
|
| 378 |
- tmp = stack.enter_context(tempfile.NamedTemporaryFile(dir=self.tmpdir))
|
|
| 377 |
+ tmp = stack.enter_context(utils._tempnamedfile(dir=self.tmpdir))
|
|
| 379 | 378 |
# Set mode bits to 0644
|
| 380 | 379 |
os.chmod(tmp.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
|
| 381 | 380 |
|
| ... | ... | @@ -843,7 +842,7 @@ class CASCache(): |
| 843 | 842 |
# already in local repository
|
| 844 | 843 |
return objpath
|
| 845 | 844 |
|
| 846 |
- with tempfile.NamedTemporaryFile(dir=self.tmpdir) as f:
|
|
| 845 |
+ with utils._tempnamedfile(dir=self.tmpdir) as f:
|
|
| 847 | 846 |
remote._fetch_blob(digest, f)
|
| 848 | 847 |
|
| 849 | 848 |
added_digest = self.add_object(path=f.name, link_directly=True)
|
| ... | ... | @@ -853,7 +852,7 @@ class CASCache(): |
| 853 | 852 |
|
| 854 | 853 |
def _batch_download_complete(self, batch):
|
| 855 | 854 |
for digest, data in batch.send():
|
| 856 |
- with tempfile.NamedTemporaryFile(dir=self.tmpdir) as f:
|
|
| 855 |
+ with utils._tempnamedfile(dir=self.tmpdir) as f:
|
|
| 857 | 856 |
f.write(data)
|
| 858 | 857 |
f.flush()
|
| 859 | 858 |
|
| ... | ... | @@ -950,7 +949,7 @@ class CASCache(): |
| 950 | 949 |
|
| 951 | 950 |
def _fetch_tree(self, remote, digest):
|
| 952 | 951 |
# download but do not store the Tree object
|
| 953 |
- with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
|
|
| 952 |
+ with utils._tempnamedfile(dir=self.tmpdir) as out:
|
|
| 954 | 953 |
remote._fetch_blob(digest, out)
|
| 955 | 954 |
|
| 956 | 955 |
tree = remote_execution_pb2.Tree()
|
| ... | ... | @@ -1032,6 +1032,36 @@ def _tempdir(suffix="", prefix="tmp", dir=None): # pylint: disable=redefined-bu |
| 1032 | 1032 |
cleanup_tempdir()
|
| 1033 | 1033 |
|
| 1034 | 1034 |
|
| 1035 |
+# _tempnamedfile()
|
|
| 1036 |
+#
|
|
| 1037 |
+# A context manager for doing work on an open temporary file
|
|
| 1038 |
+# which is guaranteed to be named and have an entry in the filesystem.
|
|
| 1039 |
+#
|
|
| 1040 |
+# Args:
|
|
| 1041 |
+# dir (str): A path to a parent directory for the temporary file
|
|
| 1042 |
+# suffix (str): A suffix for the temproary file name
|
|
| 1043 |
+# prefix (str): A prefix for the temporary file name
|
|
| 1044 |
+#
|
|
| 1045 |
+# Yields:
|
|
| 1046 |
+# (str): The temporary file handle
|
|
| 1047 |
+#
|
|
| 1048 |
+# Do not use tempfile.NamedTemporaryFile() directly, as this will
|
|
| 1049 |
+# leak files on the filesystem when BuildStream exits a process
|
|
| 1050 |
+# on SIGTERM.
|
|
| 1051 |
+#
|
|
| 1052 |
+@contextmanager
|
|
| 1053 |
+def _tempnamedfile(suffix="", prefix="tmp", dir=None): # pylint: disable=redefined-builtin
|
|
| 1054 |
+ temp = None
|
|
| 1055 |
+ |
|
| 1056 |
+ def close_tempfile():
|
|
| 1057 |
+ if temp is not None:
|
|
| 1058 |
+ temp.close()
|
|
| 1059 |
+ |
|
| 1060 |
+ with _signals.terminator(close_tempfile), \
|
|
| 1061 |
+ tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, dir=dir) as temp:
|
|
| 1062 |
+ yield temp
|
|
| 1063 |
+ |
|
| 1064 |
+ |
|
| 1035 | 1065 |
# _kill_process_tree()
|
| 1036 | 1066 |
#
|
| 1037 | 1067 |
# Brutally murder a process and all of its children
|