Chandan Singh pushed to branch chandan/import-is-not-buildelement at BuildStream / buildstream
Commits:
-
c9ce89d2
by Tristan Van Berkom at 2019-01-18T19:36:26Z
-
c536ab6a
by Tristan Van Berkom at 2019-01-18T19:36:26Z
-
99699ffc
by Tristan Van Berkom at 2019-01-18T19:36:26Z
-
8ce483d4
by Tristan Van Berkom at 2019-01-18T19:36:26Z
-
73c7252d
by Tristan Van Berkom at 2019-01-18T20:57:42Z
-
173c30ed
by Chandan Singh at 2019-01-18T21:25:36Z
4 changed files:
- buildstream/_cas/cascache.py
- buildstream/plugins/elements/import.py
- buildstream/utils.py
- tests/artifactcache/expiry.py
Changes:
... | ... | @@ -21,7 +21,7 @@ import hashlib |
21 | 21 |
import itertools
|
22 | 22 |
import os
|
23 | 23 |
import stat
|
24 |
-import tempfile
|
|
24 |
+import errno
|
|
25 | 25 |
import uuid
|
26 | 26 |
import contextlib
|
27 | 27 |
|
... | ... | @@ -129,7 +129,7 @@ class CASCache(): |
129 | 129 |
else:
|
130 | 130 |
return dest
|
131 | 131 |
|
132 |
- with tempfile.TemporaryDirectory(prefix='tmp', dir=self.tmpdir) as tmpdir:
|
|
132 |
+ with utils._tempdir(prefix='tmp', dir=self.tmpdir) as tmpdir:
|
|
133 | 133 |
checkoutdir = os.path.join(tmpdir, ref)
|
134 | 134 |
self._checkout(checkoutdir, tree)
|
135 | 135 |
|
... | ... | @@ -374,7 +374,7 @@ class CASCache(): |
374 | 374 |
for chunk in iter(lambda: tmp.read(4096), b""):
|
375 | 375 |
h.update(chunk)
|
376 | 376 |
else:
|
377 |
- tmp = stack.enter_context(tempfile.NamedTemporaryFile(dir=self.tmpdir))
|
|
377 |
+ tmp = stack.enter_context(utils._tempnamedfile(dir=self.tmpdir))
|
|
378 | 378 |
# Set mode bits to 0644
|
379 | 379 |
os.chmod(tmp.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
|
380 | 380 |
|
... | ... | @@ -545,11 +545,7 @@ class CASCache(): |
545 | 545 |
def remove(self, ref, *, defer_prune=False):
|
546 | 546 |
|
547 | 547 |
# Remove cache ref
|
548 |
- refpath = self._refpath(ref)
|
|
549 |
- if not os.path.exists(refpath):
|
|
550 |
- raise CASCacheError("Could not find ref '{}'".format(ref))
|
|
551 |
- |
|
552 |
- os.unlink(refpath)
|
|
548 |
+ self._remove_ref(ref)
|
|
553 | 549 |
|
554 | 550 |
if not defer_prune:
|
555 | 551 |
pruned = self.prune()
|
... | ... | @@ -626,6 +622,55 @@ class CASCache(): |
626 | 622 |
def _refpath(self, ref):
|
627 | 623 |
return os.path.join(self.casdir, 'refs', 'heads', ref)
|
628 | 624 |
|
625 |
+ # _remove_ref()
|
|
626 |
+ #
|
|
627 |
+ # Removes a ref.
|
|
628 |
+ #
|
|
629 |
+ # This also takes care of pruning away directories which can
|
|
630 |
+ # be removed after having removed the given ref.
|
|
631 |
+ #
|
|
632 |
+ # Args:
|
|
633 |
+ # ref (str): The ref to remove
|
|
634 |
+ #
|
|
635 |
+ # Raises:
|
|
636 |
+ # (CASCacheError): If the ref didnt exist, or a system error
|
|
637 |
+ # occurred while removing it
|
|
638 |
+ #
|
|
639 |
+ def _remove_ref(self, ref):
|
|
640 |
+ |
|
641 |
+ # Remove the ref itself
|
|
642 |
+ refpath = self._refpath(ref)
|
|
643 |
+ try:
|
|
644 |
+ os.unlink(refpath)
|
|
645 |
+ except FileNotFoundError as e:
|
|
646 |
+ raise CASCacheError("Could not find ref '{}'".format(ref)) from e
|
|
647 |
+ |
|
648 |
+ # Now remove any leading directories
|
|
649 |
+ basedir = os.path.join(self.casdir, 'refs', 'heads')
|
|
650 |
+ components = list(os.path.split(ref))
|
|
651 |
+ while components:
|
|
652 |
+ components.pop()
|
|
653 |
+ refdir = os.path.join(basedir, *components)
|
|
654 |
+ |
|
655 |
+ # Break out once we reach the base
|
|
656 |
+ if refdir == basedir:
|
|
657 |
+ break
|
|
658 |
+ |
|
659 |
+ try:
|
|
660 |
+ os.rmdir(refdir)
|
|
661 |
+ except FileNotFoundError:
|
|
662 |
+ # The parent directory did not exist, but it's
|
|
663 |
+ # parent directory might still be ready to prune
|
|
664 |
+ pass
|
|
665 |
+ except OSError as e:
|
|
666 |
+ if e.errno == errno.ENOTEMPTY:
|
|
667 |
+ # The parent directory was not empty, so we
|
|
668 |
+ # cannot prune directories beyond this point
|
|
669 |
+ break
|
|
670 |
+ |
|
671 |
+ # Something went wrong here
|
|
672 |
+ raise CASCacheError("System error while removing ref '{}': {}".format(ref, e)) from e
|
|
673 |
+ |
|
629 | 674 |
# _commit_directory():
|
630 | 675 |
#
|
631 | 676 |
# Adds local directory to content addressable store.
|
... | ... | @@ -797,7 +842,7 @@ class CASCache(): |
797 | 842 |
# already in local repository
|
798 | 843 |
return objpath
|
799 | 844 |
|
800 |
- with tempfile.NamedTemporaryFile(dir=self.tmpdir) as f:
|
|
845 |
+ with utils._tempnamedfile(dir=self.tmpdir) as f:
|
|
801 | 846 |
remote._fetch_blob(digest, f)
|
802 | 847 |
|
803 | 848 |
added_digest = self.add_object(path=f.name, link_directly=True)
|
... | ... | @@ -807,7 +852,7 @@ class CASCache(): |
807 | 852 |
|
808 | 853 |
def _batch_download_complete(self, batch):
|
809 | 854 |
for digest, data in batch.send():
|
810 |
- with tempfile.NamedTemporaryFile(dir=self.tmpdir) as f:
|
|
855 |
+ with utils._tempnamedfile(dir=self.tmpdir) as f:
|
|
811 | 856 |
f.write(data)
|
812 | 857 |
f.flush()
|
813 | 858 |
|
... | ... | @@ -904,7 +949,7 @@ class CASCache(): |
904 | 949 |
|
905 | 950 |
def _fetch_tree(self, remote, digest):
|
906 | 951 |
# download but do not store the Tree object
|
907 |
- with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
|
|
952 |
+ with utils._tempnamedfile(dir=self.tmpdir) as out:
|
|
908 | 953 |
remote._fetch_blob(digest, out)
|
909 | 954 |
|
910 | 955 |
tree = remote_execution_pb2.Tree()
|
... | ... | @@ -28,17 +28,14 @@ some configuration data. |
28 | 28 |
The empty configuration is as such:
|
29 | 29 |
.. literalinclude:: ../../../buildstream/plugins/elements/import.yaml
|
30 | 30 |
:language: yaml
|
31 |
- |
|
32 |
-See :ref:`built-in functionality documentation <core_buildelement_builtins>` for
|
|
33 |
-details on common configuration options for build elements.
|
|
34 | 31 |
"""
|
35 | 32 |
|
36 | 33 |
import os
|
37 |
-from buildstream import Element, BuildElement, ElementError
|
|
34 |
+from buildstream import Element, ElementError
|
|
38 | 35 |
|
39 | 36 |
|
40 | 37 |
# Element implementation for the 'import' kind.
|
41 |
-class ImportElement(BuildElement):
|
|
38 |
+class ImportElement(Element):
|
|
42 | 39 |
# pylint: disable=attribute-defined-outside-init
|
43 | 40 |
|
44 | 41 |
# This plugin has been modified to avoid the use of Sandbox.get_directory
|
... | ... | @@ -93,10 +90,6 @@ class ImportElement(BuildElement): |
93 | 90 |
# And we're done
|
94 | 91 |
return '/output'
|
95 | 92 |
|
96 |
- def prepare(self, sandbox):
|
|
97 |
- # We inherit a non-default prepare from BuildElement.
|
|
98 |
- Element.prepare(self, sandbox)
|
|
99 |
- |
|
100 | 93 |
def generate_script(self):
|
101 | 94 |
build_root = self.get_variable('build-root')
|
102 | 95 |
install_root = self.get_variable('install-root')
|
... | ... | @@ -1032,6 +1032,36 @@ def _tempdir(suffix="", prefix="tmp", dir=None): # pylint: disable=redefined-bu |
1032 | 1032 |
cleanup_tempdir()
|
1033 | 1033 |
|
1034 | 1034 |
|
1035 |
+# _tempnamedfile()
|
|
1036 |
+#
|
|
1037 |
+# A context manager for doing work on an open temporary file
|
|
1038 |
+# which is guaranteed to be named and have an entry in the filesystem.
|
|
1039 |
+#
|
|
1040 |
+# Args:
|
|
1041 |
+# dir (str): A path to a parent directory for the temporary file
|
|
1042 |
+# suffix (str): A suffix for the temproary file name
|
|
1043 |
+# prefix (str): A prefix for the temporary file name
|
|
1044 |
+#
|
|
1045 |
+# Yields:
|
|
1046 |
+# (str): The temporary file handle
|
|
1047 |
+#
|
|
1048 |
+# Do not use tempfile.NamedTemporaryFile() directly, as this will
|
|
1049 |
+# leak files on the filesystem when BuildStream exits a process
|
|
1050 |
+# on SIGTERM.
|
|
1051 |
+#
|
|
1052 |
+@contextmanager
|
|
1053 |
+def _tempnamedfile(suffix="", prefix="tmp", dir=None): # pylint: disable=redefined-builtin
|
|
1054 |
+ temp = None
|
|
1055 |
+ |
|
1056 |
+ def close_tempfile():
|
|
1057 |
+ if temp is not None:
|
|
1058 |
+ temp.close()
|
|
1059 |
+ |
|
1060 |
+ with _signals.terminator(close_tempfile), \
|
|
1061 |
+ tempfile.NamedTemporaryFile(suffix=suffix, prefix=prefix, dir=dir) as temp:
|
|
1062 |
+ yield temp
|
|
1063 |
+ |
|
1064 |
+ |
|
1035 | 1065 |
# _kill_process_tree()
|
1036 | 1066 |
#
|
1037 | 1067 |
# Brutally murder a process and all of its children
|
... | ... | @@ -382,6 +382,7 @@ def test_extract_expiry(cli, datafiles, tmpdir): |
382 | 382 |
res = cli.run(project=project, args=['checkout', 'target.bst', os.path.join(str(tmpdir), 'checkout')])
|
383 | 383 |
res.assert_success()
|
384 | 384 |
|
385 |
+ # Get a snapshot of the extracts in advance
|
|
385 | 386 |
extractdir = os.path.join(project, 'cache', 'artifacts', 'extract', 'test', 'target')
|
386 | 387 |
extracts = os.listdir(extractdir)
|
387 | 388 |
assert(len(extracts) == 1)
|
... | ... | @@ -395,3 +396,16 @@ def test_extract_expiry(cli, datafiles, tmpdir): |
395 | 396 |
|
396 | 397 |
# Now the extract should be removed.
|
397 | 398 |
assert not os.path.exists(extract)
|
399 |
+ |
|
400 |
+ # As an added bonus, let's ensure that no directories have been left behind
|
|
401 |
+ #
|
|
402 |
+ # Now we should have a directory for the cached target2.bst, which
|
|
403 |
+ # replaced target.bst in the cache, we should not have a directory
|
|
404 |
+ # for the target.bst
|
|
405 |
+ refsdir = os.path.join(project, 'cache', 'artifacts', 'cas', 'refs', 'heads')
|
|
406 |
+ refsdirtest = os.path.join(refsdir, 'test')
|
|
407 |
+ refsdirtarget = os.path.join(refsdirtest, 'target')
|
|
408 |
+ refsdirtarget2 = os.path.join(refsdirtest, 'target2')
|
|
409 |
+ |
|
410 |
+ assert os.path.isdir(refsdirtarget2)
|
|
411 |
+ assert not os.path.exists(refsdirtarget)
|