Valentin David pushed to branch valentindavid/extract-expiry at BuildStream / buildstream
Commits:
-
2dc881a1
by Valentin David at 2018-08-22T09:11:21Z
2 changed files:
Changes:
| ... | ... | @@ -30,6 +30,8 @@ from urllib.parse import urlparse |
| 30 | 30 |
|
| 31 | 31 |
import grpc
|
| 32 | 32 |
|
| 33 |
+from .. import _yaml
|
|
| 34 |
+ |
|
| 33 | 35 |
from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
|
| 34 | 36 |
from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
|
| 35 | 37 |
from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
|
| ... | ... | @@ -524,6 +526,25 @@ class CASCache(ArtifactCache): |
| 524 | 526 |
#
|
| 525 | 527 |
def remove(self, ref, *, defer_prune=False):
|
| 526 | 528 |
|
| 529 |
+ # Remove extract if not used by other ref
|
|
| 530 |
+ tree = self.resolve_ref(ref)
|
|
| 531 |
+ ref_name, ref_hash = os.path.split(ref)
|
|
| 532 |
+ extract = os.path.join(self.extractdir, ref_name, tree.hash)
|
|
| 533 |
+ keys_file = os.path.join(extract, 'meta', 'keys.yaml')
|
|
| 534 |
+ if os.path.exists(keys_file):
|
|
| 535 |
+ keys_meta = _yaml.load(keys_file)
|
|
| 536 |
+ keys = [keys_meta['strong'], keys_meta['weak']]
|
|
| 537 |
+ remove_extract = True
|
|
| 538 |
+ for other_hash in keys:
|
|
| 539 |
+ if other_hash == ref_hash:
|
|
| 540 |
+ continue
|
|
| 541 |
+ remove_extract = False
|
|
| 542 |
+ break
|
|
| 543 |
+ |
|
| 544 |
+ if remove_extract:
|
|
| 545 |
+ utils._force_rmtree(extract)
|
|
| 546 |
+ |
|
| 547 |
+ # Remove cache ref
|
|
| 527 | 548 |
refpath = self._refpath(ref)
|
| 528 | 549 |
if not os.path.exists(refpath):
|
| 529 | 550 |
raise ArtifactError("Could not find artifact for ref '{}'".format(ref))
|
| ... | ... | @@ -268,3 +268,38 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success): |
| 268 | 268 |
res.assert_success()
|
| 269 | 269 |
else:
|
| 270 | 270 |
res.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
|
| 271 |
+ |
|
| 272 |
+ |
|
| 273 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
| 274 |
+def test_extract_expiry(cli, datafiles, tmpdir):
|
|
| 275 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
| 276 |
+ element_path = 'elements'
|
|
| 277 |
+ |
|
| 278 |
+ cli.configure({
|
|
| 279 |
+ 'cache': {
|
|
| 280 |
+ 'quota': 10000000,
|
|
| 281 |
+ }
|
|
| 282 |
+ })
|
|
| 283 |
+ |
|
| 284 |
+ create_element_size('target.bst', project, element_path, [], 6000000)
|
|
| 285 |
+ res = cli.run(project=project, args=['build', 'target.bst'])
|
|
| 286 |
+ res.assert_success()
|
|
| 287 |
+ assert cli.get_element_state(project, 'target.bst') == 'cached'
|
|
| 288 |
+ |
|
| 289 |
+ # Force creating extract
|
|
| 290 |
+ res = cli.run(project=project, args=['checkout', 'target.bst', os.path.join(str(tmpdir), 'checkout')])
|
|
| 291 |
+ res.assert_success()
|
|
| 292 |
+ |
|
| 293 |
+ extractdir = os.path.join(project, 'cache', 'artifacts', 'extract', 'test', 'target')
|
|
| 294 |
+ extracts = os.listdir(extractdir)
|
|
| 295 |
+ assert(len(extracts) == 1)
|
|
| 296 |
+ extract = os.path.join(extractdir, extracts[0])
|
|
| 297 |
+ |
|
| 298 |
+ # Remove target.bst from artifact cache
|
|
| 299 |
+ create_element_size('target2.bst', project, element_path, [], 6000000)
|
|
| 300 |
+ res = cli.run(project=project, args=['build', 'target2.bst'])
|
|
| 301 |
+ res.assert_success()
|
|
| 302 |
+ assert cli.get_element_state(project, 'target.bst') != 'cached'
|
|
| 303 |
+ |
|
| 304 |
+ # Now the extract should be removed.
|
|
| 305 |
+ assert not os.path.exists(extract)
|