[Notes] [Git][BuildStream/buildstream][jennis/new_artifact_subcommands] 6 commits: cli.py: Defer pruning until all specified refs are removed

[James Ennis <gitlab mg gitlab com>]

Title: GitLab

James Ennis pushed to branch jennis/new_artifact_subcommands at BuildStream / buildstream

Commits:

• 3ba9bb50
  by James Ennis at 2019-01-11T17:43:23Z

  cli.py: Defer pruning until all specified refs are removed
  

• 01344f51
  by James Ennis at 2019-01-11T17:43:25Z

  tests: Add tests for the artifact delete subcommand
  

• 8ef1aae7
  by James Ennis at 2019-01-11T17:43:25Z

  artifactcache.py: Ensure we can determine whether both elements and artifacts are cached
  
  We should be able to determine whether an element or an artifact is
  present in the local cache solely with the artifactcache API. This commit
  avoids having to use cascache's API to determine whether an artifact ref
  is cached.
  

• cec4364c
  by James Ennis at 2019-01-11T17:43:55Z

  cli.py: Print to stderr when trying to remove a nonexistent element/artifact
  
  If we specify multiple elements/artifacts for deletion, we should not stop
  and throw an exception when we come across an element/artifact that is
  not cached.
  

• 4bbb53f0
  by James Ennis at 2019-01-11T17:44:34Z

  cli.py: Ensure that artifacts with both strong and weak cache keys are removed for element deletion
  

• 0e4d4175
  by James Ennis at 2019-01-11T17:44:37Z

  cascache.py: Ensure path exists before trying to update the mtime
  
  If an artifact is pulled from the cache without its buildtree, prune
  will fail as it will expect to update the mtimes of the object files
  of the build tree
  

4 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/_frontend/cli.py
• tests/integration/artifact.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -421,18 +421,30 @@ class ArtifactCache():
421	421
422	422	# contains():
423	423	#
424		- # Check whether the artifact for the specified Element is already available
425		- # in the local artifact cache.
	424	+ # Check whether the (project state) artifact of the specified Element is
	425	+ # already available in the local artifact cache.
426	426	#
427	427	# Args:
428	428	# element (Element): The Element to check
429	429	# key (str): The cache key to use
430	430	#
431		- # Returns: True if the artifact is in the cache, False otherwise
	431	+ # Returns: True if the Element's (project state) artifact is in the cache,
	432	+ # False otherwise
432	433	#
433	434	def contains(self, element, key):
434	435	ref = self.get_artifact_fullname(element, key)
	436	+ return self.contains_ref(ref)
435	437
	438	+ # contains_ref():
	439	+ #
	440	+ # Check whether an artifact is already available in the local artifact cache.
	441	+ #
	442	+ # Args:
	443	+ # ref (str): The ref to check
	444	+ #
	445	+ # Returns: True if the artifact is in the cache, False otherwise
	446	+ #
	447	+ def contains_ref(self, ref):
436	448	return self.cas.contains(ref)
437	449
438	450	# contains_subdir_artifact():

buildstream/_artifactcache/cascache.py

---

...	...	@@ -869,6 +869,9 @@ class CASCache():
869	869	if tree.hash in reachable:
870	870	return
871	871
	872	+ if not os.path.exists(self.objpath(tree)):
	873	+ return
	874	+
872	875	if update_mtime:
873	876	os.utime(self.objpath(tree))
874	877

buildstream/_frontend/cli.py

---

...	...	@@ -6,6 +6,7 @@ from tempfile import TemporaryDirectory
6	6
7	7	import click
8	8	from .. import _yaml
	9	+from ..types import _KeyStrength
9	10	from .._exceptions import BstError, LoadError, AppError
10	11	from .._versions import BST_FORMAT_VERSION
11	12	from .complete import main_bashcomplete, complete_path, CompleteUnhandled
...	...	@@ -1100,13 +1101,40 @@ def artifact_delete(app, artifacts):
1100	1101	if element is not None:
1101	1102	elements = [element]
1102	1103
	1104	+ # Remove specified elements and artifacts
1103	1105	if elements:
1104	1106	elements = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
1105	1107	for element in elements:
1106		- cache.remove(cache.get_artifact_fullname(element, element._get_cache_key()))
	1108	+ cache_keys = set([element._get_cache_key(),
	1109	+ element._get_cache_key(strength=_KeyStrength.WEAK)])
	1110	+ for cache_key in cache_keys:
	1111	+ ref = cache.get_artifact_fullname(element, cache_key)
	1112	+ if cache.contains(element, cache_key):
	1113	+ cache.remove(ref, defer_prune=True)
	1114	+ click.echo("Removed {}.".format(ref))
	1115	+ else:
	1116	+ # If the ref is not present when we try to delete it, we should
	1117	+ # not fail but just continue to delete. The pruning will take care
	1118	+ # of any unreachable objects.
	1119	+ click.echo("WARNING: {}, not found in local cache - no delete required"
	1120	+ .format(ref), err=True)
	1121	+ continue
	1122	+
1107	1123	if artifacts:
1108		- for i, ref in enumerate(artifacts, start=1):
1109		- cache.remove(ref, defer_prune=False)
	1124	+ for ref in artifacts:
	1125	+ if cache.contains_ref(ref):
	1126	+ cache.remove(ref, defer_prune=True)
	1127	+ click.echo("Removed {}.".format(ref))
	1128	+ else:
	1129	+ # If the ref is not present when we try to delete it, we should
	1130	+ # not fail but just continue to delete. The pruning will take care
	1131	+ # of any unreachable objects.
	1132	+ click.echo("WARNING: {}, not found in local cache - no delete required"
	1133	+ .format(ref), err=True)
	1134	+ continue
	1135	+
	1136	+ # Now we've removed all the refs, prune the unreachable objects
	1137	+ cache.prune()
1110	1138
1111	1139
1112	1140	##################################################################

tests/integration/artifact.py

---

...	...	@@ -21,7 +21,7 @@
21	21	import os
22	22	import pytest
23	23
24		-from tests.testutils import cli_integration as cli
	24	+from tests.testutils import cli_integration as cli, create_artifact_share
25	25
26	26
27	27	pytestmark = pytest.mark.integration
...	...	@@ -66,3 +66,153 @@ def test_artifact_log(cli, tmpdir, datafiles):
66	66	assert result.exit_code == 0
67	67	# The artifact is cached under both a strong key and a weak key
68	68	assert (log + log) == result.output
	69	+
	70	+
	71	+# Test that we can delete the artifact of the element which corresponds
	72	+# to the current project state
	73	+@pytest.mark.integration
	74	+@pytest.mark.datafiles(DATA_DIR)
	75	+def test_artifact_delete_element(cli, tmpdir, datafiles):
	76	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	77	+ element = 'integration.bst'
	78	+
	79	+ # Build the element and ensure it's cached
	80	+ result = cli.run(project=project, args=['build', element])
	81	+ result.assert_success()
	82	+ assert cli.get_element_state(project, element) == 'cached'
	83	+
	84	+ result = cli.run(project=project, args=['artifact', 'delete', element])
	85	+ result.assert_success()
	86	+ assert cli.get_element_state(project, element) != 'cached'
	87	+
	88	+
	89	+# Test that we can delete an artifact by specifying its ref.
	90	+@pytest.mark.integration
	91	+@pytest.mark.datafiles(DATA_DIR)
	92	+def test_artifact_delete_artifact(cli, tmpdir, datafiles):
	93	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	94	+ element = 'integration.bst'
	95	+
	96	+ # Configure a local cache
	97	+ local_cache = os.path.join(str(tmpdir), 'artifacts')
	98	+ cli.configure({'artifactdir': local_cache})
	99	+
	100	+ # First build an element so that we can find its artifact
	101	+ result = cli.run(project=project, args=['build', element])
	102	+ result.assert_success()
	103	+
	104	+ # Obtain the artifact ref
	105	+ cache_key = cli.get_element_key(project, element)
	106	+ artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
	107	+
	108	+ # Explicitly check that the ARTIFACT exists in the cache
	109	+ assert os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
	110	+
	111	+ # Delete the artifact
	112	+ result = cli.run(project=project, args=['artifact', 'delete', artifact])
	113	+ result.assert_success()
	114	+
	115	+ # Check that the ARTIFACT is no longer in the cache
	116	+ assert not os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
	117	+
	118	+
	119	+# Test the `bst artifact delete` command with multiple, different arguments.
	120	+@pytest.mark.integration
	121	+@pytest.mark.datafiles(DATA_DIR)
	122	+def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles):
	123	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	124	+ element = 'integration.bst'
	125	+ dep = 'base/base-alpine.bst'
	126	+
	127	+ # Configure a local cache
	128	+ local_cache = os.path.join(str(tmpdir), 'artifacts')
	129	+ cli.configure({'artifactdir': local_cache})
	130	+
	131	+ # First build an element so that we can find its artifact
	132	+ result = cli.run(project=project, args=['build', element])
	133	+ result.assert_success()
	134	+ assert cli.get_element_state(project, element) == 'cached'
	135	+ assert cli.get_element_state(project, dep) == 'cached'
	136	+
	137	+ # Obtain the artifact ref
	138	+ cache_key = cli.get_element_key(project, element)
	139	+ artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
	140	+
	141	+ # Explicitly check that the ARTIFACT exists in the cache
	142	+ assert os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
	143	+
	144	+ # Delete the artifact
	145	+ result = cli.run(project=project, args=['artifact', 'delete', artifact, dep])
	146	+ result.assert_success()
	147	+
	148	+ # Check that the ARTIFACT is no longer in the cache
	149	+ assert not os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
	150	+
	151	+ # Check that the dependency ELEMENT is no longer cached
	152	+ assert cli.get_element_state(project, dep) != 'cached'
	153	+
	154	+
	155	+# Test that we receive the appropriate stderr when we try to delete an artifact
	156	+# that is not present in the cache.
	157	+@pytest.mark.integration
	158	+@pytest.mark.datafiles(DATA_DIR)
	159	+def test_artifact_delete_unbuilt_artifact(cli, tmpdir, datafiles):
	160	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	161	+ element = 'integration.bst'
	162	+
	163	+ # Configure a local cache
	164	+ local_cache = os.path.join(str(tmpdir), 'artifacts')
	165	+ cli.configure({'artifactdir': local_cache})
	166	+
	167	+ # Ensure the element is not cached
	168	+ assert cli.get_element_state(project, element) != 'cached'
	169	+
	170	+ # Obtain the artifact ref
	171	+ cache_key = cli.get_element_key(project, element)
	172	+ artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
	173	+
	174	+ # Try deleting the uncached artifact
	175	+ result = cli.run(project=project, args=['artifact', 'delete', artifact])
	176	+ result.assert_success()
	177	+
	178	+ expected_err = 'WARNING: {}, not found in local cache - no delete required\n'.format(artifact)
	179	+ assert result.stderr == expected_err
	180	+
	181	+
	182	+# Test that an artifact pulled from it's remote cache (without it's buildtree) will not
	183	+# throw an Exception when trying to prune the cache.
	184	+@pytest.mark.integration
	185	+@pytest.mark.datafiles(DATA_DIR)
	186	+def test_artifact_delete_pulled_artifact_without_buildtree(cli, tmpdir, datafiles):
	187	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	188	+ element = 'autotools/amhello.bst'
	189	+
	190	+ # Set up remote and local shares
	191	+ local_cache = os.path.join(str(tmpdir), 'artifacts')
	192	+ with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as remote:
	193	+ cli.configure({
	194	+ 'artifacts': {'url': remote.repo, 'push': True},
	195	+ 'artifactdir': local_cache,
	196	+ })
	197	+
	198	+ # Build the element
	199	+ result = cli.run(project=project, args=['build', element])
	200	+ result.assert_success()
	201	+
	202	+ # Make sure it's in the share
	203	+ cache_key = cli.get_element_key(project, element)
	204	+ assert remote.has_artifact('test', element, cache_key)
	205	+
	206	+ # Delete and then pull the artifact (without its buildtree)
	207	+ result = cli.run(project=project, args=['artifact', 'delete', element])
	208	+ result.assert_success()
	209	+ assert cli.get_element_state(project, element) != 'cached'
	210	+ result = cli.run(project=project, args=['pull', element])
	211	+ result.assert_success()
	212	+ assert cli.get_element_state(project, element) == 'cached'
	213	+
	214	+ # Now delete it again (it should have been pulled without the buildtree, but
	215	+ # a digest of the buildtree is pointed to in the artifact's metadata
	216	+ result = cli.run(project=project, args=['artifact', 'delete', element])
	217	+ result.assert_success()
	218	+ assert cli.get_element_state(project, element) != 'cached'

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.