[Notes] [Git][BuildStream/buildstream][jennis/new_artifact_subcommands] 9 commits: cli: Split classify_artifacts helper up

[James Ennis <gitlab mg gitlab com>]

Title: GitLab

James Ennis pushed to branch jennis/new_artifact_subcommands at BuildStream / buildstream

Commits:

• 1894178d
  by Richard Maw at 2019-01-14T10:10:25Z

  cli: Split classify_artifacts helper up
  
  Classifying artifact refs is useful for glob expansion
  even in cases where an element target name would be inappropriate.
  

• 5c74ab23
  by Richard Maw at 2019-01-14T10:10:25Z

  cli: Add artifact delete subcommand
  

• da0755cd
  by James Ennis at 2019-01-14T10:10:25Z

  artifactcache.py: Add prune() method to ArtifactCache
  
  Removal/pruning should be handled by ArtifactCache API. We should
  not be using CASCache methods to remove BuildStream's artifacts.
  

• dfae4333
  by James Ennis at 2019-01-14T10:10:25Z

  cli.py: Defer pruning until all specified refs are removed
  

• 778899bd
  by James Ennis at 2019-01-14T10:25:59Z

  tests: Add new tests for bst artifact delete
  
  These tests require a sandbox to build and have been marked to
  skip accordingly.
  

• bb9c7e72
  by James Ennis at 2019-01-14T10:26:51Z

  artifactcache.py: API Cleanup - Add contains_ref() method
  
  This ensures that we can determine whether an artifact is locally cached
  (as well as an element) using the ArtifactCache API.
  

• f1fb622a
  by James Ennis at 2019-01-14T10:41:48Z

  cli.py: Print to stderr when trying to delete an uncached artifact
  
  If we specify multiple elements/artifacts for deletion, we should not
  throw an exception, and more importantly, stop, when we come across
  an artifact that is uncached.
  

• a6b6edb9
  by James Ennis at 2019-01-14T10:41:48Z

  cli.py: Ensure that both 'weak' and 'strong' artifacts are deleted
  

• e0ace830
  by James Ennis at 2019-01-14T10:41:48Z

  cascache.py: Ensure path exists before trying to update the mtime
  
  If an artifact is pulled from the cache without its buildtree,
  CASCache.prune() will fail when it tries to update the mtimes
  of the build tree's object files.
  
  A new integration test has been added to tests/integration/artifact.py
  which reflects this.
  

4 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_artifactcache/cascache.py
• buildstream/_frontend/cli.py
• tests/integration/artifact.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -421,18 +421,30 @@ class ArtifactCache():
421	421
422	422	# contains():
423	423	#
424		- # Check whether the artifact for the specified Element is already available
425		- # in the local artifact cache.
	424	+ # Check whether the (project state) artifact of the specified Element is
	425	+ # already available in the local artifact cache.
426	426	#
427	427	# Args:
428	428	# element (Element): The Element to check
429	429	# key (str): The cache key to use
430	430	#
431		- # Returns: True if the artifact is in the cache, False otherwise
	431	+ # Returns: True if the Element's (project state) artifact is in the cache,
	432	+ # False otherwise
432	433	#
433	434	def contains(self, element, key):
434	435	ref = self.get_artifact_fullname(element, key)
	436	+ return self.contains_ref(ref)
435	437
	438	+ # contains_ref():
	439	+ #
	440	+ # Check whether an artifact is already available in the local artifact cache.
	441	+ #
	442	+ # Args:
	443	+ # ref (str): The ref to check
	444	+ #
	445	+ # Returns: True if the artifact is in the cache, False otherwise
	446	+ #
	447	+ def contains_ref(self, ref):
436	448	return self.cas.contains(ref)
437	449
438	450	# contains_subdir_artifact():
...	...	@@ -476,8 +488,7 @@ class ArtifactCache():
476	488	# (int|None) The amount of space pruned from the repository in
477	489	# Bytes, or None if defer_prune is True
478	490	#
479		- def remove(self, ref):
480		-
	491	+ def remove(self, ref, *, defer_prune=False):
481	492	# Remove extract if not used by other ref
482	493	tree = self.cas.resolve_ref(ref)
483	494	ref_name, ref_hash = os.path.split(ref)
...	...	@@ -496,7 +507,21 @@ class ArtifactCache():
496	507	if remove_extract:
497	508	utils._force_rmtree(extract)
498	509
499		- return self.cas.remove(ref)
	510	+ return self.cas.remove(ref, defer_prune=defer_prune)
	511	+
	512	+ # prune():
	513	+ #
	514	+ # Prunes the artifact cache of objects which are unreachable from
	515	+ # the repo
	516	+ #
	517	+ # Args:
	518	+ # None
	519	+ #
	520	+ # Returns:
	521	+ # (int) The amount of space pruned from the repository in bytes
	522	+ #
	523	+ def prune(self):
	524	+ return self.cas.prune()
500	525
501	526	# extract():
502	527	#

buildstream/_artifactcache/cascache.py

---

...	...	@@ -672,7 +672,6 @@ class CASCache():
672	672	# Bytes, or None if defer_prune is True
673	673	#
674	674	def remove(self, ref, *, defer_prune=False):
675		-
676	675	# Remove cache ref
677	676	refpath = self._refpath(ref)
678	677	if not os.path.exists(refpath):
...	...	@@ -870,6 +869,9 @@ class CASCache():
870	869	if tree.hash in reachable:
871	870	return
872	871
	872	+ if not os.path.exists(self.objpath(tree)):
	873	+ return
	874	+
873	875	if update_mtime:
874	876	os.utime(self.objpath(tree))
875	877

buildstream/_frontend/cli.py

---

...	...	@@ -6,6 +6,7 @@ from tempfile import TemporaryDirectory
6	6
7	7	import click
8	8	from .. import _yaml
	9	+from ..types import _KeyStrength
9	10	from .._exceptions import BstError, LoadError, AppError
10	11	from .._versions import BST_FORMAT_VERSION
11	12	from .complete import main_bashcomplete, complete_path, CompleteUnhandled
...	...	@@ -973,36 +974,47 @@ def workspace_list(app):
973	974	#############################################################
974	975	# Artifact Commands #
975	976	#############################################################
976		-def _classify_artifacts(names, cas, project_directory):
977		- element_targets = []
978		- artifact_refs = []
979		- element_globs = []
980		- artifact_globs = []
981		-
	977	+def _classify_element_targets(names, project_directory):
	978	+ globs = []
	979	+ targets = []
	980	+ unmatched = []
982	981	for name in names:
983	982	if name.endswith('.bst'):
984	983	if any(c in "*?[" for c in name):
985		- element_globs.append(name)
	984	+ globs.append(name)
986	985	else:
987		- element_targets.append(name)
	986	+ targets.append(name)
988	987	else:
989		- if any(c in "*?[" for c in name):
990		- artifact_globs.append(name)
991		- else:
992		- artifact_refs.append(name)
	988	+ unmatched.append(name)
993	989
994		- if element_globs:
	990	+ if globs:
995	991	for dirpath, _, filenames in os.walk(project_directory):
996	992	for filename in filenames:
997		- element_path = os.path.join(dirpath, filename).lstrip(project_directory).lstrip('/')
998		- if any(fnmatch(element_path, glob) for glob in element_globs):
999		- element_targets.append(element_path)
	993	+ element_path = os.path.relpath(os.path.join(dirpath, filename), start=project_directory)
	994	+ if any(fnmatch(element_path, glob) for glob in globs):
	995	+ targets.append(element_path)
	996	+ return targets, unmatched
	997	+
	998	+
	999	+def _classify_artifact_refs(names, cas):
	1000	+ globs = []
	1001	+ refs = []
	1002	+ for name in names:
	1003	+ if any(c in "*?[" for c in name):
	1004	+ globs.append(name)
	1005	+ else:
	1006	+ refs.append(name)
	1007	+ if globs:
	1008	+ refs.extend(ref for ref in cas.list_refs()
	1009	+ if any(fnmatch(ref, glob) for glob in globs))
	1010	+ return refs
1000	1011
1001		- if artifact_globs:
1002		- artifact_refs.extend(ref for ref in cas.list_refs()
1003		- if any(fnmatch(ref, glob) for glob in artifact_globs))
1004	1012
1005		- return element_targets, artifact_refs
	1013	+def _classify_artifacts(names, cas, project_directory):
	1014	+ targets, unmatched = _classify_element_targets(names, project_directory)
	1015	+ refs = _classify_artifact_refs(unmatched, cas)
	1016	+
	1017	+ return targets, refs
1006	1018
1007	1019
1008	1020	@cli.group(short_help="Manipulate cached artifacts")
...	...	@@ -1067,6 +1079,63 @@ def artifact_log(app, artifacts):
1067	1079	click.echo_via_pager(data)
1068	1080
1069	1081
	1082	+###################################################################
	1083	+# Artifact Delete Command #
	1084	+###################################################################
	1085	+@artifact.command(name='delete', short_help="Delete matching artifacts")
	1086	+@click.argument('artifacts', type=click.Path(), nargs=-1)
	1087	+@click.pass_obj
	1088	+def artifact_delete(app, artifacts):
	1089	+ '''Delete matching artifacts from the cache'''
	1090	+ from .._pipeline import PipelineSelection
	1091	+
	1092	+ with app.initialized():
	1093	+ cache = app.context.artifactcache
	1094	+
	1095	+ elements, artifacts = _classify_artifacts(artifacts, cache.cas,
	1096	+ app.project.directory)
	1097	+
	1098	+ if not elements and not artifacts:
	1099	+ element = app.context.guess_element()
	1100	+ if element is not None:
	1101	+ elements = [element]
	1102	+
	1103	+ # Remove specified elements and artifacts
	1104	+ if elements:
	1105	+ elements = app.stream.load_selection(elements, selection=PipelineSelection.NONE)
	1106	+ for element in elements:
	1107	+ cache_keys = set([element._get_cache_key(),
	1108	+ element._get_cache_key(strength=_KeyStrength.WEAK)])
	1109	+ for cache_key in cache_keys:
	1110	+ ref = cache.get_artifact_fullname(element, cache_key)
	1111	+ if cache.contains(element, cache_key):
	1112	+ cache.remove(ref, defer_prune=True)
	1113	+ click.echo("Removed {}.".format(ref))
	1114	+ else:
	1115	+ # If the ref is not present when we try to delete it, we should
	1116	+ # not fail but just continue to delete. The pruning will take care
	1117	+ # of any unreachable objects.
	1118	+ click.echo("WARNING: {}, not found in local cache - no delete required"
	1119	+ .format(ref), err=True)
	1120	+ continue
	1121	+
	1122	+ if artifacts:
	1123	+ for ref in artifacts:
	1124	+ if cache.contains_ref(ref):
	1125	+ cache.remove(ref, defer_prune=True)
	1126	+ click.echo("Removed {}.".format(ref))
	1127	+ else:
	1128	+ # If the ref is not present when we try to delete it, we should
	1129	+ # not fail but just continue to delete. The pruning will take care
	1130	+ # of any unreachable objects.
	1131	+ click.echo("WARNING: {}, not found in local cache - no delete required"
	1132	+ .format(ref), err=True)
	1133	+ continue
	1134	+
	1135	+ # Now we've removed all the refs, prune the unreachable objects
	1136	+ cache.prune()
	1137	+
	1138	+
1070	1139	##################################################################
1071	1140	# DEPRECATED Commands #
1072	1141	##################################################################

tests/integration/artifact.py

---

...	...	@@ -21,8 +21,8 @@
21	21	import os
22	22	import pytest
23	23
24		-from tests.testutils import cli_integration as cli
25		-
	24	+from tests.testutils import cli_integration as cli, create_artifact_share
	25	+from tests.testutils.site import HAVE_BWRAP, IS_LINUX
26	26
27	27	pytestmark = pytest.mark.integration
28	28
...	...	@@ -66,3 +66,157 @@ def test_artifact_log(cli, tmpdir, datafiles):
66	66	assert result.exit_code == 0
67	67	# The artifact is cached under both a strong key and a weak key
68	68	assert (log + log) == result.output
	69	+
	70	+
	71	+# Test that we can delete the artifact of the element which corresponds
	72	+# to the current project state
	73	+@pytest.mark.integration
	74	+@pytest.mark.datafiles(DATA_DIR)
	75	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	76	+def test_artifact_delete_element(cli, tmpdir, datafiles):
	77	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	78	+ element = 'integration.bst'
	79	+
	80	+ # Build the element and ensure it's cached
	81	+ result = cli.run(project=project, args=['build', element])
	82	+ result.assert_success()
	83	+ assert cli.get_element_state(project, element) == 'cached'
	84	+
	85	+ result = cli.run(project=project, args=['artifact', 'delete', element])
	86	+ result.assert_success()
	87	+ assert cli.get_element_state(project, element) != 'cached'
	88	+
	89	+
	90	+# Test that we can delete an artifact by specifying its ref.
	91	+@pytest.mark.integration
	92	+@pytest.mark.datafiles(DATA_DIR)
	93	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	94	+def test_artifact_delete_artifact(cli, tmpdir, datafiles):
	95	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	96	+ element = 'integration.bst'
	97	+
	98	+ # Configure a local cache
	99	+ local_cache = os.path.join(str(tmpdir), 'artifacts')
	100	+ cli.configure({'artifactdir': local_cache})
	101	+
	102	+ # First build an element so that we can find its artifact
	103	+ result = cli.run(project=project, args=['build', element])
	104	+ result.assert_success()
	105	+
	106	+ # Obtain the artifact ref
	107	+ cache_key = cli.get_element_key(project, element)
	108	+ artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
	109	+
	110	+ # Explicitly check that the ARTIFACT exists in the cache
	111	+ assert os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
	112	+
	113	+ # Delete the artifact
	114	+ result = cli.run(project=project, args=['artifact', 'delete', artifact])
	115	+ result.assert_success()
	116	+
	117	+ # Check that the ARTIFACT is no longer in the cache
	118	+ assert not os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
	119	+
	120	+
	121	+# Test the `bst artifact delete` command with multiple, different arguments.
	122	+@pytest.mark.integration
	123	+@pytest.mark.datafiles(DATA_DIR)
	124	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	125	+def test_artifact_delete_element_and_artifact(cli, tmpdir, datafiles):
	126	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	127	+ element = 'integration.bst'
	128	+ dep = 'base/base-alpine.bst'
	129	+
	130	+ # Configure a local cache
	131	+ local_cache = os.path.join(str(tmpdir), 'artifacts')
	132	+ cli.configure({'artifactdir': local_cache})
	133	+
	134	+ # First build an element so that we can find its artifact
	135	+ result = cli.run(project=project, args=['build', element])
	136	+ result.assert_success()
	137	+ assert cli.get_element_state(project, element) == 'cached'
	138	+ assert cli.get_element_state(project, dep) == 'cached'
	139	+
	140	+ # Obtain the artifact ref
	141	+ cache_key = cli.get_element_key(project, element)
	142	+ artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
	143	+
	144	+ # Explicitly check that the ARTIFACT exists in the cache
	145	+ assert os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
	146	+
	147	+ # Delete the artifact
	148	+ result = cli.run(project=project, args=['artifact', 'delete', artifact, dep])
	149	+ result.assert_success()
	150	+
	151	+ # Check that the ARTIFACT is no longer in the cache
	152	+ assert not os.path.exists(os.path.join(local_cache, 'cas', 'refs', 'heads', artifact))
	153	+
	154	+ # Check that the dependency ELEMENT is no longer cached
	155	+ assert cli.get_element_state(project, dep) != 'cached'
	156	+
	157	+
	158	+# Test that we receive the appropriate stderr when we try to delete an artifact
	159	+# that is not present in the cache.
	160	+@pytest.mark.integration
	161	+@pytest.mark.datafiles(DATA_DIR)
	162	+def test_artifact_delete_unbuilt_artifact(cli, tmpdir, datafiles):
	163	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	164	+ element = 'integration.bst'
	165	+
	166	+ # Configure a local cache
	167	+ local_cache = os.path.join(str(tmpdir), 'artifacts')
	168	+ cli.configure({'artifactdir': local_cache})
	169	+
	170	+ # Ensure the element is not cached
	171	+ assert cli.get_element_state(project, element) != 'cached'
	172	+
	173	+ # Obtain the artifact ref
	174	+ cache_key = cli.get_element_key(project, element)
	175	+ artifact = os.path.join('test', os.path.splitext(element)[0], cache_key)
	176	+
	177	+ # Try deleting the uncached artifact
	178	+ result = cli.run(project=project, args=['artifact', 'delete', artifact])
	179	+ result.assert_success()
	180	+
	181	+ expected_err = 'WARNING: {}, not found in local cache - no delete required\n'.format(artifact)
	182	+ assert result.stderr == expected_err
	183	+
	184	+
	185	+# Test that an artifact pulled from it's remote cache (without it's buildtree) will not
	186	+# throw an Exception when trying to prune the cache.
	187	+@pytest.mark.integration
	188	+@pytest.mark.datafiles(DATA_DIR)
	189	+@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
	190	+def test_artifact_delete_pulled_artifact_without_buildtree(cli, tmpdir, datafiles):
	191	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	192	+ element = 'autotools/amhello.bst'
	193	+
	194	+ # Set up remote and local shares
	195	+ local_cache = os.path.join(str(tmpdir), 'artifacts')
	196	+ with create_artifact_share(os.path.join(str(tmpdir), 'remote')) as remote:
	197	+ cli.configure({
	198	+ 'artifacts': {'url': remote.repo, 'push': True},
	199	+ 'artifactdir': local_cache,
	200	+ })
	201	+
	202	+ # Build the element
	203	+ result = cli.run(project=project, args=['build', element])
	204	+ result.assert_success()
	205	+
	206	+ # Make sure it's in the share
	207	+ cache_key = cli.get_element_key(project, element)
	208	+ assert remote.has_artifact('test', element, cache_key)
	209	+
	210	+ # Delete and then pull the artifact (without its buildtree)
	211	+ result = cli.run(project=project, args=['artifact', 'delete', element])
	212	+ result.assert_success()
	213	+ assert cli.get_element_state(project, element) != 'cached'
	214	+ result = cli.run(project=project, args=['pull', element])
	215	+ result.assert_success()
	216	+ assert cli.get_element_state(project, element) == 'cached'
	217	+
	218	+ # Now delete it again (it should have been pulled without the buildtree, but
	219	+ # a digest of the buildtree is pointed to in the artifact's metadata
	220	+ result = cli.run(project=project, args=['artifact', 'delete', element])
	221	+ result.assert_success()
	222	+ assert cli.get_element_state(project, element) != 'cached'

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.