[Notes] [Git][BuildStream/buildstream][jennis/migrate_pull_push_commands] 9 commits: Move push and pull to the new artifact subcommand group

[James Ennis <gitlab mg gitlab com>]

Title: GitLab

James Ennis pushed to branch jennis/migrate_pull_push_commands at BuildStream / buildstream

Commits:

• 94b309c3
  by James Ennis at 2019-01-15T16:53:29Z

  Move push and pull to the new artifact subcommand group
  
  This commit also ensures that if we try to use the 'old' commands,
  BuildStream will fail and instruct the user to use the new command.
  

• aceb3995
  by Richard Maw at 2019-01-15T16:53:29Z

  cli: Split classify_artifacts helper up
  
  Classifying artifact refs is useful for glob expansion
  even in cases where an element target name would be inappropriate.
  

• cb6d1c46
  by James Ennis at 2019-01-15T16:53:29Z

  artifactcache.py: Add new method ArtifactCache.create_remote()
  
  This allows for the creation of a CASRemote object using the
  ArtifactCache API
  

• 0aeeef62
  by James Ennis at 2019-01-15T16:53:29Z

  artifactcache.py: Allow ArtifactCache to handle pulling/pushing refs
  
  With the addition of the artifact subcommands, which are able to
  process artifact refs, we should not have to rely on CASCache
  internals to handle artifacts. This commit ensures that the
  pulling/pushing of elements (and their cache keys) is handled
  by ArtifactCache.
  

• 7e40da0a
  by James Ennis at 2019-01-15T16:53:29Z

  artifactcache.py: API Cleanup - Add contains_ref() method
  
  This ensures that we can determine whether an artifact is locally cached
  (as well as an element) using the ArtifactCache API.
  

• 672ce447
  by James Ennis at 2019-01-15T16:53:30Z

  WIP: Ensure push/pull can deal with artifact refs
  

• 1fe8ab79
  by James Ennis at 2019-01-15T16:53:30Z

  cli.py: Handle the pulling of an artifact's buildtree
  

• cbd7918e
  by James Ennis at 2019-01-15T16:53:30Z

  cli.py: Handle the --remote option for pulling artifacts
  

• e70b6f9e
  by James Ennis at 2019-01-15T16:53:30Z

  WIP: cli.py: Spawn pull jobs in another process to isolate gRPC calls
  

12 changed files:

• buildstream/_artifactcache/artifactcache.py
• buildstream/_frontend/cli.py
• tests/artifactcache/config.py
• tests/artifactcache/junctions.py
• tests/completions/completions.py
• tests/frontend/help.py
• tests/frontend/pull.py
• tests/frontend/push.py
• tests/frontend/workspace.py
• tests/integration/build-tree.py
• tests/integration/pullbuildtrees.py
• tests/sandboxes/remote-exec-config.py

Changes:

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -399,7 +399,7 @@ class ArtifactCache():
399	399	if remote_spec.push:
400	400	self._has_push_remotes = True
401	401
402		- remotes[remote_spec.url] = CASRemote(remote_spec)
	402	+ remotes[remote_spec.url] = self.create_remote(remote_spec)
403	403
404	404	for project in self.context.get_projects():
405	405	remote_specs = self.global_remote_specs
...	...	@@ -419,20 +419,35 @@ class ArtifactCache():
419	419
420	420	self._remotes[project] = project_remotes
421	421
	422	+ def create_remote(self, remote_spec):
	423	+ return CASRemote(remote_spec)
	424	+
422	425	# contains():
423	426	#
424		- # Check whether the artifact for the specified Element is already available
425		- # in the local artifact cache.
	427	+ # Check whether the (project state) artifact of the specified Element is
	428	+ # already available in the local artifact cache.
426	429	#
427	430	# Args:
428	431	# element (Element): The Element to check
429	432	# key (str): The cache key to use
430	433	#
431		- # Returns: True if the artifact is in the cache, False otherwise
	434	+ # Returns: True if the Element's (project state) artifact is in the cache,
	435	+ # False otherwise
432	436	#
433	437	def contains(self, element, key):
434	438	ref = self.get_artifact_fullname(element, key)
	439	+ return self.contains_ref(ref)
435	440
	441	+ # contains_ref():
	442	+ #
	443	+ # Check whether an artifact is already available in the local artifact cache.
	444	+ #
	445	+ # Args:
	446	+ # ref (str): The ref to check
	447	+ #
	448	+ # Returns: True if the artifact is in the cache, False otherwise
	449	+ #
	450	+ def contains_ref(self, ref):
436	451	return self.cas.contains(ref)
437	452
438	453	# contains_subdir_artifact():
...	...	@@ -658,7 +673,7 @@ class ArtifactCache():
658	673	display_key = element._get_brief_display_key()
659	674	element.status("Pulling artifact {} <- {}".format(display_key, remote.spec.url))
660	675
661		- if self.cas.pull(ref, remote, progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs):
	676	+ if self.pull_ref(ref, remote, progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs):
662	677	element.info("Pulled artifact {} <- {}".format(display_key, remote.spec.url))
663	678	if subdir:
664	679	# Attempt to extract subdir into artifact extract dir if it already exists
...	...	@@ -678,6 +693,23 @@ class ArtifactCache():
678	693
679	694	return False
680	695
	696	+ # pull_ref():
	697	+ #
	698	+ # Pull artifact from one of the configured remote repositories.
	699	+ #
	700	+ # Args:
	701	+ # ref (str): The ref to pull
	702	+ # progress (callable): The progress callback, if any
	703	+ # subdir (str): The optional specific subdir to pull
	704	+ # excluded_subdirs (list): The optional list of subdirs to not pull
	705	+ #
	706	+ # Returns:
	707	+ # (bool): True if pull was successful, False if artifact was not available
	708	+ #
	709	+ def pull_ref(self, ref, remote, *, progress=None, subdir=None, excluded_subdirs=None):
	710	+ return self.cas.pull(ref, remote, progress=progress, subdir=subdir, excluded_subdirs=excluded_subdirs)
	711	+
	712	+
681	713	# pull_tree():
682	714	#
683	715	# Pull a single Tree rather than an artifact.

buildstream/_frontend/cli.py

---

	1	+import multiprocessing
1	2	import os
	3	+import signal
2	4	import sys
3	5	from contextlib import ExitStack
4	6	from fnmatch import fnmatch
...	...	@@ -355,78 +357,6 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
355	357	build_all=all_)
356	358
357	359
358		-##################################################################
359		-# Pull Command #
360		-##################################################################
361		-@cli.command(short_help="Pull a built artifact")
362		-@click.option('--deps', '-d', default='none',
363		- type=click.Choice(['none', 'all']),
364		- help='The dependency artifacts to pull (default: none)')
365		-@click.option('--remote', '-r',
366		- help="The URL of the remote cache (defaults to the first configured cache)")
367		-@click.argument('elements', nargs=-1,
368		- type=click.Path(readable=False))
369		-@click.pass_obj
370		-def pull(app, elements, deps, remote):
371		- """Pull a built artifact from the configured remote artifact cache.
372		-
373		- By default the artifact will be pulled one of the configured caches
374		- if possible, following the usual priority order. If the `--remote` flag
375		- is given, only the specified cache will be queried.
376		-
377		- Specify `--deps` to control which artifacts to pull:
378		-
379		- \b
380		- none: No dependencies, just the element itself
381		- all: All dependencies
382		- """
383		-
384		- with app.initialized(session_name="Pull"):
385		- if not elements:
386		- guessed_target = app.context.guess_element()
387		- if guessed_target:
388		- elements = (guessed_target,)
389		-
390		- app.stream.pull(elements, selection=deps, remote=remote)
391		-
392		-
393		-##################################################################
394		-# Push Command #
395		-##################################################################
396		-@cli.command(short_help="Push a built artifact")
397		-@click.option('--deps', '-d', default='none',
398		- type=click.Choice(['none', 'all']),
399		- help='The dependencies to push (default: none)')
400		-@click.option('--remote', '-r', default=None,
401		- help="The URL of the remote cache (defaults to the first configured cache)")
402		-@click.argument('elements', nargs=-1,
403		- type=click.Path(readable=False))
404		-@click.pass_obj
405		-def push(app, elements, deps, remote):
406		- """Push a built artifact to a remote artifact cache.
407		-
408		- The default destination is the highest priority configured cache. You can
409		- override this by passing a different cache URL with the `--remote` flag.
410		-
411		- If bst has been configured to include build trees on artifact pulls,
412		- an attempt will be made to pull any required build trees to avoid the
413		- skipping of partial artifacts being pushed.
414		-
415		- Specify `--deps` to control which artifacts to push:
416		-
417		- \b
418		- none: No dependencies, just the element itself
419		- all: All dependencies
420		- """
421		- with app.initialized(session_name="Push"):
422		- if not elements:
423		- guessed_target = app.context.guess_element()
424		- if guessed_target:
425		- elements = (guessed_target,)
426		-
427		- app.stream.push(elements, selection=deps, remote=remote)
428		-
429		-
430	360	##################################################################
431	361	# Show Command #
432	362	##################################################################
...	...	@@ -973,36 +903,47 @@ def workspace_list(app):
973	903	#############################################################
974	904	# Artifact Commands #
975	905	#############################################################
976		-def _classify_artifacts(names, cas, project_directory):
977		- element_targets = []
978		- artifact_refs = []
979		- element_globs = []
980		- artifact_globs = []
981		-
	906	+def _classify_element_targets(names, project_directory):
	907	+ globs = []
	908	+ targets = []
	909	+ unmatched = []
982	910	for name in names:
983	911	if name.endswith('.bst'):
984	912	if any(c in "*?[" for c in name):
985		- element_globs.append(name)
	913	+ globs.append(name)
986	914	else:
987		- element_targets.append(name)
	915	+ targets.append(name)
988	916	else:
989		- if any(c in "*?[" for c in name):
990		- artifact_globs.append(name)
991		- else:
992		- artifact_refs.append(name)
	917	+ unmatched.append(name)
993	918
994		- if element_globs:
	919	+ if globs:
995	920	for dirpath, _, filenames in os.walk(project_directory):
996	921	for filename in filenames:
997		- element_path = os.path.join(dirpath, filename).lstrip(project_directory).lstrip('/')
998		- if any(fnmatch(element_path, glob) for glob in element_globs):
999		- element_targets.append(element_path)
	922	+ element_path = os.path.relpath(os.path.join(dirpath, filename), start=project_directory)
	923	+ if any(fnmatch(element_path, glob) for glob in globs):
	924	+ targets.append(element_path)
	925	+ return targets, unmatched
	926	+
	927	+
	928	+def _classify_artifact_refs(names, cas):
	929	+ globs = []
	930	+ refs = []
	931	+ for name in names:
	932	+ if any(c in "*?[" for c in name):
	933	+ globs.append(name)
	934	+ else:
	935	+ refs.append(name)
	936	+ if globs:
	937	+ refs.extend(ref for ref in cas.list_refs()
	938	+ if any(fnmatch(ref, glob) for glob in globs))
	939	+ return refs
	940	+
1000	941
1001		- if artifact_globs:
1002		- artifact_refs.extend(ref for ref in cas.list_refs()
1003		- if any(fnmatch(ref, glob) for glob in artifact_globs))
	942	+def _classify_artifacts(names, cas, project_directory):
	943	+ targets, unmatched = _classify_element_targets(names, project_directory)
	944	+ refs = _classify_artifact_refs(unmatched, cas)
1004	945
1005		- return element_targets, artifact_refs
	946	+ return targets, refs
1006	947
1007	948
1008	949	@cli.group(short_help="Manipulate cached artifacts")
...	...	@@ -1010,6 +951,176 @@ def artifact():
1010	951	"""Manipulate cached artifacts"""
1011	952
1012	953
	954	+################################################################
	955	+# Artifact Pull Command #
	956	+################################################################
	957	+@artifact.command(name="pull", short_help="Pull a built artifact")
	958	+@click.option('--deps', '-d', default='none',
	959	+ type=click.Choice(['none', 'all']),
	960	+ help='The dependency artifacts to pull (default: none)')
	961	+@click.option('--remote', '-r',
	962	+ help="The URL of the remote cache (defaults to the first configured cache)")
	963	+@click.argument('artifacts', type=click.Path(), nargs=-1)
	964	+@click.pass_obj
	965	+def artifact_pull(app, artifacts, deps, remote):
	966	+ """Pull a built artifact from the configured remote artifact cache.
	967	+
	968	+ By default the artifact will be pulled one of the configured caches
	969	+ if possible, following the usual priority order. If the `--remote` flag
	970	+ is given, only the specified cache will be queried.
	971	+
	972	+ Specify `--deps` to control which artifacts to pull:
	973	+
	974	+ \b
	975	+ none: No dependencies, just the element itself
	976	+ all: All dependencies
	977	+ """
	978	+ from .. import _signals
	979	+ from .. import utils
	980	+ from .._exceptions import CASError
	981	+
	982	+ with app.initialized(session_name="Pull"):
	983	+ cache = app.context.artifactcache
	984	+
	985	+ elements, artifacts = _classify_artifacts(artifacts, cache.cas,
	986	+ app.project.directory)
	987	+
	988	+ # Guess the element if we're in a workspace
	989	+ if not elements and not artifacts:
	990	+ guessed_target = app.context.guess_element()
	991	+ if guessed_target:
	992	+ elements = (guessed_target,)
	993	+
	994	+ if artifacts and deps != 'none':
	995	+ raise AppError("--deps may not be used with artifact refs")
	996	+
	997	+ if elements:
	998	+ app.stream.pull(elements, selection=deps, remote=remote)
	999	+ if artifacts:
	1000	+ # Determine all available remotes
	1001	+ usr_remotes = app.context.artifact_cache_specs # list of CASRemoteSpec objects
	1002	+
	1003	+ project = app.context.get_toplevel_project() # Should I get all projects here...?
	1004	+ project.ensure_fully_loaded()
	1005	+ project_remotes = project.artifact_cache_specs # list of CASRemoteSpec objects
	1006	+
	1007	+ remotes = usr_remotes + project_remotes
	1008	+
	1009	+ if remote: # Check if this is one of the available remotes
	1010	+ # Here we are restricting the user to only use a remote that is defined
	1011	+ # in either the project.conf or the user config, should the user be able to *try*
	1012	+ # and pull from any specified remote?
	1013	+ remote_found = False
	1014	+ for spec in remotes:
	1015	+ if remote == spec.url:
	1016	+ remotes = [spec]
	1017	+ remote_found = True
	1018	+ if not remote_found:
	1019	+ raise AppError("Remote: '{}' not found.".format(remote))
	1020	+
	1021	+ # Pull buildtrees?
	1022	+ excluded_subdirs = ["buildtree"] if app.context.pull_buildtrees else None
	1023	+
	1024	+ # Define function to be used by multiprocessing
	1025	+ def _pull_artifact(artifact_cache_obj, remote_specs, refs, excluded_subdirs, queue):
	1026	+ remotes = [artifact_cache_obj.create_remote(spec) for spec in remote_specs]
	1027	+ pull_attempts = {'skipped': [], 'pulled': [], 'failed': []}
	1028	+ for ref in refs:
	1029	+ if artifact_cache_obj.contains_ref(ref):
	1030	+ pull_attempts['skipped'].append(ref)
	1031	+ continue
	1032	+
	1033	+ pulled = False
	1034	+ for remote in remotes:
	1035	+ try:
	1036	+ if artifact_cache_obj.pull_ref(ref, remote, excluded_subdirs=excluded_subdirs):
	1037	+ pull_attempts['pulled'].append((ref, remote.spec.url))
	1038	+ break
	1039	+ except CASError as e:
	1040	+ queue.put(e)
	1041	+ raise
	1042	+ if not pulled:
	1043	+ pull_attempts['failed'].append(ref)
	1044	+
	1045	+ queue.put(pull_attempts)
	1046	+
	1047	+ q = multiprocessing.Queue()
	1048	+ p = multiprocessing.Process(target=_pull_artifact, args=(cache, remotes, artifacts, excluded_subdirs, q))
	1049	+ try:
	1050	+ with _signals.blocked([signal.SIGINT], ignore=False):
	1051	+ p.start()
	1052	+
	1053	+ result = q.get()
	1054	+ p.join()
	1055	+ except KeyboardInterrupt:
	1056	+ utils._kill_process_tree(p.pid)
	1057	+ raise
	1058	+
	1059	+ # Output to user
	1060	+ if isinstance(result, dict):
	1061	+ for ref in result['skipped']:
	1062	+ click.echo("'{}' already available - pull skipped.".format(ref))
	1063	+ for ref, remote in result['pulled']:
	1064	+ click.echo("'{}' pulled from '{}'.".format(ref, remote))
	1065	+ for ref in result['failed']:
	1066	+ urls = [remote.url for remote in remotes]
	1067	+ click.echo("'{}' not available in remotes: {}".format(ref, urls)) # Ugly list print
	1068	+ else:
	1069	+ raise result
	1070	+
	1071	+
	1072	+##################################################################
	1073	+# Artifact Push Command #
	1074	+##################################################################
	1075	+@artifact.command(name="push", short_help="Push a built artifact")
	1076	+@click.option('--deps', '-d', default='none',
	1077	+ type=click.Choice(['none', 'all']),
	1078	+ help='The dependencies to push (default: none)')
	1079	+@click.option('--remote', '-r', default=None,
	1080	+ help="The URL of the remote cache (defaults to the first configured cache)")
	1081	+@click.argument('artifacts', type=click.Path(), nargs=-1)
	1082	+@click.pass_obj
	1083	+def artifact_push(app, artifacts, deps, remote):
	1084	+ """Push a built artifact to a remote artifact cache.
	1085	+
	1086	+ The default destination is the highest priority configured cache. You can
	1087	+ override this by passing a different cache URL with the `--remote` flag.
	1088	+
	1089	+ If bst has been configured to include build trees on artifact pulls,
	1090	+ an attempt will be made to pull any required build trees to avoid the
	1091	+ skipping of partial artifacts being pushed.
	1092	+
	1093	+ Specify `--deps` to control which artifacts to push:
	1094	+
	1095	+ \b
	1096	+ none: No dependencies, just the element itself
	1097	+ all: All dependencies
	1098	+ """
	1099	+ with app.initialized(session_name="Push"):
	1100	+ cache = app.context.artifactcache
	1101	+
	1102	+ elements, artifacts = _classify_artifacts(artifacts, cache.cas,
	1103	+ app.project.directory)
	1104	+
	1105	+ # Guess the element if we're in a workspace
	1106	+ if not elements:
	1107	+ guessed_target = app.context.guess_element()
	1108	+ if guessed_target:
	1109	+ elements = (guessed_target,)
	1110	+
	1111	+ if artifacts and deps != 'none':
	1112	+ raise AppError("--deps may not be used with artifact refs")
	1113	+
	1114	+ if elements:
	1115	+ app.stream.push(elements, selection=deps, remote=remote)
	1116	+ # FIXME: We can only obtain project/user config through the stream API,
	1117	+ # which we need to determine the remote in order for pull to pull from.
	1118	+ # We can't just go straight to artifactcache here. Thus Stream.Pull()
	1119	+ # will fail because it expects a list of element names (.bst).
	1120	+ if artifacts:
	1121	+ app.stream.push(artifacts, selection='none', remote=remote)
	1122	+
	1123	+
1013	1124	################################################################
1014	1125	# Artifact Log Command #
1015	1126	################################################################
...	...	@@ -1116,3 +1227,37 @@ def fetch(app, elements, deps, track_, except_, track_cross_junctions):
1116	1227	def track(app, elements, deps, except_, cross_junctions):
1117	1228	click.echo("This command is now obsolete. Use `bst source track` instead.", err=True)
1118	1229	sys.exit(1)
	1230	+
	1231	+
	1232	+################################################################
	1233	+# Pull Command #
	1234	+################################################################
	1235	+@cli.command(short_help="Pull a built artifact", hidden=True)
	1236	+@click.option('--deps', '-d', default='none',
	1237	+ type=click.Choice(['none', 'all']),
	1238	+ help='The dependency artifacts to pull (default: none)')
	1239	+@click.option('--remote', '-r',
	1240	+ help="The URL of the remote cache (defaults to the first configured cache)")
	1241	+@click.argument('elements', nargs=-1,
	1242	+ type=click.Path(readable=False))
	1243	+@click.pass_obj
	1244	+def pull(app, elements, deps, remote):
	1245	+ click.echo("This command is now obsolete. Use `bst artifact pull` instead.", err=True)
	1246	+ sys.exit(1)
	1247	+
	1248	+
	1249	+##################################################################
	1250	+# Push Command #
	1251	+##################################################################
	1252	+@cli.command(short_help="Push a built artifact", hidden=True)
	1253	+@click.option('--deps', '-d', default='none',
	1254	+ type=click.Choice(['none', 'all']),
	1255	+ help='The dependencies to push (default: none)')
	1256	+@click.option('--remote', '-r', default=None,
	1257	+ help="The URL of the remote cache (defaults to the first configured cache)")
	1258	+@click.argument('elements', nargs=-1,
	1259	+ type=click.Path(readable=False))
	1260	+@click.pass_obj
	1261	+def push(app, elements, deps, remote):
	1262	+ click.echo("This command is now obsolete. Use `bst artifact push` instead.", err=True)
	1263	+ sys.exit(1)

tests/artifactcache/config.py

---

...	...	@@ -138,5 +138,5 @@ def test_missing_certs(cli, datafiles, config_key, config_value):
138	138	# Use `pull` here to ensure we try to initialize the remotes, triggering the error
139	139	#
140	140	# This does not happen for a simple `bst show`.
141		- result = cli.run(project=project, args=['pull', 'element.bst'])
	141	+ result = cli.run(project=project, args=['artifact', 'pull', 'element.bst'])
142	142	result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)

tests/artifactcache/junctions.py

---

...	...	@@ -58,7 +58,7 @@ def test_push_pull(cli, tmpdir, datafiles):
58	58	project_set_artifacts(base_project, base_share.repo)
59	59
60	60	# Now try bst push
61		- result = cli.run(project=project, args=['push', '--deps', 'all', 'target.bst'])
	61	+ result = cli.run(project=project, args=['artifact', 'push', '--deps', 'all', 'target.bst'])
62	62	assert result.exit_code == 0
63	63
64	64	# And finally assert that the artifacts are in the right shares
...	...	@@ -78,7 +78,7 @@ def test_push_pull(cli, tmpdir, datafiles):
78	78	assert state != 'cached'
79	79
80	80	# Now try bst pull
81		- result = cli.run(project=project, args=['pull', '--deps', 'all', 'target.bst'])
	81	+ result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', 'target.bst'])
82	82	assert result.exit_code == 0
83	83
84	84	# And assert that they are again in the local cache, without having built

tests/completions/completions.py

---

...	...	@@ -11,8 +11,6 @@ MAIN_COMMANDS = [
11	11	'checkout ',
12	12	'help ',
13	13	'init ',
14		- 'pull ',
15		- 'push ',
16	14	'shell ',
17	15	'show ',
18	16	'source ',
...	...	@@ -54,6 +52,12 @@ SOURCE_COMMANDS = [
54	52	'track ',
55	53	]
56	54
	55	+ARTIFACT_COMMANDS = [
	56	+ 'push ',
	57	+ 'pull ',
	58	+ 'log ',
	59	+]
	60	+
57	61	WORKSPACE_COMMANDS = [
58	62	'close ',
59	63	'list ',
...	...	@@ -117,8 +121,7 @@ def assert_completion_failed(cli, cmd, word_idx, expected, cwd=None):
117	121	@pytest.mark.parametrize("cmd,word_idx,expected", [
118	122	('bst', 0, []),
119	123	('bst ', 1, MAIN_COMMANDS),
120		- ('bst pu', 1, ['pull ', 'push ']),
121		- ('bst pul', 1, ['pull ']),
	124	+ ('bst artifact ', 2, ARTIFACT_COMMANDS),
122	125	('bst source ', 2, SOURCE_COMMANDS),
123	126	('bst w ', 1, ['workspace ']),
124	127	('bst workspace ', 2, WORKSPACE_COMMANDS),
...	...	@@ -272,9 +275,8 @@ def test_argument_element_invalid(datafiles, cli, project, cmd, word_idx, expect
272	275	@pytest.mark.parametrize("cmd,word_idx,expected", [
273	276	('bst he', 1, ['help ']),
274	277	('bst help ', 2, MAIN_COMMANDS),
	278	+ ('bst help artifact ', 3, ARTIFACT_COMMANDS),
275	279	('bst help in', 2, ['init ']),
276		- ('bst help p', 2, ['pull ', 'push ']),
277		- ('bst help p', 2, ['pull ', 'push ']),
278	280	('bst help source ', 3, SOURCE_COMMANDS),
279	281	('bst help w', 2, ['workspace ']),
280	282	('bst help workspace ', 3, WORKSPACE_COMMANDS),

tests/frontend/help.py

---

...	...	@@ -18,10 +18,9 @@ def test_help_main(cli):
18	18
19	19
20	20	@pytest.mark.parametrize("command", [
	21	+ ('artifact'),
21	22	('build'),
22	23	('checkout'),
23		- ('pull'),
24		- ('push'),
25	24	('shell'),
26	25	('show'),
27	26	('source'),

tests/frontend/pull.py

---

...	...	@@ -70,7 +70,7 @@ def test_push_pull_all(cli, tmpdir, datafiles):
70	70	assert cli.get_element_state(project, element_name) != 'cached'
71	71
72	72	# Now try bst pull
73		- result = cli.run(project=project, args=['pull', '--deps', 'all', 'target.bst'])
	73	+ result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', 'target.bst'])
74	74	result.assert_success()
75	75
76	76	# And assert that it's again in the local cache, without having built
...	...	@@ -111,7 +111,7 @@ def test_pull_secondary_cache(cli, tmpdir, datafiles):
111	111	assert cli.get_element_state(project, 'target.bst') != 'cached'
112	112
113	113	# Now try bst pull
114		- result = cli.run(project=project, args=['pull', 'target.bst'])
	114	+ result = cli.run(project=project, args=['artifact', 'pull', 'target.bst'])
115	115	result.assert_success()
116	116
117	117	# And assert that it's again in the local cache, without having built,
...	...	@@ -146,7 +146,7 @@ def test_push_pull_specific_remote(cli, tmpdir, datafiles):
146	146
147	147	# Now try `bst push` to the good_share.
148	148	result = cli.run(project=project, args=[
149		- 'push', 'target.bst', '--remote', good_share.repo
	149	+ 'artifact', 'push', 'target.bst', '--remote', good_share.repo
150	150	])
151	151	result.assert_success()
152	152
...	...	@@ -161,7 +161,7 @@ def test_push_pull_specific_remote(cli, tmpdir, datafiles):
161	161	artifacts = os.path.join(cli.directory, 'artifacts')
162	162	shutil.rmtree(artifacts)
163	163
164		- result = cli.run(project=project, args=['pull', 'target.bst', '--remote',
	164	+ result = cli.run(project=project, args=['artifact', 'pull', 'target.bst', '--remote',
165	165	good_share.repo])
166	166	result.assert_success()
167	167
...	...	@@ -216,7 +216,7 @@ def test_push_pull_non_strict(cli, tmpdir, datafiles):
216	216	assert cli.get_element_state(project, 'target.bst') == 'waiting'
217	217
218	218	# Now try bst pull
219		- result = cli.run(project=project, args=['pull', '--deps', 'all', 'target.bst'])
	219	+ result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', 'target.bst'])
220	220	result.assert_success()
221	221
222	222	# And assert that the target is again in the local cache, without having built
...	...	@@ -291,7 +291,7 @@ def test_push_pull_cross_junction(cli, tmpdir, datafiles):
291	291	assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'buildable'
292	292
293	293	# Now try bst pull
294		- result = cli.run(project=project, args=['pull', 'junction.bst:import-etc.bst'])
	294	+ result = cli.run(project=project, args=['artifact', 'pull', 'junction.bst:import-etc.bst'])
295	295	result.assert_success()
296	296
297	297	# And assert that it's again in the local cache, without having built

tests/frontend/push.py

---

...	...	@@ -82,7 +82,7 @@ def test_push(cli, tmpdir, datafiles):
82	82	with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as share2:
83	83
84	84	# Try pushing with no remotes configured. This should fail.
85		- result = cli.run(project=project, args=['push', 'target.bst'])
	85	+ result = cli.run(project=project, args=['artifact', 'push', 'target.bst'])
86	86	result.assert_main_error(ErrorDomain.STREAM, None)
87	87
88	88	# Configure bst to pull but not push from a cache and run `bst push`.
...	...	@@ -90,7 +90,7 @@ def test_push(cli, tmpdir, datafiles):
90	90	cli.configure({
91	91	'artifacts': {'url': share1.repo, 'push': False},
92	92	})
93		- result = cli.run(project=project, args=['push', 'target.bst'])
	93	+ result = cli.run(project=project, args=['artifact', 'push', 'target.bst'])
94	94	result.assert_main_error(ErrorDomain.STREAM, None)
95	95
96	96	# Configure bst to push to one of the caches and run `bst push`. This works.
...	...	@@ -100,7 +100,7 @@ def test_push(cli, tmpdir, datafiles):
100	100	{'url': share2.repo, 'push': True},
101	101	]
102	102	})
103		- result = cli.run(project=project, args=['push', 'target.bst'])
	103	+ result = cli.run(project=project, args=['artifact', 'push', 'target.bst'])
104	104
105	105	assert_not_shared(cli, share1, project, 'target.bst')
106	106	assert_shared(cli, share2, project, 'target.bst')
...	...	@@ -114,7 +114,7 @@ def test_push(cli, tmpdir, datafiles):
114	114	{'url': share2.repo, 'push': True},
115	115	]
116	116	})
117		- result = cli.run(project=project, args=['push', 'target.bst'])
	117	+ result = cli.run(project=project, args=['artifact', 'push', 'target.bst'])
118	118
119	119	assert_shared(cli, share1, project, 'target.bst')
120	120	assert_shared(cli, share2, project, 'target.bst')
...	...	@@ -156,7 +156,7 @@ def test_push_all(cli, tmpdir, datafiles):
156	156
157	157	# Now try bst push all the deps
158	158	result = cli.run(project=project, args=[
159		- 'push', 'target.bst',
	159	+ 'artifact', 'push', 'target.bst',
160	160	'--deps', 'all'
161	161	])
162	162	result.assert_success()
...	...	@@ -346,7 +346,7 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
346	346	assert cli.get_element_state(project, 'element1.bst') != 'cached'
347	347
348	348	# Pull the element1 from the remote cache (this should update its mtime)
349		- result = cli.run(project=project, args=['pull', 'element1.bst', '--remote',
	349	+ result = cli.run(project=project, args=['artifact', 'pull', 'element1.bst', '--remote',
350	350	share.repo])
351	351	result.assert_success()
352	352
...	...	@@ -386,7 +386,7 @@ def test_push_cross_junction(cli, tmpdir, datafiles):
386	386	cli.configure({
387	387	'artifacts': {'url': share.repo, 'push': True},
388	388	})
389		- result = cli.run(project=project, args=['push', 'junction.bst:import-etc.bst'])
	389	+ result = cli.run(project=project, args=['artifact', 'push', 'junction.bst:import-etc.bst'])
390	390
391	391	cache_key = cli.get_element_key(project, 'junction.bst:import-etc.bst')
392	392	assert share.has_artifact('subtest', 'import-etc.bst', cache_key)
...	...	@@ -407,7 +407,7 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles):
407	407	result.assert_success()
408	408	assert "SKIPPED Push" not in result.stderr
409	409
410		- result = cli.run(project=project, args=['push', 'target.bst'])
	410	+ result = cli.run(project=project, args=['artifact', 'push', 'target.bst'])
411	411
412	412	result.assert_success()
413	413	assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated"

tests/frontend/workspace.py

---

...	...	@@ -1105,10 +1105,10 @@ def test_external_push_pull(cli, datafiles, tmpdir_factory, guess_element):
1105	1105	'artifacts': {'url': share.repo, 'push': True}
1106	1106	})
1107	1107
1108		- result = cli.run(project=project, args=['-C', workspace, 'push'] + arg_elm)
	1108	+ result = cli.run(project=project, args=['-C', workspace, 'artifact', 'push'] + arg_elm)
1109	1109	result.assert_success()
1110	1110
1111		- result = cli.run(project=project, args=['-C', workspace, 'pull', '--deps', 'all'] + arg_elm)
	1111	+ result = cli.run(project=project, args=['-C', workspace, 'artifact', 'pull', '--deps', 'all'] + arg_elm)
1112	1112	result.assert_success()
1113	1113
1114	1114

tests/integration/build-tree.py

---

...	...	@@ -130,7 +130,8 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
130	130	assert cli.get_element_state(project, element_name) != 'cached'
131	131
132	132	# Pull from cache, ensuring cli options is set to pull the buildtree
133		- result = cli.run(project=project, args=['--pull-buildtrees', 'pull', '--deps', 'all', element_name])
	133	+ result = cli.run(project=project,
	134	+ args=['--pull-buildtrees', 'artifact', 'pull', '--deps', 'all', element_name])
134	135	result.assert_success()
135	136
136	137	# Check it's using the cached build tree
...	...	@@ -164,7 +165,7 @@ def test_buildtree_options(cli, tmpdir, datafiles):
164	165	assert cli.get_element_state(project, element_name) != 'cached'
165	166
166	167	# Pull from cache, but do not include buildtrees.
167		- result = cli.run(project=project, args=['pull', '--deps', 'all', element_name])
	168	+ result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', element_name])
168	169	result.assert_success()
169	170
170	171	# The above is the simplest way I know to create a local cache without any buildtrees.

tests/integration/pullbuildtrees.py

---

...	...	@@ -55,12 +55,12 @@ def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
55	55	# Pull artifact with default config, assert that pulling again
56	56	# doesn't create a pull job, then assert with buildtrees user
57	57	# config set creates a pull job.
58		- result = cli.run(project=project, args=['pull', element_name])
	58	+ result = cli.run(project=project, args=['artifact', 'pull', element_name])
59	59	assert element_name in result.get_pulled_elements()
60		- result = cli.run(project=project, args=['pull', element_name])
	60	+ result = cli.run(project=project, args=['artifact', 'pull', element_name])
61	61	assert element_name not in result.get_pulled_elements()
62	62	cli.configure({'cache': {'pull-buildtrees': True}})
63		- result = cli.run(project=project, args=['pull', element_name])
	63	+ result = cli.run(project=project, args=['artifact', 'pull', element_name])
64	64	assert element_name in result.get_pulled_elements()
65	65	default_state(cli, tmpdir, share1)
66	66
...	...	@@ -68,13 +68,13 @@ def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
68	68	# with buildtrees cli flag set creates a pull job.
69	69	# Also assert that the buildtree is added to the artifact's
70	70	# extract dir
71		- result = cli.run(project=project, args=['pull', element_name])
	71	+ result = cli.run(project=project, args=['artifact', 'pull', element_name])
72	72	assert element_name in result.get_pulled_elements()
73	73	elementdigest = share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
74	74	buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello',
75	75	elementdigest.hash, 'buildtree')
76	76	assert not os.path.isdir(buildtreedir)
77		- result = cli.run(project=project, args=['--pull-buildtrees', 'pull', element_name])
	77	+ result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
78	78	assert element_name in result.get_pulled_elements()
79	79	assert os.path.isdir(buildtreedir)
80	80	default_state(cli, tmpdir, share1)
...	...	@@ -83,21 +83,21 @@ def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
83	83	# that pulling with the same user config doesn't creates a pull job,
84	84	# or when buildtrees cli flag is set.
85	85	cli.configure({'cache': {'pull-buildtrees': True}})
86		- result = cli.run(project=project, args=['pull', element_name])
	86	+ result = cli.run(project=project, args=['artifact', 'pull', element_name])
87	87	assert element_name in result.get_pulled_elements()
88		- result = cli.run(project=project, args=['pull', element_name])
	88	+ result = cli.run(project=project, args=['artifact', 'pull', element_name])
89	89	assert element_name not in result.get_pulled_elements()
90		- result = cli.run(project=project, args=['--pull-buildtrees', 'pull', element_name])
	90	+ result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
91	91	assert element_name not in result.get_pulled_elements()
92	92	default_state(cli, tmpdir, share1)
93	93
94	94	# Pull artifact with default config and buildtrees cli flag set, then assert
95	95	# that pulling with pullbuildtrees set in user config doesn't create a pull
96	96	# job.
97		- result = cli.run(project=project, args=['--pull-buildtrees', 'pull', element_name])
	97	+ result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
98	98	assert element_name in result.get_pulled_elements()
99	99	cli.configure({'cache': {'pull-buildtrees': True}})
100		- result = cli.run(project=project, args=['pull', element_name])
	100	+ result = cli.run(project=project, args=['artifact', 'pull', element_name])
101	101	assert element_name not in result.get_pulled_elements()
102	102	default_state(cli, tmpdir, share1)
103	103
...	...	@@ -105,10 +105,10 @@ def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
105	105	# can't be pushed to an artifact share, then assert that a complete build element
106	106	# can be. This will attempt a partial pull from share1 and then a partial push
107	107	# to share2
108		- result = cli.run(project=project, args=['pull', element_name])
	108	+ result = cli.run(project=project, args=['artifact', 'pull', element_name])
109	109	assert element_name in result.get_pulled_elements()
110	110	cli.configure({'artifacts': {'url': share2.repo, 'push': True}})
111		- result = cli.run(project=project, args=['push', element_name])
	111	+ result = cli.run(project=project, args=['artifact', 'push', element_name])
112	112	assert element_name not in result.get_pushed_elements()
113	113	assert not share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
114	114
...	...	@@ -116,10 +116,10 @@ def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
116	116	# successfully pushed to the remote. This will attempt to pull the buildtree
117	117	# from share1 and then a 'complete' push to share2
118	118	cli.configure({'artifacts': {'url': share1.repo, 'push': False}})
119		- result = cli.run(project=project, args=['--pull-buildtrees', 'pull', element_name])
	119	+ result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
120	120	assert element_name in result.get_pulled_elements()
121	121	cli.configure({'artifacts': {'url': share2.repo, 'push': True}})
122		- result = cli.run(project=project, args=['push', element_name])
	122	+ result = cli.run(project=project, args=['artifact', 'push', element_name])
123	123	assert element_name in result.get_pushed_elements()
124	124	assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
125	125	default_state(cli, tmpdir, share1)
...	...	@@ -128,10 +128,10 @@ def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
128	128	# if pull-buildtrees is set, however as share3 is the only defined remote and is empty,
129	129	# assert that no element artifact buildtrees are pulled (no available remote buildtree) and thus the
130	130	# artifact cannot be pushed.
131		- result = cli.run(project=project, args=['pull', element_name])
	131	+ result = cli.run(project=project, args=['artifact', 'pull', element_name])
132	132	assert element_name in result.get_pulled_elements()
133	133	cli.configure({'artifacts': {'url': share3.repo, 'push': True}})
134		- result = cli.run(project=project, args=['--pull-buildtrees', 'push', element_name])
	134	+ result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'push', element_name])
135	135	assert "Attempting to fetch missing artifact buildtrees" in result.stderr
136	136	assert element_name not in result.get_pulled_elements()
137	137	assert not os.path.isdir(buildtreedir)
...	...	@@ -143,7 +143,7 @@ def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache):
143	143	# to the empty share3. This gives the ability to attempt push currently partial artifacts to a remote,
144	144	# without exlipictly requiring a bst pull.
145	145	cli.configure({'artifacts': [{'url': share1.repo, 'push': False}, {'url': share3.repo, 'push': True}]})
146		- result = cli.run(project=project, args=['--pull-buildtrees', 'push', element_name])
	146	+ result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'push', element_name])
147	147	assert "Attempting to fetch missing artifact buildtrees" in result.stderr
148	148	assert element_name in result.get_pulled_elements()
149	149	assert os.path.isdir(buildtreedir)

tests/sandboxes/remote-exec-config.py

---

...	...	@@ -42,7 +42,7 @@ def test_old_and_new_configs(cli, datafiles):
42	42	# Use `pull` here to ensure we try to initialize the remotes, triggering the error
43	43	#
44	44	# This does not happen for a simple `bst show`.
45		- result = cli.run(project=project, args=['pull', 'element.bst'])
	45	+ result = cli.run(project=project, args=['artifact', 'pull', 'element.bst'])
46	46	result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one")
47	47
48	48
...	...	@@ -97,5 +97,5 @@ def test_empty_config(cli, datafiles):
97	97	# Use `pull` here to ensure we try to initialize the remotes, triggering the error
98	98	#
99	99	# This does not happen for a simple `bst show`.
100		- result = cli.run(project=project, args=['pull', 'element.bst'])
	100	+ result = cli.run(project=project, args=['artifact', 'pull', 'element.bst'])
101	101	result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA, "specify one")

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.