[Notes] [Git][BuildStream/buildstream][tpollard/buildremote] Add --remote, -r option to bst build, inline with pull & push

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch tpollard/buildremote at BuildStream / buildstream

Commits:

• 158ad0aa
  by Tom Pollard at 2019-01-30T17:15:44Z

  Add --remote, -r option to bst build, inline with pull & push
  
  Providing a remote will limit build's pull/push remote actions to
  the given remote specifically, ignoring those defined via user or
  project configuration.
  

6 changed files:

• NEWS
• buildstream/_frontend/cli.py
• buildstream/_stream.py
• tests/frontend/completions.py
• tests/frontend/pull.py
• tests/frontend/push.py

Changes:

NEWS

---

...	...	@@ -122,6 +122,10 @@ buildstream 1.3.1
122	122	'shell', 'show', 'source-checkout', 'track', 'workspace close' and 'workspace reset'
123	123	commands are affected.
124	124
	125	+ o bst 'build' now has '--remote, -r' option, inline with bst 'push' & 'build'.
	126	+ Providing a remote will limit build's pull/push remote actions to the given
	127	+ remote specifically, ignoring those defined via user or project configuration.
	128	+
125	129
126	130	=================
127	131	buildstream 1.1.5

buildstream/_frontend/cli.py

---

...	...	@@ -338,10 +338,12 @@ def init(app, project_name, format_version, element_path, force):
338	338	help="Allow tracking to cross junction boundaries")
339	339	@click.option('--track-save', default=False, is_flag=True,
340	340	help="Deprecated: This is ignored")
	341	+@click.option('--remote', '-r', default=None,
	342	+ help="The URL of the remote cache (defaults to the first configured cache)")
341	343	@click.argument('elements', nargs=-1,
342	344	type=click.Path(readable=False))
343	345	@click.pass_obj
344		-def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions):
	346	+def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions, remote):
345	347	"""Build elements in a pipeline
346	348
347	349	Specifying no elements will result in building the default targets
...	...	@@ -376,7 +378,8 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
376	378	track_except=track_except,
377	379	track_cross_junctions=track_cross_junctions,
378	380	ignore_junction_targets=ignore_junction_targets,
379		- build_all=all_)
	381	+ build_all=all_,
	382	+ remote=remote)
380	383
381	384
382	385	##################################################################

buildstream/_stream.py

---

...	...	@@ -197,26 +197,36 @@ class Stream():
197	197	# ignore_junction_targets (bool): Whether junction targets should be filtered out
198	198	# build_all (bool): Whether to build all elements, or only those
199	199	# which are required to build the target.
	200	+ # remote (str): The URL of a specific remote server to push to, or None
	201	+ #
	202	+ # If `remote` specified as None, then regular configuration will be used
	203	+ # to determine where to push artifacts to.
200	204	#
201	205	def build(self, targets, *,
202	206	track_targets=None,
203	207	track_except=None,
204	208	track_cross_junctions=False,
205	209	ignore_junction_targets=False,
206		- build_all=False):
	210	+ build_all=False,
	211	+ remote=None):
207	212
208	213	if build_all:
209	214	selection = PipelineSelection.ALL
210	215	else:
211	216	selection = PipelineSelection.PLAN
212	217
	218	+ use_config = True
	219	+ if remote:
	220	+ use_config = False
	221	+
213	222	elements, track_elements = \
214	223	self._load(targets, track_targets,
215	224	selection=selection, track_selection=PipelineSelection.ALL,
216	225	track_except_targets=track_except,
217	226	track_cross_junctions=track_cross_junctions,
218	227	ignore_junction_targets=ignore_junction_targets,
219		- use_artifact_config=True,
	228	+ use_artifact_config=use_config,
	229	+ artifact_remote_url=remote,
220	230	fetch_subprojects=True,
221	231	dynamic_plan=True)
222	232

tests/frontend/completions.py

---

...	...	@@ -141,7 +141,8 @@ def test_commands(cli, cmd, word_idx, expected):
141	141	('bst --no-colors build -', 3, ['--all ', '--track ', '--track-all ',
142	142	'--track-except ',
143	143	'--track-cross-junctions ', '-J ',
144		- '--track-save ']),
	144	+ '--track-save ',
	145	+ '--remote ', '-r ']),
145	146
146	147	# Test the behavior of completing after an option that has a
147	148	# parameter that cannot be completed, vs an option that has

tests/frontend/pull.py

---

...	...	@@ -408,3 +408,61 @@ def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles):
408	408
409	409	assert "INFO Remote ({}) does not have".format(share.repo) in result.stderr
410	410	assert "SKIPPED Pull" in result.stderr
	411	+
	412	+
	413	+@pytest.mark.datafiles(DATA_DIR)
	414	+def test_build_remote_option(caplog, cli, tmpdir, datafiles):
	415	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	416	+ caplog.set_level(1)
	417	+
	418	+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
	419	+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject,\
	420	+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare3')) as sharecli:
	421	+
	422	+ # Add shareproject repo url to project.conf
	423	+ with open(os.path.join(project, "project.conf"), "a") as projconf:
	424	+ projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo))
	425	+
	426	+ # Configure shareuser remote in user conf
	427	+ cli.configure({
	428	+ 'artifacts': {'url': shareuser.repo, 'push': True}
	429	+ })
	430	+
	431	+ # Push the artifacts to the shareuser and shareproject remotes
	432	+
	433	+ result = cli.run(project=project, args=['build', 'target.bst'])
	434	+
	435	+ # Assert that shareuser and shareproject have the artfifacts cached,
	436	+ # but sharecli doesn't, then delete local cache
	437	+ result.assert_success()
	438	+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
	439	+ for element_name in all_elements:
	440	+ assert element_name in result.get_pushed_elements()
	441	+ assert_not_shared(cli, sharecli, project, element_name)
	442	+ assert_shared(cli, shareuser, project, element_name)
	443	+ assert_shared(cli, shareproject, project, element_name)
	444	+ cli.remove_artifact_from_cache(project, element_name)
	445	+
	446	+ # Now check that a build with cli set as sharecli results in nothing being pulled,
	447	+ # as it doesn't have them cached and shareuser/shareproject should be ignored. This
	448	+ # wil however result in the artifacts being built and pushed to it
	449	+
	450	+ result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst'])
	451	+
	452	+ result.assert_success()
	453	+ for element_name in all_elements:
	454	+ assert element_name not in result.get_pulled_elements()
	455	+ assert_shared(cli, sharecli, project, element_name)
	456	+ cli.remove_artifact_from_cache(project, element_name)
	457	+
	458	+ # Now check that a clean build with cli set as sharecli should result in artifacts only
	459	+ # being pulled from it, as that was provided via the cli and is populated
	460	+
	461	+ result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst'])
	462	+
	463	+ result.assert_success()
	464	+ for element_name in all_elements:
	465	+ assert element_name in result.get_pulled_elements()
	466	+ assert shareproject.repo not in result.stderr
	467	+ assert shareuser.repo not in result.stderr
	468	+ assert sharecli.repo in result.stderr

tests/frontend/push.py

---

...	...	@@ -416,3 +416,33 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles):
416	416	assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated"
417	417	assert "INFO Remote ({}) already has ".format(share.repo) in result.stderr
418	418	assert "SKIPPED Push" in result.stderr
	419	+
	420	+
	421	+@pytest.mark.datafiles(DATA_DIR)
	422	+def test_build_remote_option(caplog, cli, tmpdir, datafiles):
	423	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	424	+ caplog.set_level(1)
	425	+
	426	+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
	427	+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject,\
	428	+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare3')) as sharecli:
	429	+
	430	+ # Add shareproject repo url to project.conf
	431	+ with open(os.path.join(project, "project.conf"), "a") as projconf:
	432	+ projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo))
	433	+
	434	+ # Configure shareuser remote in user conf
	435	+ cli.configure({
	436	+ 'artifacts': {'url': shareuser.repo, 'push': True}
	437	+ })
	438	+
	439	+ result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst'])
	440	+
	441	+ # Artifacts should have only been pushed to sharecli, as that was provided via the cli
	442	+ result.assert_success()
	443	+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
	444	+ for element_name in all_elements:
	445	+ assert element_name in result.get_pushed_elements()
	446	+ assert_shared(cli, sharecli, project, element_name)
	447	+ assert_not_shared(cli, shareuser, project, element_name)
	448	+ assert_not_shared(cli, shareproject, project, element_name)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.