[Notes] [Git][BuildStream/buildstream][tpollard/752] Add ability to build without interacting with available artifact

[Tom Pollard <gitlab mg gitlab com>]

Title: GitLab

Tom Pollard pushed to branch tpollard/752 at BuildStream / buildstream

Commits:

• 847f75cf
  by Tom Pollard at 2018-11-15T11:03:56Z

  Add ability to build without interacting with available artifact
  servers.
  
  By adding a new user context and bst main option, we can provide
  more granular control of what remotes are utilised during relevant
  bst pull and push actions.
  
  _artifactcache/artifactcache.py: Expand setup_remotes() to consider
  the user context value of 'use_remotes'. Allowing the user to omit
  interacting with all, or all non user defined remotes.
  
  tests/frontend: pull.py & push.py addition of tests for checking
  expected behaviour. Ensures interaction with user, project or no
  remotes.
  

4 changed files:

• NEWS
• buildstream/_artifactcache/artifactcache.py
• tests/frontend/pull.py
• tests/frontend/push.py

Changes:

NEWS

---

...	...	@@ -45,6 +45,15 @@ buildstream 1.3.1
45	45	instead of just a specially-formatted build-root with a `root` and `scratch`
46	46	subdirectory.
47	47
	48	+ o bst interaction with defined artifact servers can be controlled more granularly.
	49	+ This can be done via the user configuration option `useremotes` or via the bst cli
	50	+ main option '--use-remotes'. This can be set as 'none', 'user' or the default value
	51	+ 'all'. Unless specifically overriden, when considering wether to pull or push to
	52	+ available artifact servers (be it user or project defined) this optional config option
	53	+ will be used. Setting this value to 'user' for example and performing a build would
	54	+ lead to any project or junction defined artifact server to be ignored, whilst still
	55	+ attempting to any user defined remotes.
	56	+
48	57
49	58	=================
50	59	buildstream 1.1.5

buildstream/_artifactcache/artifactcache.py

---

...	...	@@ -156,7 +156,7 @@ class ArtifactCache():
156	156	# Sets up which remotes to use
157	157	#
158	158	# Args:
159		- # use_config (bool): Whether to use project configuration
	159	+ # use_config (bool): Whether to use configuration
160	160	# remote_url (str): Remote artifact cache URL
161	161	#
162	162	# This requires that all of the projects which are to be processed in the session
...	...	@@ -175,11 +175,16 @@ class ArtifactCache():
175	175	self._set_remotes([ArtifactCacheSpec(remote_url, push=True)])
176	176	has_remote_caches = True
177	177	if use_config:
178		- for project in self.context.get_projects():
179		- artifact_caches = _configured_remote_artifact_cache_specs(self.context, project)
180		- if artifact_caches: # artifact_caches is a list of ArtifactCacheSpec instances
181		- self._set_remotes(artifact_caches, project=project)
182		- has_remote_caches = True
	178	+ if self.context.use_remotes == 'all':
	179	+ for project in self.context.get_projects():
	180	+ artifact_caches = _configured_remote_artifact_cache_specs(self.context, project)
	181	+ if artifact_caches: # artifact_caches is a list of ArtifactCacheSpec instances
	182	+ self._set_remotes(artifact_caches, project=project)
	183	+ has_remote_caches = True
	184	+ # If configured to only use user configured remotes, pass existing user cache spec
	185	+ elif self.context.use_remotes == 'user' and self.context.artifact_cache_specs:
	186	+ self._set_remotes(self.context.artifact_cache_specs)
	187	+ has_remote_caches = True
183	188	if has_remote_caches:
184	189	self._initialize_remotes()
185	190

tests/frontend/pull.py

---

...	...	@@ -358,3 +358,75 @@ def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles):
358	358
359	359	assert "INFO Remote ({}) does not have".format(share.repo) in result.stderr
360	360	assert "SKIPPED Pull" in result.stderr
	361	+
	362	+
	363	+# Tests that:
	364	+#
	365	+# * The bst main option --use-remotes limits remote action
	366	+# as expected for pull jobs
	367	+#
	368	+@pytest.mark.datafiles(DATA_DIR)
	369	+def test_useremotes_cli_options(cli, tmpdir, datafiles):
	370	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	371	+
	372	+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
	373	+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject:
	374	+
	375	+ # Add shareproject repo url to project.conf
	376	+ with open(os.path.join(project, "project.conf"), "a") as projconf:
	377	+ projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo))
	378	+
	379	+ # First build the target element and push to the remotes.
	380	+ # We need the artifact available in the remotes to test against.
	381	+ cli.configure({
	382	+ 'artifacts': {'url': shareuser.repo, 'push': True}
	383	+ })
	384	+ result = cli.run(project=project, args=['build', 'target.bst'])
	385	+ result.assert_success()
	386	+ assert cli.get_element_state(project, 'target.bst') == 'cached'
	387	+
	388	+ # Assert that everything is now cached in the remotes.
	389	+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
	390	+ for element_name in all_elements:
	391	+ assert_shared(cli, shareuser, project, element_name)
	392	+ assert_shared(cli, shareproject, project, element_name)
	393	+
	394	+ # Now we've pushed, delete the user's local artifact cache
	395	+ artifacts = os.path.join(cli.directory, 'artifacts')
	396	+ shutil.rmtree(artifacts)
	397	+
	398	+ # Assert that nothing is cached locally anymore
	399	+ for element_name in all_elements:
	400	+ assert cli.get_element_state(project, element_name) != 'cached'
	401	+
	402	+ # Attempt bst build with --use-remotes set as none, this should lead to
	403	+ # a complete rebuild without pulling from either artifact remote cache
	404	+ result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst'])
	405	+ result.assert_success()
	406	+ for element_name in all_elements:
	407	+ assert element_name not in result.get_pulled_elements()
	408	+
	409	+ # Delete local cache again
	410	+ artifacts = os.path.join(cli.directory, 'artifacts')
	411	+ shutil.rmtree(artifacts)
	412	+
	413	+ # Attempt bst build with --use-remotes set as user, as the shareuser is
	414	+ # passed in as user config and not via a project, assert project remote
	415	+ # was not attempted by it not being in the output
	416	+ result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst'])
	417	+ result.assert_success()
	418	+ for element_name in all_elements:
	419	+ assert element_name in result.get_pulled_elements()
	420	+ assert shareproject.repo not in result.stderr
	421	+
	422	+ # Delete local cache again
	423	+ artifacts = os.path.join(cli.directory, 'artifacts')
	424	+ shutil.rmtree(artifacts)
	425	+
	426	+ # Attempt bst build with --use-remotes set as all, this time
	427	+ # assert that project remote is attempted and in the output
	428	+ result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst'])
	429	+ result.assert_success()
	430	+ for element_name in all_elements:
	431	+ assert element_name in result.get_pulled_elements()
	432	+ assert shareproject.repo in result.stderr

tests/frontend/push.py

---

...	...	@@ -409,3 +409,68 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles):
409	409	assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated"
410	410	assert "INFO Remote ({}) already has ".format(share.repo) in result.stderr
411	411	assert "SKIPPED Push" in result.stderr
	412	+
	413	+
	414	+# Tests that:
	415	+#
	416	+# * The bst main option --use-remotes limits remote action
	417	+# as expected for push jobs
	418	+#
	419	+@pytest.mark.datafiles(DATA_DIR)
	420	+def test_useremotes_cli_options(cli, tmpdir, datafiles):
	421	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	422	+
	423	+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
	424	+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject:
	425	+
	426	+ # Add shareproject repo url to project.conf
	427	+ with open(os.path.join(project, "project.conf"), "a") as projconf:
	428	+ projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo))
	429	+
	430	+ # Configure shareuser remote in user conf
	431	+ cli.configure({
	432	+ 'artifacts': {'url': shareuser.repo, 'push': True}
	433	+ })
	434	+
	435	+ # First build the target element with --use-remotes set as none.
	436	+ # This should lead to a complete build without pushing to either artifact
	437	+ # remote cache
	438	+ result = cli.run(project=project, args=['--use-remotes', 'none', 'build', 'target.bst'])
	439	+ result.assert_success()
	440	+ assert not result.get_pushed_elements()
	441	+ assert cli.get_element_state(project, 'target.bst') == 'cached'
	442	+
	443	+ # Delete the artifacts from the local artifact cache
	444	+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
	445	+ for element_name in all_elements:
	446	+ cli.remove_artifact_from_cache(project, element_name)
	447	+
	448	+ # Assert that nothing is cached locally anymore
	449	+ for element_name in all_elements:
	450	+ assert cli.get_element_state(project, element_name) != 'cached'
	451	+
	452	+ # Attempt bst build with --use-remotes set as user, this should lead to
	453	+ # a complete rebuild, with artifacts pushed to the shareuser remote artifact cache
	454	+ # only. Assert project remote was not attempted by it not being in the output
	455	+ result = cli.run(project=project, args=['--use-remotes', 'user', 'build', 'target.bst'])
	456	+ result.assert_success()
	457	+ for element_name in all_elements:
	458	+ assert element_name in result.get_pushed_elements()
	459	+ for element_name in all_elements:
	460	+ assert_shared(cli, shareuser, project, element_name)
	461	+ assert shareproject.repo not in result.stderr
	462	+
	463	+ # Delete the artifacts from the local artifact cache
	464	+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
	465	+ for element_name in all_elements:
	466	+ cli.remove_artifact_from_cache(project, element_name)
	467	+
	468	+ # Attempt bst build with --use-remotes set as all, this should lead to
	469	+ # a complete rebuild, with artifacts pushed to both the shareuser and
	470	+ # shareproject remote artifacts caches
	471	+ result = cli.run(project=project, args=['--use-remotes', 'all', 'build', 'target.bst'])
	472	+ result.assert_success()
	473	+ for element_name in all_elements:
	474	+ assert element_name in result.get_pushed_elements()
	475	+ for element_name in all_elements:
	476	+ assert_shared(cli, shareproject, project, element_name)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.