[Notes] [Git][BuildStream/buildstream][danielsilverstone-ct/roaring-bitmaps] 4 commits: Add --remote, -r option to bst build, inline with pull & push

[Daniel Silverstone <gitlab mg gitlab com>]

Title: GitLab

Daniel Silverstone pushed to branch danielsilverstone-ct/roaring-bitmaps at BuildStream / buildstream

Commits:

• cd8e5e27
  by Tom Pollard at 2019-02-05T11:22:01Z

  Add --remote, -r option to bst build, inline with pull & push
  
  Providing a remote will limit build's pull/push remote actions to
  the given remote specifically, ignoring those defined via user or
  project configuration.
  

• a46650d0
  by Tom Pollard at 2019-02-05T12:41:30Z

  Merge branch 'tpollard/buildremote' into 'master'
  
  Add --remote, -r option to bst build, inline with pull & push
  
  See merge request BuildStream/buildstream!1119

• 9c94e8e5
  by Daniel Silverstone at 2019-02-05T13:15:24Z

  requirements: Add roaringbitmap to the requirements
  
  In order to support use of roaring bitmaps in the loader, we need to
  depend on it here.
  
  Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>
  

• 7682ef49
  by Daniel Silverstone at 2019-02-05T13:15:24Z

  loadelement.py: Use roaring bitmaps in dep cache
  
  The dependency cache in LoadElement causes the peak RAM consumption of
  the loader to be exceedingly large, upwards of 25GB for a test of a
  Debian stack.  By switching from the old dict cache to a roaring bitmap
  based cache, we reduce the cost of the loader cache in RAM terms to
  around 5.5GB for the same stack, which makes it plausible for the short
  term.
  
  Signed-off-by: Daniel Silverstone <daniel silverstone codethink co uk>
  

9 changed files:

• NEWS
• buildstream/_frontend/cli.py
• buildstream/_loader/loadelement.py
• buildstream/_stream.py
• requirements/requirements.in
• requirements/requirements.txt
• tests/frontend/completions.py
• tests/frontend/pull.py
• tests/frontend/push.py

Changes:

NEWS

---

...	...	@@ -122,6 +122,10 @@ buildstream 1.3.1
122	122	'shell', 'show', 'source-checkout', 'track', 'workspace close' and 'workspace reset'
123	123	commands are affected.
124	124
	125	+ o bst 'build' now has '--remote, -r' option, inline with bst 'push' & 'pull'.
	126	+ Providing a remote will limit build's pull/push remote actions to the given
	127	+ remote specifically, ignoring those defined via user or project configuration.
	128	+
125	129
126	130	=================
127	131	buildstream 1.1.5

buildstream/_frontend/cli.py

---

...	...	@@ -338,10 +338,12 @@ def init(app, project_name, format_version, element_path, force):
338	338	help="Allow tracking to cross junction boundaries")
339	339	@click.option('--track-save', default=False, is_flag=True,
340	340	help="Deprecated: This is ignored")
	341	+@click.option('--remote', '-r', default=None,
	342	+ help="The URL of the remote cache (defaults to the first configured cache)")
341	343	@click.argument('elements', nargs=-1,
342	344	type=click.Path(readable=False))
343	345	@click.pass_obj
344		-def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions):
	346	+def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions, remote):
345	347	"""Build elements in a pipeline
346	348
347	349	Specifying no elements will result in building the default targets
...	...	@@ -376,7 +378,8 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
376	378	track_except=track_except,
377	379	track_cross_junctions=track_cross_junctions,
378	380	ignore_junction_targets=ignore_junction_targets,
379		- build_all=all_)
	381	+ build_all=all_,
	382	+ remote=remote)
380	383
381	384
382	385	##################################################################
...	...	@@ -1012,7 +1015,7 @@ def artifact_checkout(app, force, deps, integrate, hardlinks, tar, directory, el
1012	1015	@click.option('--deps', '-d', default='none',
1013	1016	type=click.Choice(['none', 'all']),
1014	1017	help='The dependency artifacts to pull (default: none)')
1015		-@click.option('--remote', '-r',
	1018	+@click.option('--remote', '-r', default=None,
1016	1019	help="The URL of the remote cache (defaults to the first configured cache)")
1017	1020	@click.argument('elements', nargs=-1,
1018	1021	type=click.Path(readable=False))

buildstream/_loader/loadelement.py

---

...	...	@@ -19,6 +19,9 @@
19	19
20	20	# System imports
21	21	from collections.abc import Mapping
	22	+from itertools import count
	23	+
	24	+from roaringbitmap import RoaringBitmap, ImmutableRoaringBitmap # pylint: disable=no-name-in-module
22	25
23	26	# BuildStream toplevel imports
24	27	from .._exceptions import LoadError, LoadErrorReason
...	...	@@ -54,6 +57,8 @@ class LoadElement():
54	57	self.element = element
55	58	self.dep_type = dep_type
56	59
	60	+ _counter = count()
	61	+
57	62	def __init__(self, node, filename, loader):
58	63
59	64	#
...	...	@@ -63,6 +68,7 @@ class LoadElement():
63	68	self.name = filename # The element name
64	69	self.full_name = None # The element full name (with associated junction)
65	70	self.deps = None # The list of Dependency objects
	71	+ self.node_id = next(self._counter)
66	72
67	73	#
68	74	# Private members
...	...	@@ -107,7 +113,7 @@ class LoadElement():
107	113	#
108	114	def depends(self, other):
109	115	self._ensure_depends_cache()
110		- return self._dep_cache.get(other.full_name) is not None
	116	+ return other.node_id in self._dep_cache
111	117
112	118	###########################################
113	119	# Private Methods #
...	...	@@ -117,7 +123,8 @@ class LoadElement():
117	123	if self._dep_cache:
118	124	return
119	125
120		- self._dep_cache = {}
	126	+ self._dep_cache = RoaringBitmap()
	127	+
121	128	for dep in self.dependencies:
122	129	elt = dep.element
123	130
...	...	@@ -125,11 +132,13 @@ class LoadElement():
125	132	elt._ensure_depends_cache()
126	133
127	134	# We depend on this element
128		- self._dep_cache[elt.full_name] = True
	135	+ self._dep_cache.add(elt.node_id)
129	136
130	137	# And we depend on everything this element depends on
131	138	self._dep_cache.update(elt._dep_cache)
132	139
	140	+ self._dep_cache = ImmutableRoaringBitmap(self._dep_cache)
	141	+
133	142
134	143	# _extract_depends_from_node():
135	144	#

buildstream/_stream.py

---

...	...	@@ -197,26 +197,36 @@ class Stream():
197	197	# ignore_junction_targets (bool): Whether junction targets should be filtered out
198	198	# build_all (bool): Whether to build all elements, or only those
199	199	# which are required to build the target.
	200	+ # remote (str): The URL of a specific remote server to push to, or None
	201	+ #
	202	+ # If `remote` specified as None, then regular configuration will be used
	203	+ # to determine where to push artifacts to.
200	204	#
201	205	def build(self, targets, *,
202	206	track_targets=None,
203	207	track_except=None,
204	208	track_cross_junctions=False,
205	209	ignore_junction_targets=False,
206		- build_all=False):
	210	+ build_all=False,
	211	+ remote=None):
207	212
208	213	if build_all:
209	214	selection = PipelineSelection.ALL
210	215	else:
211	216	selection = PipelineSelection.PLAN
212	217
	218	+ use_config = True
	219	+ if remote:
	220	+ use_config = False
	221	+
213	222	elements, track_elements = \
214	223	self._load(targets, track_targets,
215	224	selection=selection, track_selection=PipelineSelection.ALL,
216	225	track_except_targets=track_except,
217	226	track_cross_junctions=track_cross_junctions,
218	227	ignore_junction_targets=ignore_junction_targets,
219		- use_artifact_config=True,
	228	+ use_artifact_config=use_config,
	229	+ artifact_remote_url=remote,
220	230	fetch_subprojects=True,
221	231	dynamic_plan=True)
222	232

requirements/requirements.in

---

...	...	@@ -13,3 +13,6 @@ psutil
13	13	# See issues #571 and #790.
14	14	ruamel.yaml >= 0.15.41, < 0.15.52
15	15	setuptools
	16	+# (Potentially) short-term need for roaring bitmaps for the
	17	+# loader dependency sorting
	18	+roaringbitmap

requirements/requirements.txt

---

...	...	@@ -13,6 +13,9 @@ psutil==5.4.8
13	13	# See issues #571 and #790.
14	14	ruamel.yaml==0.15.51
15	15	setuptools==39.0.1
	16	+# (Potentially) short-term need for roaring bitmaps for the
	17	+# loader dependency sorting
	18	+roaringbitmap==0.6
16	19	## The following requirements were added by pip freeze:
17	20	MarkupSafe==1.1.0
18	21	six==1.12.0

tests/frontend/completions.py

---

...	...	@@ -141,7 +141,8 @@ def test_commands(cli, cmd, word_idx, expected):
141	141	('bst --no-colors build -', 3, ['--all ', '--track ', '--track-all ',
142	142	'--track-except ',
143	143	'--track-cross-junctions ', '-J ',
144		- '--track-save ']),
	144	+ '--track-save ',
	145	+ '--remote ', '-r ']),
145	146
146	147	# Test the behavior of completing after an option that has a
147	148	# parameter that cannot be completed, vs an option that has

tests/frontend/pull.py

---

...	...	@@ -408,3 +408,56 @@ def test_pull_missing_notifies_user(caplog, cli, tmpdir, datafiles):
408	408
409	409	assert "INFO Remote ({}) does not have".format(share.repo) in result.stderr
410	410	assert "SKIPPED Pull" in result.stderr
	411	+
	412	+
	413	+@pytest.mark.datafiles(DATA_DIR)
	414	+def test_build_remote_option(caplog, cli, tmpdir, datafiles):
	415	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	416	+ caplog.set_level(1)
	417	+
	418	+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
	419	+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject,\
	420	+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare3')) as sharecli:
	421	+
	422	+ # Add shareproject repo url to project.conf
	423	+ with open(os.path.join(project, "project.conf"), "a") as projconf:
	424	+ projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo))
	425	+
	426	+ # Configure shareuser remote in user conf
	427	+ cli.configure({
	428	+ 'artifacts': {'url': shareuser.repo, 'push': True}
	429	+ })
	430	+
	431	+ # Push the artifacts to the shareuser and shareproject remotes.
	432	+ # Assert that shareuser and shareproject have the artfifacts cached,
	433	+ # but sharecli doesn't, then delete locally cached elements
	434	+ result = cli.run(project=project, args=['build', 'target.bst'])
	435	+ result.assert_success()
	436	+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
	437	+ for element_name in all_elements:
	438	+ assert element_name in result.get_pushed_elements()
	439	+ assert_not_shared(cli, sharecli, project, element_name)
	440	+ assert_shared(cli, shareuser, project, element_name)
	441	+ assert_shared(cli, shareproject, project, element_name)
	442	+ cli.remove_artifact_from_cache(project, element_name)
	443	+
	444	+ # Now check that a build with cli set as sharecli results in nothing being pulled,
	445	+ # as it doesn't have them cached and shareuser/shareproject should be ignored. This
	446	+ # will however result in the artifacts being built and pushed to it
	447	+ result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst'])
	448	+ result.assert_success()
	449	+ for element_name in all_elements:
	450	+ assert element_name not in result.get_pulled_elements()
	451	+ assert_shared(cli, sharecli, project, element_name)
	452	+ cli.remove_artifact_from_cache(project, element_name)
	453	+
	454	+ # Now check that a clean build with cli set as sharecli should result in artifacts only
	455	+ # being pulled from it, as that was provided via the cli and is populated
	456	+ result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst'])
	457	+ result.assert_success()
	458	+ for element_name in all_elements:
	459	+ assert cli.get_element_state(project, element_name) == 'cached'
	460	+ assert element_name in result.get_pulled_elements()
	461	+ assert shareproject.repo not in result.stderr
	462	+ assert shareuser.repo not in result.stderr
	463	+ assert sharecli.repo in result.stderr

tests/frontend/push.py

---

...	...	@@ -416,3 +416,33 @@ def test_push_already_cached(caplog, cli, tmpdir, datafiles):
416	416	assert not result.get_pushed_elements(), "No elements should have been pushed since the cache was populated"
417	417	assert "INFO Remote ({}) already has ".format(share.repo) in result.stderr
418	418	assert "SKIPPED Push" in result.stderr
	419	+
	420	+
	421	+@pytest.mark.datafiles(DATA_DIR)
	422	+def test_build_remote_option(caplog, cli, tmpdir, datafiles):
	423	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	424	+ caplog.set_level(1)
	425	+
	426	+ with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare1')) as shareuser,\
	427	+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare2')) as shareproject,\
	428	+ create_artifact_share(os.path.join(str(tmpdir), 'artifactshare3')) as sharecli:
	429	+
	430	+ # Add shareproject repo url to project.conf
	431	+ with open(os.path.join(project, "project.conf"), "a") as projconf:
	432	+ projconf.write("artifacts:\n url: {}\n push: True".format(shareproject.repo))
	433	+
	434	+ # Configure shareuser remote in user conf
	435	+ cli.configure({
	436	+ 'artifacts': {'url': shareuser.repo, 'push': True}
	437	+ })
	438	+
	439	+ result = cli.run(project=project, args=['build', '--remote', sharecli.repo, 'target.bst'])
	440	+
	441	+ # Artifacts should have only been pushed to sharecli, as that was provided via the cli
	442	+ result.assert_success()
	443	+ all_elements = ['target.bst', 'import-bin.bst', 'compose-all.bst']
	444	+ for element_name in all_elements:
	445	+ assert element_name in result.get_pushed_elements()
	446	+ assert_shared(cli, sharecli, project, element_name)
	447	+ assert_not_shared(cli, shareuser, project, element_name)
	448	+ assert_not_shared(cli, shareproject, project, element_name)

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.