[Notes] [Git][BuildStream/buildstream][valentindavid/faster_startup-1.2] 3 commits: Don't strip pathname from debug symbol files

[Valentin David <gitlab mg gitlab com>]

Title: GitLab

Valentin David pushed to branch valentindavid/faster_startup-1.2 at BuildStream / buildstream

Commits:

• 0bcfdfb9
  by Sam Thursfield at 2018-08-22T11:36:29Z

  Don't strip pathname from debug symbol files
  
  When GDB looks for debug symbols it looks for the full path of the file
  inside the configured debug-file-directory. For example, if the
  debug-file-directory is set to a default of /usr/lib/debug, and you are
  debugging /usr/bin/python3, GDB will look for its debug symbols at this
  location: /usr/lib/debug/usr/bin/python3.
  
  BuildStream has been putting all debug files inside /usr/lib/debug
  under their $(basename), so in the above example GDB would fail to find
  any debug symbols for /usr/bin/python3 because they would be in the
  incorrect locatoin of /usr/lib/debug/python3.
  

• b567666d
  by Tristan Van Berkom at 2018-08-22T13:04:22Z

  Merge branch 'tristan/debug-symbols-location-1.2' into 'bst-1.2'
  
  Don't strip pathname from debug symbol files
  
  See merge request BuildStream/buildstream!698

• 6a7fdf12
  by Jonathan Maw at 2018-08-22T13:22:27Z

  Use ArtifactCache's get_cache_size when calculating the quota
  
  It would have been more convenient to store the quota size in the
  ArtifactCache, except that the scheduler needs the quota size and
  doesn't have access to the ArtifactCache.
  

11 changed files:

• buildstream/_context.py
• buildstream/_frontend/app.py
• buildstream/data/projectconfig.yaml
• tests/cachekey/project/elements/build1.expected
• tests/cachekey/project/elements/build2.expected
• tests/cachekey/project/target.expected
• tests/examples/autotools.py
• tests/examples/flatpak-autotools.py
• tests/integration/autotools.py
• tests/integration/cmake.py
• tests/integration/compose.py

Changes:

buildstream/_context.py

---

...	...	@@ -124,6 +124,8 @@ class Context():
124	124	self._workspaces = None
125	125	self._log_handle = None
126	126	self._log_filename = None
	127	+ self._config_cache_quota = None
	128	+ self._artifactdir_volume = None
127	129
128	130	# load()
129	131	#
...	...	@@ -187,67 +189,19 @@ class Context():
187	189	while not os.path.exists(artifactdir_volume):
188	190	artifactdir_volume = os.path.dirname(artifactdir_volume)
189	191
	192	+ self._artifactdir_volume = artifactdir_volume
	193	+
190	194	# We read and parse the cache quota as specified by the user
191	195	cache_quota = _yaml.node_get(cache, str, 'quota', default_value='infinity')
192	196	try:
193		- cache_quota = utils._parse_size(cache_quota, artifactdir_volume)
	197	+ cache_quota = utils._parse_size(cache_quota, self._artifactdir_volume)
194	198	except utils.UtilError as e:
195	199	raise LoadError(LoadErrorReason.INVALID_DATA,
196	200	"{}\nPlease specify the value in bytes or as a % of full disk space.\n"
197	201	"\nValid values are, for example: 800M 10G 1T 50%\n"
198	202	.format(str(e))) from e
199	203
200		- # Headroom intended to give BuildStream a bit of leeway.
201		- # This acts as the minimum size of cache_quota and also
202		- # is taken from the user requested cache_quota.
203		- #
204		- if 'BST_TEST_SUITE' in os.environ:
205		- headroom = 0
206		- else:
207		- headroom = 2e9
208		-
209		- stat = os.statvfs(artifactdir_volume)
210		- available_space = (stat.f_bsize * stat.f_bavail)
211		-
212		- # Again, the artifact directory may not yet have been created yet
213		- #
214		- if not os.path.exists(self.artifactdir):
215		- cache_size = 0
216		- else:
217		- cache_size = utils._get_dir_size(self.artifactdir)
218		-
219		- # Ensure system has enough storage for the cache_quota
220		- #
221		- # If cache_quota is none, set it to the maximum it could possibly be.
222		- #
223		- # Also check that cache_quota is atleast as large as our headroom.
224		- #
225		- if cache_quota is None: # Infinity, set to max system storage
226		- cache_quota = cache_size + available_space
227		- if cache_quota < headroom: # Check minimum
228		- raise LoadError(LoadErrorReason.INVALID_DATA,
229		- "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
230		- "BuildStream requires a minimum cache quota of 2G.")
231		- elif cache_quota > cache_size + available_space: # Check maximum
232		- raise LoadError(LoadErrorReason.INVALID_DATA,
233		- ("Your system does not have enough available " +
234		- "space to support the cache quota specified.\n" +
235		- "You currently have:\n" +
236		- "- {used} of cache in use at {local_cache_path}\n" +
237		- "- {available} of available system storage").format(
238		- used=utils._pretty_size(cache_size),
239		- local_cache_path=self.artifactdir,
240		- available=utils._pretty_size(available_space)))
241		-
242		- # Place a slight headroom (2e9 (2GB) on the cache_quota) into
243		- # cache_quota to try and avoid exceptions.
244		- #
245		- # Of course, we might still end up running out during a build
246		- # if we end up writing more than 2G, but hey, this stuff is
247		- # already really fuzzy.
248		- #
249		- self.cache_quota = cache_quota - headroom
250		- self.cache_lower_threshold = self.cache_quota / 2
	204	+ self._config_cache_quota = cache_quota
251	205
252	206	# Load artifact share configuration
253	207	self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
...	...	@@ -571,6 +525,53 @@ class Context():
571	525	def get_log_filename(self):
572	526	return self._log_filename
573	527
	528	+ def set_cache_quota(self, cache_size):
	529	+ # Headroom intended to give BuildStream a bit of leeway.
	530	+ # This acts as the minimum size of cache_quota and also
	531	+ # is taken from the user requested cache_quota.
	532	+ #
	533	+ if 'BST_TEST_SUITE' in os.environ:
	534	+ headroom = 0
	535	+ else:
	536	+ headroom = 2e9
	537	+
	538	+ stat = os.statvfs(self._artifactdir_volume)
	539	+ available_space = (stat.f_bsize * stat.f_bavail)
	540	+
	541	+ # Ensure system has enough storage for the cache_quota
	542	+ #
	543	+ # If cache_quota is none, set it to the maximum it could possibly be.
	544	+ #
	545	+ # Also check that cache_quota is atleast as large as our headroom.
	546	+ #
	547	+ cache_quota = self._config_cache_quota
	548	+ if cache_quota is None: # Infinity, set to max system storage
	549	+ cache_quota = cache_size + available_space
	550	+ if cache_quota < headroom: # Check minimum
	551	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	552	+ "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
	553	+ "BuildStream requires a minimum cache quota of 2G.")
	554	+ elif cache_quota > cache_size + available_space: # Check maximum
	555	+ raise LoadError(LoadErrorReason.INVALID_DATA,
	556	+ ("Your system does not have enough available " +
	557	+ "space to support the cache quota specified.\n" +
	558	+ "You currently have:\n" +
	559	+ "- {used} of cache in use at {local_cache_path}\n" +
	560	+ "- {available} of available system storage").format(
	561	+ used=utils._pretty_size(cache_size),
	562	+ local_cache_path=self.artifactdir,
	563	+ available=utils._pretty_size(available_space)))
	564	+
	565	+ # Place a slight headroom (2e9 (2GB) on the cache_quota) into
	566	+ # cache_quota to try and avoid exceptions.
	567	+ #
	568	+ # Of course, we might still end up running out during a build
	569	+ # if we end up writing more than 2G, but hey, this stuff is
	570	+ # already really fuzzy.
	571	+ #
	572	+ self.cache_quota = cache_quota - headroom
	573	+ self.cache_lower_threshold = self.cache_quota / 2
	574	+
574	575	# _record_message()
575	576	#
576	577	# Records the message if recording is enabled

buildstream/_frontend/app.py

---

...	...	@@ -202,6 +202,10 @@ class App():
202	202
203	203	Platform.create_instance(self.context)
204	204
	205	+ platform = Platform.get_platform()
	206	+ cache_size = platform._artifact_cache.calculate_cache_size()
	207	+ self.context.set_cache_quota(cache_size)
	208	+
205	209	# Create the logger right before setting the message handler
206	210	self.logger = LogLine(self.context,
207	211	self._content_profile,

buildstream/data/projectconfig.yaml

---

...	...	@@ -72,7 +72,7 @@ variables:
72	72	# Generic implementation for stripping debugging symbols
73	73	strip-binaries: |
74	74
75		- find "%{install-root}" -type f \
	75	+ cd "%{install-root}" && find -type f \
76	76	'(' -perm -111 -o -name '*.so*' \
77	77	-o -name '*.cmxs' -o -name '*.node' ')' \
78	78	-exec sh -ec \
...	...	@@ -80,7 +80,7 @@ variables:
80	80	if [ "$hdr" != "$(printf \\x7fELF)" ]; then
81	81	exit 0
82	82	fi
83		- debugfile="%{install-root}%{debugdir}/$(basename "$1")"
	83	+ debugfile="%{install-root}%{debugdir}/$1"
84	84	mkdir -p "$(dirname "$debugfile")"
85	85	objcopy %{objcopy-extract-args} "$1" "$debugfile"
86	86	chmod 644 "$debugfile"

tests/cachekey/project/elements/build1.expected

---

1		-e7de3dd12a1e5307e07859ddf2192443a0ccb1ff48e0adcc6c18f9edc2bd0d7d
		\ No newline at end of file
	1	+afab4c1a67d1e06489083fa1559bda0b2c8df9b7bc239820ed7cdab30c988a4e
		\ No newline at end of file

tests/cachekey/project/elements/build2.expected

---

1		-d74957e0f20a7664e9ceed6cc2ba6c140bd8d8d0712d02066feb442638e8e6ed
		\ No newline at end of file
	1	+47395a4e6c86372b181ad1fd6443e11b1ab54c480b7be5e5fe816d84eec3b369
		\ No newline at end of file

tests/cachekey/project/target.expected

---

1		-01f611e61e948f32035b659d33cdae662d863c99051d0e6746f9c5626138655f
		\ No newline at end of file
	1	+46f48e5c0ff52370ff0cf2bb23bd2c79da23141e6c17b9aa720f7d97b7194340
		\ No newline at end of file

tests/examples/autotools.py

---

...	...	@@ -28,7 +28,9 @@ def test_autotools_build(cli, tmpdir, datafiles):
28	28
29	29	assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
30	30	'/usr/share', '/usr/lib/debug',
31		- '/usr/lib/debug/hello', '/usr/bin/hello',
	31	+ '/usr/lib/debug/usr', '/usr/lib/debug/usr/bin',
	32	+ '/usr/lib/debug/usr/bin/hello',
	33	+ '/usr/bin/hello',
32	34	'/usr/share/doc', '/usr/share/doc/amhello',
33	35	'/usr/share/doc/amhello/README'])
34	36

tests/examples/flatpak-autotools.py

---

...	...	@@ -47,8 +47,10 @@ def test_autotools_build(cli, tmpdir, datafiles):
47	47
48	48	assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
49	49	'/usr/share', '/usr/lib/debug',
50		- '/usr/lib/debug/hello', '/usr/bin/hello',
51		- '/usr/share/doc', '/usr/share/doc/amhello',
	50	+ '/usr/lib/debug/usr', '/usr/lib/debug/usr/bin',
	51	+ '/usr/lib/debug/usr/bin/hello',
	52	+ '/usr/bin/hello', '/usr/share/doc',
	53	+ '/usr/share/doc/amhello',
52	54	'/usr/share/doc/amhello/README'])
53	55
54	56

tests/integration/autotools.py

---

...	...	@@ -31,8 +31,10 @@ def test_autotools_build(cli, tmpdir, datafiles):
31	31
32	32	assert_contains(checkout, ['/usr', '/usr/lib', '/usr/bin',
33	33	'/usr/share', '/usr/lib/debug',
34		- '/usr/lib/debug/hello', '/usr/bin/hello',
35		- '/usr/share/doc', '/usr/share/doc/amhello',
	34	+ '/usr/lib/debug/usr', '/usr/lib/debug/usr/bin',
	35	+ '/usr/lib/debug/usr/bin/hello',
	36	+ '/usr/bin/hello', '/usr/share/doc',
	37	+ '/usr/share/doc/amhello',
36	38	'/usr/share/doc/amhello/README'])
37	39
38	40

tests/integration/cmake.py

---

...	...	@@ -27,7 +27,9 @@ def test_cmake_build(cli, tmpdir, datafiles):
27	27	assert result.exit_code == 0
28	28
29	29	assert_contains(checkout, ['/usr', '/usr/bin', '/usr/bin/hello',
30		- '/usr/lib/debug', '/usr/lib/debug/hello'])
	30	+ '/usr/lib/debug', '/usr/lib/debug/usr',
	31	+ '/usr/lib/debug/usr/bin',
	32	+ '/usr/lib/debug/usr/bin/hello'])
31	33
32	34
33	35	@pytest.mark.datafiles(DATA_DIR)

tests/integration/compose.py

---

...	...	@@ -39,7 +39,8 @@ def create_compose_element(name, path, config={}):
39	39	# Test flat inclusion
40	40	([], [], ['/usr', '/usr/lib', '/usr/bin',
41	41	'/usr/share', '/usr/lib/debug',
42		- '/usr/lib/debug/hello', '/usr/bin/hello',
	42	+ '/usr/lib/debug/usr', '/usr/lib/debug/usr/bin',
	43	+ '/usr/lib/debug/usr/bin/hello', '/usr/bin/hello',
43	44	'/usr/share/doc', '/usr/share/doc/amhello',
44	45	'/usr/share/doc/amhello/README',
45	46	'/tests', '/tests/test']),
...	...	@@ -53,13 +54,17 @@ def create_compose_element(name, path, config={}):
53	54	'/usr/share/doc/amhello/README']),
54	55	# Test with only runtime excluded
55	56	([], ['runtime'], ['/usr', '/usr/lib', '/usr/share',
56		- '/usr/lib/debug', '/usr/lib/debug/hello',
	57	+ '/usr/lib/debug', '/usr/lib/debug/usr',
	58	+ '/usr/lib/debug/usr/bin',
	59	+ '/usr/lib/debug/usr/bin/hello',
57	60	'/usr/share/doc', '/usr/share/doc/amhello',
58	61	'/usr/share/doc/amhello/README',
59	62	'/tests', '/tests/test']),
60	63	# Test with runtime and doc excluded
61	64	([], ['runtime', 'doc'], ['/usr', '/usr/lib', '/usr/share',
62		- '/usr/lib/debug', '/usr/lib/debug/hello',
	65	+ '/usr/lib/debug', '/usr/lib/debug/usr',
	66	+ '/usr/lib/debug/usr/bin',
	67	+ '/usr/lib/debug/usr/bin/hello',
63	68	'/tests', '/tests/test']),
64	69	# Test with runtime simultaneously in- and excluded
65	70	(['runtime'], ['runtime'], ['/usr', '/usr/lib', '/usr/share']),
...	...	@@ -72,7 +77,8 @@ def create_compose_element(name, path, config={}):
72	77	# Test excluding a custom 'test' domain
73	78	([], ['test'], ['/usr', '/usr/lib', '/usr/bin',
74	79	'/usr/share', '/usr/lib/debug',
75		- '/usr/lib/debug/hello', '/usr/bin/hello',
	80	+ '/usr/lib/debug/usr', '/usr/lib/debug/usr/bin',
	81	+ '/usr/lib/debug/usr/bin/hello', '/usr/bin/hello',
76	82	'/usr/share/doc', '/usr/share/doc/amhello',
77	83	'/usr/share/doc/amhello/README'])
78	84	])

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.