[Notes] [Git][BuildStream/buildstream][willsalmon/log_formating] 8 commits: tox.ini: Specify minimum version of click-man

[Will Salmon <gitlab mg gitlab com>]

Title: GitLab

Will Salmon pushed to branch willsalmon/log_formating at BuildStream / buildstream

Commits:

• 36746730
  by Chandan Singh at 2019-01-31T10:50:05Z

  tox.ini: Specify minimum version of click-man
  
  `click-man` versions < 0.3.0 do not properly support multiple
  entrypoints. Since this was added to `tox` after `0.3.0` was released,
  `tox` should never be pulling older versions. But, let's add it here for
  documentation purposes.
  
  See
  https://gitlab.com/BuildStream/buildstream/merge_requests/1107#note_135187046
  for some background on this.
  

• fa4a21ce
  by Chandan Singh at 2019-01-31T12:15:43Z

  Merge branch 'chandan/min-version-click-man' into 'master'
  
  tox.ini: Specify minimum version of click-man
  
  See merge request BuildStream/buildstream!1120

• dd791373
  by Chandan Singh at 2019-01-31T14:32:44Z

  testutils/site.py: Support parsing more exotic git versions
  
  We use output of `git --version` to determine if we can run some tests
  that rely on features from newer git versions. Usually, we expect the
  output to be like:
  
      git version 2.17.2
  
  On some platforms, like MacOS, there could be a suffix after the version
  string, so that it looks something like:
  
      git version 2.17.2 (Apple Git-113)
  
  This causes things to fail like so:
  
      ValueError: invalid literal for int() with base 10: '2 (Apple Git-113)\n'
  
  Fix logic around `HAVE_OLD_GIT` such that we split the output of
  `git --version` without limit on how many times we split. Previously we
  used to split only twice so the suffixes like `(Apple Git-113)` are not
  part of the parsed version.
  

• 96c0fbd6
  by Chandan Singh at 2019-01-31T15:39:19Z

  Merge branch 'chandan/fix-git-version-mac' into 'master'
  
  testutils/site.py: Support parsing more exotic git versions
  
  See merge request BuildStream/buildstream!1118

• d25e2795
  by Benjamin Schubert at 2019-01-31T17:06:23Z

  Add LoadElement in dependency list for LoadElement idrectly
  
  This removes the need for the 'Dependency' list to then be matched
  with the corresponding LoadElement and will allow iterating the
  graph more easily
  

• 2d0eebbf
  by Benjamin Schubert at 2019-01-31T17:06:23Z

  Pass element directly to collect_element
  
  This simplifies the loading
  

• 583bd97d
  by Benjamin Schubert at 2019-02-01T10:26:37Z

  Merge branch 'bschubert/loader' into 'master'
  
  Cleanup loader by linking LoadElements sooner
  
  See merge request BuildStream/buildstream!1122

• 874e683d
  by William Salmon at 2019-02-01T15:13:53Z

  Add more log formatting options
  
  The 'wallclock-us' is a option that we thought we already had.
  
  The base64 option is not human readable but should be quicker and dose
  not lose information converting back and forth from human readable.
  

6 changed files:

• buildstream/_frontend/widget.py
• buildstream/_loader/loadelement.py
• buildstream/_loader/loader.py
• tests/frontend/logging.py
• tests/testutils/site.py
• tox.ini

Changes:

buildstream/_frontend/widget.py

---

...	...	@@ -8,7 +8,7 @@
8	8	#
9	9	# This library is distributed in the hope that it will be useful,
10	10	# but WITHOUT ANY WARRANTY; without even the implied warranty of
11		-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	11	+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12	12	# Lesser General Public License for more details.
13	13	#
14	14	# You should have received a copy of the GNU Lesser General Public
...	...	@@ -17,6 +17,7 @@
17	17	# Authors:
18	18	# Tristan Van Berkom <tristan vanberkom codethink co uk>
19	19	import datetime
	20	+import base64
20	21	import os
21	22	from collections import defaultdict, OrderedDict
22	23	from contextlib import ExitStack
...	...	@@ -94,12 +95,24 @@ class FixedText(Widget):
94	95
95	96	# Used to add the wallclock time this message was created at
96	97	class WallclockTime(Widget):
	98	+ def __init__(self, context, content_profile, format_profile, output_format=False):
	99	+ self._output_format = output_format
	100	+ super(WallclockTime, self).__init__(context, content_profile, format_profile)
	101	+
97	102	def render(self, message):
	103	+
98	104	fields = [self.content_profile.fmt("{:02d}".format(x)) for x in
99	105	[message.creation_time.hour,
100	106	message.creation_time.minute,
101		- message.creation_time.second]]
102		- return self.format_profile.fmt(":").join(fields)
	107	+ message.creation_time.second,
	108	+ ]
	109	+ ]
	110	+ text = self.format_profile.fmt(":").join(fields)
	111	+
	112	+ if self._output_format == 'us':
	113	+ text += self.format_profile.fmt(".{:06d}".format(message.creation_time.microsecond))
	114	+
	115	+ return text
103	116
104	117
105	118	# A widget for rendering the debugging column
...	...	@@ -326,6 +339,8 @@ class LogLine(Widget):
326	339	"elapsed": TimeCode(context, content_profile, format_profile, microseconds=False),
327	340	"elapsed-us": TimeCode(context, content_profile, format_profile, microseconds=True),
328	341	"wallclock": WallclockTime(context, content_profile, format_profile),
	342	+ "wallclock-us": WallclockTime(context, content_profile, format_profile, output_format='us'),
	343	+ "wallclock-base64": WallclockTime(context, content_profile, format_profile, output_format='base64'),
329	344	"key": CacheKey(context, content_profile, format_profile, err_profile),
330	345	"element": ElementName(context, content_profile, format_profile),
331	346	"action": TypeName(context, content_profile, format_profile),

buildstream/_loader/loadelement.py

---

...	...	@@ -39,6 +39,20 @@ from .types import Symbol, Dependency
39	39	# loader (Loader): The Loader object for this element
40	40	#
41	41	class LoadElement():
	42	+ # Dependency():
	43	+ #
	44	+ # A link from a LoadElement to its dependencies.
	45	+ #
	46	+ # Keeps a link to one of the current Element's dependencies, together with
	47	+ # its dependency type.
	48	+ #
	49	+ # Args:
	50	+ # element (LoadElement): a LoadElement on which there is a dependency
	51	+ # dep_type (str): the type of dependency this dependency link is
	52	+ class Dependency:
	53	+ def __init__(self, element, dep_type):
	54	+ self.element = element
	55	+ self.dep_type = dep_type
42	56
43	57	def __init__(self, node, filename, loader):
44	58
...	...	@@ -74,8 +88,11 @@ class LoadElement():
74	88	'build-depends', 'runtime-depends',
75	89	])
76	90
77		- # Extract the Dependencies
78		- self.deps = _extract_depends_from_node(self.node)
	91	+ self.dependencies = []
	92	+
	93	+ @property
	94	+ def junction(self):
	95	+ return self._loader.project.junction
79	96
80	97	# depends():
81	98	#
...	...	@@ -101,8 +118,8 @@ class LoadElement():
101	118	return
102	119
103	120	self._dep_cache = {}
104		- for dep in self.deps:
105		- elt = self._loader.get_element_for_dep(dep)
	121	+ for dep in self.dependencies:
	122	+ elt = dep.element
106	123
107	124	# Ensure the cache of the element we depend on
108	125	elt._ensure_depends_cache()

buildstream/_loader/loader.py

---

...	...	@@ -19,7 +19,6 @@
19	19
20	20	import os
21	21	from functools import cmp_to_key
22		-from collections import namedtuple
23	22	from collections.abc import Mapping
24	23	import tempfile
25	24	import shutil
...	...	@@ -32,8 +31,8 @@ from .._profile import Topics, profile_start, profile_end
32	31	from .._includes import Includes
33	32	from .._yamlcache import YamlCache
34	33
35		-from .types import Symbol, Dependency
36		-from .loadelement import LoadElement
	34	+from .types import Symbol
	35	+from .loadelement import LoadElement, _extract_depends_from_node
37	36	from . import MetaElement
38	37	from . import MetaSource
39	38	from ..types import CoreWarnings
...	...	@@ -112,7 +111,7 @@ class Loader():
112	111
113	112	# First pass, recursively load files and populate our table of LoadElements
114	113	#
115		- deps = []
	114	+ target_elements = []
116	115
117	116	# XXX This will need to be changed to the context's top-level project if this method
118	117	# is ever used for subprojects
...	...	@@ -122,10 +121,10 @@ class Loader():
122	121	with YamlCache.open(self._context, cache_file) as yaml_cache:
123	122	for target in targets:
124	123	profile_start(Topics.LOAD_PROJECT, target)
125		- junction, name, loader = self._parse_name(target, rewritable, ticker,
126		- fetch_subprojects=fetch_subprojects)
127		- loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
128		- deps.append(Dependency(name, junction=junction))
	124	+ _junction, name, loader = self._parse_name(target, rewritable, ticker,
	125	+ fetch_subprojects=fetch_subprojects)
	126	+ element = loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
	127	+ target_elements.append(element)
129	128	profile_end(Topics.LOAD_PROJECT, target)
130	129
131	130	#
...	...	@@ -134,29 +133,29 @@ class Loader():
134	133
135	134	# Set up a dummy element that depends on all top-level targets
136	135	# to resolve potential circular dependencies between them
137		- DummyTarget = namedtuple('DummyTarget', ['name', 'full_name', 'deps'])
138		-
139		- dummy = DummyTarget(name='', full_name='', deps=deps)
140		- self._elements[''] = dummy
	136	+ dummy_target = LoadElement("", "", self)
	137	+ dummy_target.dependencies.extend(
	138	+ LoadElement.Dependency(element, Symbol.RUNTIME)
	139	+ for element in target_elements
	140	+ )
141	141
142	142	profile_key = "_".join(t for t in targets)
143	143	profile_start(Topics.CIRCULAR_CHECK, profile_key)
144		- self._check_circular_deps('')
	144	+ self._check_circular_deps(dummy_target)
145	145	profile_end(Topics.CIRCULAR_CHECK, profile_key)
146	146
147	147	ret = []
148	148	#
149	149	# Sort direct dependencies of elements by their dependency ordering
150	150	#
151		- for target in targets:
152		- profile_start(Topics.SORT_DEPENDENCIES, target)
153		- junction, name, loader = self._parse_name(target, rewritable, ticker,
154		- fetch_subprojects=fetch_subprojects)
155		- loader._sort_dependencies(name)
156		- profile_end(Topics.SORT_DEPENDENCIES, target)
	151	+ for element in target_elements:
	152	+ loader = element._loader
	153	+ profile_start(Topics.SORT_DEPENDENCIES, element.name)
	154	+ loader._sort_dependencies(element)
	155	+ profile_end(Topics.SORT_DEPENDENCIES, element.name)
157	156	# Finally, wrap what we have into LoadElements and return the target
158	157	#
159		- ret.append(loader._collect_element(name))
	158	+ ret.append(loader._collect_element(element))
160	159
161	160	return ret
162	161
...	...	@@ -184,22 +183,6 @@ class Loader():
184	183	if os.path.exists(self._tempdir):
185	184	shutil.rmtree(self._tempdir)
186	185
187		- # get_element_for_dep():
188		- #
189		- # Gets a cached LoadElement by Dependency object
190		- #
191		- # This is used by LoadElement
192		- #
193		- # Args:
194		- # dep (Dependency): The dependency to search for
195		- #
196		- # Returns:
197		- # (LoadElement): The cached LoadElement
198		- #
199		- def get_element_for_dep(self, dep):
200		- loader = self._get_loader_for_dep(dep)
201		- return loader._elements[dep.name]
202		-
203	186	###########################################
204	187	# Private Methods #
205	188	###########################################
...	...	@@ -272,8 +255,10 @@ class Loader():
272	255
273	256	self._elements[filename] = element
274	257
	258	+ dependencies = _extract_depends_from_node(node)
	259	+
275	260	# Load all dependency files for the new LoadElement
276		- for dep in element.deps:
	261	+ for dep in dependencies:
277	262	if dep.junction:
278	263	self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, yaml_cache)
279	264	loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
...	...	@@ -288,7 +273,9 @@ class Loader():
288	273	"{}: Cannot depend on junction"
289	274	.format(dep.provenance))
290	275
291		- deps_names = [dep.name for dep in element.deps]
	276	+ element.dependencies.append(LoadElement.Dependency(dep_element, dep.dep_type))
	277	+
	278	+ deps_names = [dep.name for dep in dependencies]
292	279	self._warn_invalid_elements(deps_names)
293	280
294	281	return element
...	...	@@ -299,12 +286,12 @@ class Loader():
299	286	# dependencies already resolved.
300	287	#
301	288	# Args:
302		- # element_name (str): The element-path relative element name to check
	289	+ # element (str): The element to check
303	290	#
304	291	# Raises:
305	292	# (LoadError): In case there was a circular dependency error
306	293	#
307		- def _check_circular_deps(self, element_name, check_elements=None, validated=None, sequence=None):
	294	+ def _check_circular_deps(self, element, check_elements=None, validated=None, sequence=None):
308	295
309	296	if check_elements is None:
310	297	check_elements = {}
...	...	@@ -313,38 +300,31 @@ class Loader():
313	300	if sequence is None:
314	301	sequence = []
315	302
316		- element = self._elements[element_name]
317		-
318		- # element name must be unique across projects
319		- # to be usable as key for the check_elements and validated dicts
320		- element_name = element.full_name
321		-
322	303	# Skip already validated branches
323		- if validated.get(element_name) is not None:
	304	+ if validated.get(element) is not None:
324	305	return
325	306
326		- if check_elements.get(element_name) is not None:
	307	+ if check_elements.get(element) is not None:
327	308	# Create `chain`, the loop of element dependencies from this
328	309	# element back to itself, by trimming everything before this
329	310	# element from the sequence under consideration.
330		- chain = sequence[sequence.index(element_name):]
331		- chain.append(element_name)
	311	+ chain = sequence[sequence.index(element.full_name):]
	312	+ chain.append(element.full_name)
332	313	raise LoadError(LoadErrorReason.CIRCULAR_DEPENDENCY,
333	314	("Circular dependency detected at element: {}\n" +
334	315	"Dependency chain: {}")
335		- .format(element.name, " -> ".join(chain)))
	316	+ .format(element.full_name, " -> ".join(chain)))
336	317
337	318	# Push / Check each dependency / Pop
338		- check_elements[element_name] = True
339		- sequence.append(element_name)
340		- for dep in element.deps:
341		- loader = self._get_loader_for_dep(dep)
342		- loader._check_circular_deps(dep.name, check_elements, validated, sequence)
343		- del check_elements[element_name]
	319	+ check_elements[element] = True
	320	+ sequence.append(element.full_name)
	321	+ for dep in element.dependencies:
	322	+ dep.element._loader._check_circular_deps(dep.element, check_elements, validated, sequence)
	323	+ del check_elements[element]
344	324	sequence.pop()
345	325
346	326	# Eliminate duplicate paths
347		- validated[element_name] = True
	327	+ validated[element] = True
348	328
349	329	# _sort_dependencies():
350	330	#
...	...	@@ -357,28 +337,21 @@ class Loader():
357	337	# sorts throughout the build process.
358	338	#
359	339	# Args:
360		- # element_name (str): The element-path relative element name to sort
	340	+ # element (LoadElement): The element to sort
361	341	#
362		- def _sort_dependencies(self, element_name, visited=None):
	342	+ def _sort_dependencies(self, element, visited=None):
363	343	if visited is None:
364		- visited = {}
	344	+ visited = set()
365	345
366		- element = self._elements[element_name]
367		-
368		- # element name must be unique across projects
369		- # to be usable as key for the visited dict
370		- element_name = element.full_name
371		-
372		- if visited.get(element_name) is not None:
	346	+ if element in visited:
373	347	return
374	348
375		- for dep in element.deps:
376		- loader = self._get_loader_for_dep(dep)
377		- loader._sort_dependencies(dep.name, visited=visited)
	349	+ for dep in element.dependencies:
	350	+ dep.element._loader._sort_dependencies(dep.element, visited=visited)
378	351
379	352	def dependency_cmp(dep_a, dep_b):
380		- element_a = self.get_element_for_dep(dep_a)
381		- element_b = self.get_element_for_dep(dep_b)
	353	+ element_a = dep_a.element
	354	+ element_b = dep_b.element
382	355
383	356	# Sort on inter element dependency first
384	357	if element_a.depends(element_b):
...	...	@@ -395,21 +368,21 @@ class Loader():
395	368	return -1
396	369
397	370	# All things being equal, string comparison.
398		- if dep_a.name > dep_b.name:
	371	+ if element_a.name > element_b.name:
399	372	return 1
400		- elif dep_a.name < dep_b.name:
	373	+ elif element_a.name < element_b.name:
401	374	return -1
402	375
403	376	# Sort local elements before junction elements
404	377	# and use string comparison between junction elements
405		- if dep_a.junction and dep_b.junction:
406		- if dep_a.junction > dep_b.junction:
	378	+ if element_a.junction and element_b.junction:
	379	+ if element_a.junction > element_b.junction:
407	380	return 1
408		- elif dep_a.junction < dep_b.junction:
	381	+ elif element_a.junction < element_b.junction:
409	382	return -1
410		- elif dep_a.junction:
	383	+ elif element_a.junction:
411	384	return -1
412		- elif dep_b.junction:
	385	+ elif element_b.junction:
413	386	return 1
414	387
415	388	# This wont ever happen
...	...	@@ -418,26 +391,23 @@ class Loader():
418	391	# Now dependency sort, we ensure that if any direct dependency
419	392	# directly or indirectly depends on another direct dependency,
420	393	# it is found later in the list.
421		- element.deps.sort(key=cmp_to_key(dependency_cmp))
	394	+ element.dependencies.sort(key=cmp_to_key(dependency_cmp))
422	395
423		- visited[element_name] = True
	396	+ visited.add(element)
424	397
425	398	# _collect_element()
426	399	#
427	400	# Collect the toplevel elements we have
428	401	#
429	402	# Args:
430		- # element_name (str): The element-path relative element name to sort
	403	+ # element (LoadElement): The element for which to load a MetaElement
431	404	#
432	405	# Returns:
433	406	# (MetaElement): A recursively loaded MetaElement
434	407	#
435		- def _collect_element(self, element_name):
436		-
437		- element = self._elements[element_name]
438		-
	408	+ def _collect_element(self, element):
439	409	# Return the already built one, if we already built it
440		- meta_element = self._meta_elements.get(element_name)
	410	+ meta_element = self._meta_elements.get(element.name)
441	411	if meta_element:
442	412	return meta_element
443	413
...	...	@@ -461,10 +431,10 @@ class Loader():
461	431	del source[Symbol.DIRECTORY]
462	432
463	433	index = sources.index(source)
464		- meta_source = MetaSource(element_name, index, element_kind, kind, source, directory)
	434	+ meta_source = MetaSource(element.name, index, element_kind, kind, source, directory)
465	435	meta_sources.append(meta_source)
466	436
467		- meta_element = MetaElement(self.project, element_name, element_kind,
	437	+ meta_element = MetaElement(self.project, element.name, element_kind,
468	438	elt_provenance, meta_sources,
469	439	_yaml.node_get(node, Mapping, Symbol.CONFIG, default_value={}),
470	440	_yaml.node_get(node, Mapping, Symbol.VARIABLES, default_value={}),
...	...	@@ -475,12 +445,12 @@ class Loader():
475	445	element_kind == 'junction')
476	446
477	447	# Cache it now, make sure it's already there before recursing
478		- self._meta_elements[element_name] = meta_element
	448	+ self._meta_elements[element.name] = meta_element
479	449
480	450	# Descend
481		- for dep in element.deps:
482		- loader = self._get_loader_for_dep(dep)
483		- meta_dep = loader._collect_element(dep.name)
	451	+ for dep in element.dependencies:
	452	+ loader = dep.element._loader
	453	+ meta_dep = loader._collect_element(dep.element)
484	454	if dep.dep_type != 'runtime':
485	455	meta_element.build_dependencies.append(meta_dep)
486	456	if dep.dep_type != 'build':
...	...	@@ -539,7 +509,7 @@ class Loader():
539	509	return None
540	510
541	511	# meta junction element
542		- meta_element = self._collect_element(filename)
	512	+ meta_element = self._collect_element(self._elements[filename])
543	513	if meta_element.kind != 'junction':
544	514	raise LoadError(LoadErrorReason.INVALID_DATA,
545	515	"{}: Expected junction but element kind is {}".format(filename, meta_element.kind))
...	...	@@ -601,23 +571,6 @@ class Loader():
601	571
602	572	return loader
603	573
604		- # _get_loader_for_dep():
605		- #
606		- # Gets the appropriate Loader for a Dependency object
607		- #
608		- # Args:
609		- # dep (Dependency): A Dependency object
610		- #
611		- # Returns:
612		- # (Loader): The Loader object to use for this Dependency
613		- #
614		- def _get_loader_for_dep(self, dep):
615		- if dep.junction:
616		- # junction dependency, delegate to appropriate loader
617		- return self._loaders[dep.junction]
618		- else:
619		- return self
620		-
621	574	# _parse_name():
622	575	#
623	576	# Get junction and base name of element along with loader for the sub-project

tests/frontend/logging.py

---

...	...	@@ -52,7 +52,8 @@ def test_custom_logging(cli, tmpdir, datafiles):
52	52	element_path = os.path.join(project, 'elements')
53	53	element_name = 'fetch-test-git.bst'
54	54
55		- custom_log_format = '%{elapsed},%{elapsed-us},%{wallclock},%{key},%{element},%{action},%{message}'
	55	+ custom_log_format = ('%{elapsed},%{elapsed-us},%{wallclock},%{wallclock-us},'
	56	+ '%{key},%{element},%{action},%{message}')
56	57	user_config = {'logging': {'message-format': custom_log_format}}
57	58	cli.configure(user_config)
58	59
...	...	@@ -77,7 +78,8 @@ def test_custom_logging(cli, tmpdir, datafiles):
77	78	result = cli.run(project=project, args=['source', 'fetch', element_name])
78	79	result.assert_success()
79	80
80		- m = re.search(r"\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6},\d\d:\d\d:\d\d,\s*,.*,SUCCESS,Checking sources", result.stderr)
	81	+ m = re.search(r"\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6},\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6}\s*,.*"
	82	+ r",SUCCESS,Checking sources", result.stderr)
81	83	assert(m is not None)
82	84
83	85

tests/testutils/site.py

---

...	...	@@ -18,7 +18,7 @@ try:
18	18	utils.get_host_tool('git')
19	19	HAVE_GIT = True
20	20	out = str(subprocess.check_output(['git', '--version']), "utf-8")
21		- version = tuple(int(x) for x in out.split(' ', 2)[2].split('.'))
	21	+ version = tuple(int(x) for x in out.split(' ')[2].split('.'))
22	22	HAVE_OLD_GIT = version < (1, 8, 5)
23	23	except ProgramNotFoundError:
24	24	HAVE_GIT = False

tox.ini

---

...	...	@@ -88,5 +88,5 @@ whitelist_externals =
88	88	commands =
89	89	python3 setup.py --command-packages=click_man.commands man_pages
90	90	deps =
91		- click-man
	91	+ click-man >= 0.3.0
92	92	-rrequirements/requirements.txt

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.