[Notes] [Git][BuildStream/buildstream][bschubert/pipeline] WIP: notes

[Benjamin Schubert <gitlab mg gitlab com>]

Title: GitLab

Benjamin Schubert pushed to branch bschubert/pipeline at BuildStream / buildstream

Commits:

• 19079360
  by Benjamin Schubert at 2019-01-11T10:24:58Z

  WIP: notes
  

6 changed files:

• buildstream/_pipeline.py
• buildstream/_scheduler/queues/buildqueue.py
• buildstream/_scheduler/queues/fetchqueue.py
• buildstream/_scheduler/queues/pullqueue.py
• buildstream/_stream.py
• buildstream/element.py

Changes:

buildstream/_pipeline.py

---

...	...	@@ -136,9 +136,6 @@ class Pipeline():
136	136	# Preflight
137	137	element._preflight()
138	138
139		- # Determine initial element state.
140		- element._update_state()
141		-
142	139	# dependencies()
143	140	#
144	141	# Generator function to iterate over elements and optionally

buildstream/_scheduler/queues/buildqueue.py

---

...	...	@@ -106,10 +106,6 @@ class BuildQueue(Queue):
106	106	# Inform element in main process that assembly is done
107	107	element._assemble_done()
108	108
109		- # Update the state of all reverse dependencies
110		- for reverse_dependency in element.reverse_build_dependencies:
111		- reverse_dependency._update_state()
112		-
113	109	# This has to be done after _assemble_done, such that the
114	110	# element may register its cache key as required
115	111	#

buildstream/_scheduler/queues/fetchqueue.py

---

...	...	@@ -44,9 +44,6 @@ class FetchQueue(Queue):
44	44	element._fetch()
45	45
46	46	def status(self, element):
47		- # state of dependencies may have changed, recalculate element state
48		- element._update_state()
49		-
50	47	if not element._is_required():
51	48	# Artifact is not currently required but it may be requested later.
52	49	# Keep it in the queue.
...	...	@@ -72,7 +69,7 @@ class FetchQueue(Queue):
72	69	if status == JobStatus.FAIL:
73	70	return
74	71
75		- element._update_state()
76		-
	72	+ element._fetch_done()
	73	+
77	74	# Successful fetch, we must be CACHED now
78	75	assert element._get_consistency() == Consistency.CACHED

buildstream/_scheduler/queues/pullqueue.py

---

...	...	@@ -39,9 +39,6 @@ class PullQueue(Queue):
39	39	raise SkipJob(self.action_name)
40	40
41	41	def status(self, element):
42		- # state of dependencies may have changed, recalculate element state
43		- element._update_state()
44		-
45	42	if not element._is_required():
46	43	# Artifact is not currently required but it may be requested later.
47	44	# Keep it in the queue.

buildstream/_stream.py

---

...	...	@@ -1018,17 +1018,6 @@ class Stream():
1018	1018
1019	1019	_, status = self._scheduler.run(self.queues)
1020	1020
1021		- # Force update element states after a run, such that the summary
1022		- # is more coherent
1023		- try:
1024		- for element in self.total_elements:
1025		- element._update_state()
1026		- except BstError as e:
1027		- self._message(MessageType.ERROR, "Error resolving final state", detail=str(e))
1028		- set_last_task_error(e.domain, e.reason)
1029		- except Exception as e: # pylint: disable=broad-except
1030		- self._message(MessageType.BUG, "Unhandled exception while resolving final state", detail=str(e))
1031		-
1032	1021	if status == SchedStatus.ERROR:
1033	1022	raise StreamError()
1034	1023	elif status == SchedStatus.TERMINATED:

buildstream/element.py

---

...	...	@@ -197,7 +197,7 @@ class Element(Plugin):
197	197
198	198	self.__runtime_dependencies = [] # Direct runtime dependency Elements
199	199	self.__build_dependencies = [] # Direct build dependency Elements
200		- self.reverse_build_dependencies = [] # Direct reverse build dependency Elements
	200	+ self.__reverse_build_dependencies = [] # Direct reverse build dependency Elements
201	201	self.__sources = [] # List of Sources
202	202	self.__weak_cache_key = None # Our cached weak cache key
203	203	self.__strict_cache_key = None # Our cached cache key for strict builds
...	...	@@ -228,6 +228,8 @@ class Element(Plugin):
228	228	self.__metadata_workspaced = {} # Boolean of whether it's workspaced
229	229	self.__metadata_workspaced_dependencies = {} # List of which dependencies are workspaced
230	230
	231	+ self.__is_workspaced = None
	232	+
231	233	# Ensure we have loaded this class's defaults
232	234	self.__init_defaults(plugin_conf)
233	235
...	...	@@ -371,6 +373,13 @@ class Element(Plugin):
371	373	#############################################################
372	374	# Public Methods #
373	375	#############################################################
	376	+ @property
	377	+ def is_workspaced(self):
	378	+ if self.__is_workspaced is None:
	379	+ self.__is_workspaced = self._get_workspace() is not None
	380	+
	381	+ return self.__is_workspaced
	382	+
374	383	def sources(self):
375	384	"""A generator function to enumerate the element sources
376	385
...	...	@@ -440,6 +449,27 @@ class Element(Plugin):
440	449	if should_yield and (recurse or recursed) and scope != Scope.BUILD:
441	450	yield self
442	451
	452	+ def reverse_build_dependencies(self, recurse=False):
	453	+ if not recurse:
	454	+ yield from self.__reverse_build_dependencies
	455	+ return
	456	+
	457	+ # visited = set()
	458	+
	459	+ def recurse_rdeps(element):
	460	+ # if element in visited:
	461	+ # return
	462	+
	463	+ # visited.add(element)
	464	+
	465	+ yield element
	466	+
	467	+ for rdep in element.__reverse_build_dependencies:
	468	+ yield from recurse_rdeps(rdep)
	469	+
	470	+ for rdep in self.__reverse_build_dependencies:
	471	+ yield from recurse_rdeps(rdep)
	472	+
443	473	def search(self, scope, name):
444	474	"""Search for a dependency by name
445	475
...	...	@@ -931,7 +961,7 @@ class Element(Plugin):
931	961	for meta_dep in meta.build_dependencies:
932	962	dependency = Element._new_from_meta(meta_dep)
933	963	element.__build_dependencies.append(dependency)
934		- dependency.reverse_build_dependencies.append(element)
	964	+ dependency.__reverse_build_dependencies.append(element)
935	965
936	966	return element
937	967
...	...	@@ -1281,6 +1311,9 @@ class Element(Plugin):
1281	1311	for source in self.sources():
1282	1312	source._preflight()
1283	1313
	1314	+ # Determine initial element state.
	1315	+ self._update_state()
	1316	+
1284	1317	# _schedule_tracking():
1285	1318	#
1286	1319	# Force an element state to be inconsistent. Any sources appear to be
...	...	@@ -1308,6 +1341,9 @@ class Element(Plugin):
1308	1341
1309	1342	self._update_state()
1310	1343
	1344	+ for rdep in self.reverse_build_dependencies(recurse=True):
	1345	+ rdep._update_state()
	1346	+
1311	1347	# _track():
1312	1348	#
1313	1349	# Calls track() on the Element sources
...	...	@@ -1448,6 +1484,7 @@ class Element(Plugin):
1448	1484	# This unblocks pull/fetch/build.
1449	1485	#
1450	1486	def _set_required(self):
	1487	+ # FIXME: this should enqueue stuff in the queue, it should not be here by default
1451	1488	if self.__required:
1452	1489	# Already done
1453	1490	return
...	...	@@ -1458,6 +1495,7 @@ class Element(Plugin):
1458	1495	for dep in self.dependencies(Scope.RUN, recurse=False):
1459	1496	dep._set_required()
1460	1497
	1498	+ # FIXME: this should not be done at all here
1461	1499	self._update_state()
1462	1500
1463	1501	# _is_required():
...	...	@@ -1501,10 +1539,16 @@ class Element(Plugin):
1501	1539
1502	1540	self.__assemble_scheduled = False
1503	1541	self.__assemble_done = True
1504		-
	1542	+ # FIXME: only if workspaced
1505	1543	self._update_state()
1506	1544
1507		- if self._get_workspace() and self._cached_success():
	1545	+ if self.is_workspaced:
	1546	+
	1547	+ # Update the state of all reverse dependencies
	1548	+ for reverse_dependency in self.reverse_build_dependencies(recurse=True):
	1549	+ reverse_dependency._update_state()
	1550	+
	1551	+ if self.is_workspaced and self._cached_success():
1508	1552	assert utils._is_main_process(), \
1509	1553	"Attempted to save workspace configuration from child process"
1510	1554	#
...	...	@@ -2037,6 +2081,9 @@ class Element(Plugin):
2037	2081	source._fetch(previous_sources)
2038	2082	previous_sources.append(source)
2039	2083
	2084	+ def _fetch_done(self):
	2085	+ self._update_state()
	2086	+
2040	2087	#############################################################
2041	2088	# Private Local Methods #
2042	2089	#############################################################

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.