[Notes] [Git][BuildStream/buildstream][bschubert/pipeline] Stop updating state in queue.status()

[Benjamin Schubert <gitlab mg gitlab com>]

Title: GitLab

Benjamin Schubert pushed to branch bschubert/pipeline at BuildStream / buildstream

Commits:

• b69df031
  by Benjamin Schubert at 2019-01-15T11:32:29Z

  Stop updating state in queue.status()
  
  Statuses of an element can be changed when:
  
  1) It is pulled
  2) It is fetched
  3) It is workspaced and it finished building
  4) One of its dependencies is tracked
  5) One of its dependencies is workspaced and finished building
  
  We can therefore update the statuses at those moments and we don't
  need to check all the time.
  
  This reduces considerably the calls to update_states that are done
  

4 changed files:

• buildstream/_scheduler/queues/buildqueue.py
• buildstream/_scheduler/queues/fetchqueue.py
• buildstream/_scheduler/queues/pullqueue.py
• buildstream/element.py

Changes:

buildstream/_scheduler/queues/buildqueue.py

---

...	...	@@ -71,9 +71,6 @@ class BuildQueue(Queue):
71	71	return element._assemble()
72	72
73	73	def status(self, element):
74		- # state of dependencies may have changed, recalculate element state
75		- element._update_state()
76		-
77	74	if not element._is_required():
78	75	# Artifact is not currently required but it may be requested later.
79	76	# Keep it in the queue.

buildstream/_scheduler/queues/fetchqueue.py

---

...	...	@@ -44,9 +44,6 @@ class FetchQueue(Queue):
44	44	element._fetch()
45	45
46	46	def status(self, element):
47		- # state of dependencies may have changed, recalculate element state
48		- element._update_state()
49		-
50	47	if not element._is_required():
51	48	# Artifact is not currently required but it may be requested later.
52	49	# Keep it in the queue.

buildstream/_scheduler/queues/pullqueue.py

---

...	...	@@ -39,9 +39,6 @@ class PullQueue(Queue):
39	39	raise SkipJob(self.action_name)
40	40
41	41	def status(self, element):
42		- # state of dependencies may have changed, recalculate element state
43		- element._update_state()
44		-
45	42	if not element._is_required():
46	43	# Artifact is not currently required but it may be requested later.
47	44	# Keep it in the queue.

buildstream/element.py

---

...	...	@@ -197,6 +197,7 @@ class Element(Plugin):
197	197
198	198	self.__runtime_dependencies = [] # Direct runtime dependency Elements
199	199	self.__build_dependencies = [] # Direct build dependency Elements
	200	+ self.__reverse_build_dependencies = [] # Direct reverse dependency Elements
200	201	self.__sources = [] # List of Sources
201	202	self.__weak_cache_key = None # Our cached weak cache key
202	203	self.__strict_cache_key = None # Our cached cache key for strict builds
...	...	@@ -439,6 +440,21 @@ class Element(Plugin):
439	440	if should_yield and (recurse or recursed) and scope != Scope.BUILD:
440	441	yield self
441	442
	443	+ def reverse_build_dependencies(self):
	444	+ """Get all reverse dependencies for the given element in a topological order.
	445	+
	446	+ Currently, dependencies might be returned multiple times.
	447	+ """
	448	+ # FIXME: we should return each entry only once in topological order
	449	+ def recurse_rdeps(element):
	450	+ yield element
	451	+
	452	+ for rdep in element.__reverse_build_dependencies:
	453	+ yield from recurse_rdeps(rdep)
	454	+
	455	+ for rdep in self.__reverse_build_dependencies:
	456	+ yield from recurse_rdeps(rdep)
	457	+
442	458	def search(self, scope, name):
443	459	"""Search for a dependency by name
444	460
...	...	@@ -930,6 +946,7 @@ class Element(Plugin):
930	946	for meta_dep in meta.build_dependencies:
931	947	dependency = Element._new_from_meta(meta_dep)
932	948	element.__build_dependencies.append(dependency)
	949	+ dependency.__reverse_build_dependencies.append(element)
933	950
934	951	return element
935	952
...	...	@@ -1306,6 +1323,9 @@ class Element(Plugin):
1306	1323
1307	1324	self._update_state()
1308	1325
	1326	+ for reverse_dep in self.reverse_build_dependencies():
	1327	+ reverse_dep._update_state()
	1328	+
1309	1329	# _track():
1310	1330	#
1311	1331	# Calls track() on the Element sources
...	...	@@ -1503,6 +1523,9 @@ class Element(Plugin):
1503	1523	self._update_state()
1504	1524
1505	1525	if self._get_workspace() and self._cached_success():
	1526	+ for rdep in self.reverse_build_dependencies():
	1527	+ rdep._update_state()
	1528	+
1506	1529	assert utils._is_main_process(), \
1507	1530	"Attempted to save workspace configuration from child process"
1508	1531	#

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.