[gnome-continuous-yocto/gnomeostree-3.28-rocko: 920/8267] bitbake: taskdata/runqueue: Rewrite without use of ID indirection
- From: Emmanuele Bassi <ebassi src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-continuous-yocto/gnomeostree-3.28-rocko: 920/8267] bitbake: taskdata/runqueue: Rewrite without use of ID indirection
- Date: Sat, 16 Dec 2017 21:06:08 +0000 (UTC)
commit af04a52bf7d7ae94c187e85028a80dee6d7853f0
Author: Richard Purdie <richard purdie linuxfoundation org>
Date: Sun Jun 12 23:55:48 2016 +0100
bitbake: taskdata/runqueue: Rewrite without use of ID indirection
I'm not sure what possesed me when I wrote this code originally but its
indirection of everyting to use numeric IDs and position dependent lists
is horrific. Given the way python internals work, its completely and
utterly pointless from performance perspective. It also makes the code
hard to understand and debug since any numeric ID has to be translated
into something human readable.
The hard part is that the IDs are infectous and spread from taskdata
into runqueue and even partly into cooker for the dependency graph
processing. The only real way to deal with this is to convert everything
to use a more sane data structure.
This patch:
* Uses "<fn>:<taskname>" as the ID for tasks rather than a number
* Changes to dict() based structures rather than position dependent lists
* Drops the build name, runtime name and filename ID indexes
On the most part there shouldn't be user visible changes. Sadly we did
leak datastructures to the setscene verify function which has to be
rewritten. To handle this, the variable name used to specifiy the version
changes from BB_SETSCENE_VERIFY_FUNCTION to BB_SETSCENE_VERIFY_FUNCTION2
allowing multiple versions of bitbake to work with suitably written
metadata. Anyone with custom schedulers may also need to change them.
I believe the benefits in code readability and easier debugging far
outweigh those issues though. It also means we have a saner codebase
to add multiconfig support on top of.
During development, I did have some of the original code coexisting with
the new data stores to allow comparision of the data and check it was
working correcty, particuarly for taskdata. I have also compared
task-depends.dot files before and after the change. There should be no
functionality changes in this patch, its purely a data structure change
and that is visible in the patch.
(Bitbake rev: 2c88afb60da54e58f555411a7bd7b006b0c29306)
Signed-off-by: Richard Purdie <richard purdie linuxfoundation org>
bitbake/lib/bb/cooker.py | 72 ++--
bitbake/lib/bb/runqueue.py | 958 +++++++++++++++++++++-----------------------
bitbake/lib/bb/taskdata.py | 412 +++++++------------
3 files changed, 639 insertions(+), 803 deletions(-)
---
diff --git a/bitbake/lib/bb/cooker.py b/bitbake/lib/bb/cooker.py
index ce818f3..9075654 100644
--- a/bitbake/lib/bb/cooker.py
+++ b/bitbake/lib/bb/cooker.py
@@ -623,9 +623,7 @@ class BBCooker:
taskdata, runlist, pkgs_to_build = self.buildTaskData(pkgs_to_build, None,
self.configuration.abort, allowincomplete=True)
- targetid = taskdata.getbuild_id(pkgs_to_build[0])
- fnid = taskdata.build_targets[targetid][0]
- fn = taskdata.fn_index[fnid]
+ fn = taskdata.build_targets[pkgs_to_build[0]][0]
else:
envdata = self.data
@@ -714,7 +712,7 @@ class BBCooker:
def buildDependTree(self, rq, taskdata):
- seen_fnids = []
+ seen_fns = []
depend_tree = {}
depend_tree["depends"] = {}
depend_tree["tdepends"] = {}
@@ -732,10 +730,9 @@ class BBCooker:
version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
depend_tree['providermap'][name] = (pn, version)
- for task in range(len(rq.rqdata.runq_fnid)):
- taskname = rq.rqdata.runq_task[task]
- fnid = rq.rqdata.runq_fnid[task]
- fn = taskdata.fn_index[fnid]
+ for tid in rq.rqdata.runtaskentries:
+ taskname = bb.runqueue.taskname_from_tid(tid)
+ fn = bb.runqueue.fn_from_tid(tid)
pn = self.recipecache.pkg_fn[fn]
version = "%s:%s-%s" % self.recipecache.pkg_pepvpr[fn]
if pn not in depend_tree["pn"]:
@@ -756,24 +753,24 @@ class BBCooker:
depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn]
- for dep in rq.rqdata.runq_depends[task]:
- depfn = taskdata.fn_index[rq.rqdata.runq_fnid[dep]]
+ for dep in rq.rqdata.runtaskentries[tid].depends:
+ depfn = bb.runqueue.fn_from_tid(dep)
deppn = self.recipecache.pkg_fn[depfn]
- dotname = "%s.%s" % (pn, rq.rqdata.runq_task[task])
+ dotname = "%s.%s" % (pn, bb.runqueue.taskname_from_tid(dep))
if not dotname in depend_tree["tdepends"]:
depend_tree["tdepends"][dotname] = []
- depend_tree["tdepends"][dotname].append("%s.%s" % (deppn, rq.rqdata.runq_task[dep]))
- if fnid not in seen_fnids:
- seen_fnids.append(fnid)
+ depend_tree["tdepends"][dotname].append("%s.%s" % (deppn,
bb.runqueue.taskname_from_tid(dep)))
+ if fn not in seen_fns:
+ seen_fns.append(fn)
packages = []
depend_tree["depends"][pn] = []
- for dep in taskdata.depids[fnid]:
- depend_tree["depends"][pn].append(taskdata.build_names_index[dep])
+ for dep in taskdata.depids[fn]:
+ depend_tree["depends"][pn].append(dep)
depend_tree["rdepends-pn"][pn] = []
- for rdep in taskdata.rdepids[fnid]:
- depend_tree["rdepends-pn"][pn].append(taskdata.run_names_index[rdep])
+ for rdep in taskdata.rdepids[fn]:
+ depend_tree["rdepends-pn"][pn].append(rdep)
rdepends = self.recipecache.rundeps[fn]
for package in rdepends:
@@ -805,12 +802,8 @@ class BBCooker:
Create a dependency tree of pkgs_to_build, returning the data.
"""
_, taskdata = self.prepareTreeData(pkgs_to_build, task)
- tasks_fnid = []
- if len(taskdata.tasks_name) != 0:
- for task in range(len(taskdata.tasks_name)):
- tasks_fnid.append(taskdata.tasks_fnid[task])
- seen_fnids = []
+ seen_fns = []
depend_tree = {}
depend_tree["depends"] = {}
depend_tree["pn"] = {}
@@ -825,9 +818,8 @@ class BBCooker:
cachefields = getattr(cache_class, 'cachefields', [])
extra_info = extra_info + cachefields
- for task in range(len(tasks_fnid)):
- fnid = tasks_fnid[task]
- fn = taskdata.fn_index[fnid]
+ for tid in taskdata.taskentries:
+ fn = bb.runqueue.fn_from_tid(tid)
pn = self.recipecache.pkg_fn[fn]
if pn not in depend_tree["pn"]:
@@ -843,33 +835,27 @@ class BBCooker:
for ei in extra_info:
depend_tree["pn"][pn][ei] = vars(self.recipecache)[ei][fn]
- if fnid not in seen_fnids:
- seen_fnids.append(fnid)
+ if fn not in seen_fns:
+ seen_fns.append(fn)
depend_tree["depends"][pn] = []
- for dep in taskdata.depids[fnid]:
- item = taskdata.build_names_index[dep]
+ for item in taskdata.depids[fn]:
pn_provider = ""
- targetid = taskdata.getbuild_id(item)
- if targetid in taskdata.build_targets and taskdata.build_targets[targetid]:
- id = taskdata.build_targets[targetid][0]
- fn_provider = taskdata.fn_index[id]
+ if dep in taskdata.build_targets and taskdata.build_targets[dep]:
+ fn_provider = taskdata.build_targets[dep][0]
pn_provider = self.recipecache.pkg_fn[fn_provider]
else:
pn_provider = item
depend_tree["depends"][pn].append(pn_provider)
depend_tree["rdepends-pn"][pn] = []
- for rdep in taskdata.rdepids[fnid]:
- item = taskdata.run_names_index[rdep]
+ for rdep in taskdata.rdepids[fn]:
pn_rprovider = ""
- targetid = taskdata.getrun_id(item)
- if targetid in taskdata.run_targets and taskdata.run_targets[targetid]:
- id = taskdata.run_targets[targetid][0]
- fn_rprovider = taskdata.fn_index[id]
+ if rdep in taskdata.run_targets and taskdata.run_targets[rdep]:
+ fn_rprovider = taskdata.run_targets[rdep][0]
pn_rprovider = self.recipecache.pkg_fn[fn_rprovider]
else:
- pn_rprovider = item
+ pn_rprovider = rdep
depend_tree["rdepends-pn"][pn].append(pn_rprovider)
depend_tree["rdepends-pkg"].update(rdepends)
@@ -1350,7 +1336,7 @@ class BBCooker:
return False
if not retval:
- bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, item, failures,
interrupted), self.expanded_data)
+ bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, item,
failures, interrupted), self.expanded_data)
self.command.finishAsyncCommand(msg)
return False
if retval is True:
@@ -1386,7 +1372,7 @@ class BBCooker:
return False
if not retval:
- bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runq_fnid), buildname, targets,
failures, interrupted), self.data)
+ bb.event.fire(bb.event.BuildCompleted(len(rq.rqdata.runtaskentries), buildname, targets,
failures, interrupted), self.data)
self.command.finishAsyncCommand(msg)
return False
if retval is True:
diff --git a/bitbake/lib/bb/runqueue.py b/bitbake/lib/bb/runqueue.py
index de618ff..b32e8e6 100644
--- a/bitbake/lib/bb/runqueue.py
+++ b/bitbake/lib/bb/runqueue.py
@@ -42,6 +42,12 @@ logger = logging.getLogger("BitBake.RunQueue")
__find_md5__ = re.compile( r'(?i)(?<![a-z0-9])[a-f0-9]{32}(?![a-z0-9])' )
+def fn_from_tid(tid):
+ return tid.rsplit(":", 1)[0]
+
+def taskname_from_tid(tid):
+ return tid.rsplit(":", 1)[1]
+
class RunQueueStats:
"""
Holds statistics on the tasks handled by the associated runQueue
@@ -97,19 +103,18 @@ class RunQueueScheduler(object):
"""
self.rq = runqueue
self.rqdata = rqdata
- self.numTasks = len(self.rqdata.runq_fnid)
+ self.numTasks = len(self.rqdata.runtaskentries)
- self.prio_map = []
- self.prio_map.extend(range(self.numTasks))
+ self.prio_map = [self.rqdata.runtaskentries.keys()]
self.buildable = []
self.stamps = {}
- for taskid in range(self.numTasks):
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[taskid]]
- taskname = self.rqdata.runq_task[taskid]
- self.stamps[taskid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
- if taskid in self.rq.runq_buildable:
- self.buildable.append(taskid)
+ for tid in self.rqdata.runtaskentries:
+ fn = fn_from_tid(tid)
+ taskname = taskname_from_tid(tid)
+ self.stamps[tid] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
+ if tid in self.rq.runq_buildable:
+ self.buildable.append(tid)
self.rev_prio_map = None
@@ -121,26 +126,26 @@ class RunQueueScheduler(object):
if not self.buildable:
return None
if len(self.buildable) == 1:
- taskid = self.buildable[0]
- stamp = self.stamps[taskid]
+ tid = self.buildable[0]
+ stamp = self.stamps[tid]
if stamp not in self.rq.build_stamps.values():
- return taskid
+ return tid
if not self.rev_prio_map:
- self.rev_prio_map = list(range(self.numTasks))
- for taskid in range(self.numTasks):
- self.rev_prio_map[self.prio_map[taskid]] = taskid
+ self.rev_prio_map = {}
+ for tid in self.rqdata.runtaskentries:
+ self.rev_prio_map[tid] = self.prio_map.index(tid)
best = None
bestprio = None
- for taskid in self.buildable:
- prio = self.rev_prio_map[taskid]
+ for tid in self.buildable:
+ prio = self.rev_prio_map[tid]
if bestprio is None or bestprio > prio:
- stamp = self.stamps[taskid]
+ stamp = self.stamps[tid]
if stamp in self.rq.build_stamps.values():
continue
bestprio = prio
- best = taskid
+ best = tid
return best
@@ -167,14 +172,17 @@ class RunQueueSchedulerSpeed(RunQueueScheduler):
"""
RunQueueScheduler.__init__(self, runqueue, rqdata)
- sortweight = sorted(copy.deepcopy(self.rqdata.runq_weight))
- copyweight = copy.deepcopy(self.rqdata.runq_weight)
- self.prio_map = []
+ weights = {}
+ for tid in self.rqdata.runtaskentries:
+ weight = self.rqdata.runtaskentries[tid].weight
+ if not weight in weights:
+ weights[weight] = []
+ weights[weight].append(tid)
- for weight in sortweight:
- idx = copyweight.index(weight)
- self.prio_map.append(idx)
- copyweight[idx] = -1
+ self.prio_map = []
+ for weight in sorted(weights):
+ for w in weights[weight]:
+ self.prio_map.append(w)
self.prio_map.reverse()
@@ -191,25 +199,33 @@ class RunQueueSchedulerCompletion(RunQueueSchedulerSpeed):
def __init__(self, runqueue, rqdata):
RunQueueSchedulerSpeed.__init__(self, runqueue, rqdata)
- #FIXME - whilst this groups all fnids together it does not reorder the
- #fnid groups optimally.
+ #FIXME - whilst this groups all fns together it does not reorder the
+ #fn groups optimally.
basemap = copy.deepcopy(self.prio_map)
self.prio_map = []
while (len(basemap) > 0):
entry = basemap.pop(0)
self.prio_map.append(entry)
- fnid = self.rqdata.runq_fnid[entry]
+ fn = fn_from_tid(entry)
todel = []
for entry in basemap:
- entry_fnid = self.rqdata.runq_fnid[entry]
- if entry_fnid == fnid:
+ entry_fn = fn_from_tid(entry)
+ if entry_fn == fn:
todel.append(basemap.index(entry))
self.prio_map.append(entry)
todel.reverse()
for idx in todel:
del basemap[idx]
+class RunTaskEntry(object):
+ def __init__(self):
+ self.depends = set()
+ self.revdeps = set()
+ self.hash = None
+ self.task = None
+ self.weight = 1
+
class RunQueueData:
"""
BitBake Run Queue implementation
@@ -228,48 +244,29 @@ class RunQueueData:
self.reset()
def reset(self):
- self.runq_fnid = []
- self.runq_task = []
- self.runq_depends = []
- self.runq_revdeps = []
- self.runq_hash = []
+ self.runtaskentries = {}
def runq_depends_names(self, ids):
import re
ret = []
- for id in self.runq_depends[ids]:
- nam = os.path.basename(self.get_user_idstring(id))
+ for id in ids:
+ nam = os.path.basename(id)
nam = re.sub("_[^,]*,", ",", nam)
ret.extend([nam])
return ret
- def get_task_name(self, task):
- return self.runq_task[task]
-
- def get_task_file(self, task):
- return self.taskData.fn_index[self.runq_fnid[task]]
+ def get_task_hash(self, tid):
+ return self.runtaskentries[tid].hash
- def get_task_hash(self, task):
- return self.runq_hash[task]
-
- def get_user_idstring(self, task, task_name_suffix = ""):
- fn = self.taskData.fn_index[self.runq_fnid[task]]
- taskname = self.runq_task[task] + task_name_suffix
- return "%s, %s" % (fn, taskname)
+ def get_user_idstring(self, tid, task_name_suffix = ""):
+ return tid + task_name_suffix
def get_short_user_idstring(self, task, task_name_suffix = ""):
- fn = self.taskData.fn_index[self.runq_fnid[task]]
+ fn = fn_from_tid(task)
pn = self.dataCache.pkg_fn[fn]
- taskname = self.runq_task[task] + task_name_suffix
+ taskname = taskname_from_tid(task) + task_name_suffix
return "%s:%s" % (pn, taskname)
-
- def get_task_id(self, fnid, taskname):
- for listid in range(len(self.runq_fnid)):
- if self.runq_fnid[listid] == fnid and self.runq_task[listid] == taskname:
- return listid
- return None
-
def circular_depchains_handler(self, tasks):
"""
Some tasks aren't buildable, likely due to circular dependency issues.
@@ -314,11 +311,11 @@ class RunQueueData:
return True
return False
- def find_chains(taskid, prev_chain):
- prev_chain.append(taskid)
+ def find_chains(tid, prev_chain):
+ prev_chain.append(tid)
total_deps = []
- total_deps.extend(self.runq_revdeps[taskid])
- for revdep in self.runq_revdeps[taskid]:
+ total_deps.extend(self.runtaskentries[tid].revdeps)
+ for revdep in self.runtaskentries[tid].revdeps:
if revdep in prev_chain:
idx = prev_chain.index(revdep)
# To prevent duplicates, reorder the chain to start with the lowest taskid
@@ -329,7 +326,7 @@ class RunQueueData:
valid_chains.append(new_chain)
msgs.append("Dependency loop #%d found:\n" % len(valid_chains))
for dep in new_chain:
- msgs.append(" Task %s (%s) (dependent Tasks %s)\n" % (dep,
self.get_user_idstring(dep), self.runq_depends_names(dep)))
+ msgs.append(" Task %s (dependent Tasks %s)\n" % (dep,
self.runq_depends_names(self.runtaskentries[dep].depends)))
msgs.append("\n")
if len(valid_chains) > 10:
msgs.append("Aborted dependency loops search after 10 matches.\n")
@@ -366,25 +363,25 @@ class RunQueueData:
possible to execute due to circular dependencies.
"""
- numTasks = len(self.runq_fnid)
- weight = []
- deps_left = []
- task_done = []
+ numTasks = len(self.runtaskentries)
+ weight = {}
+ deps_left = {}
+ task_done = {}
- for listid in range(numTasks):
- task_done.append(False)
- weight.append(1)
- deps_left.append(len(self.runq_revdeps[listid]))
+ for tid in self.runtaskentries:
+ task_done[tid] = False
+ weight[tid] = 1
+ deps_left[tid] = len(self.runtaskentries[tid].revdeps)
- for listid in endpoints:
- weight[listid] = 10
- task_done[listid] = True
+ for tid in endpoints:
+ weight[tid] = 10
+ task_done[tid] = True
while True:
next_points = []
- for listid in endpoints:
- for revdep in self.runq_depends[listid]:
- weight[revdep] = weight[revdep] + weight[listid]
+ for tid in endpoints:
+ for revdep in self.runtaskentries[tid].depends:
+ weight[revdep] = weight[revdep] + weight[tid]
deps_left[revdep] = deps_left[revdep] - 1
if deps_left[revdep] == 0:
next_points.append(revdep)
@@ -395,14 +392,15 @@ class RunQueueData:
# Circular dependency sanity check
problem_tasks = []
- for task in range(numTasks):
- if task_done[task] is False or deps_left[task] != 0:
- problem_tasks.append(task)
- logger.debug(2, "Task %s (%s) is not buildable", task, self.get_user_idstring(task))
- logger.debug(2, "(Complete marker was %s and the remaining dependency count was %s)\n",
task_done[task], deps_left[task])
+ for tid in self.runtaskentries:
+ if task_done[tid] is False or deps_left[tid] != 0:
+ problem_tasks.append(tid)
+ logger.debug(2, "Task %s is not buildable", tid)
+ logger.debug(2, "(Complete marker was %s and the remaining dependency count was %s)\n",
task_done[tid], deps_left[tid])
+ self.runtaskentries[tid].weight = weight[tid]
if problem_tasks:
- message = "Unbuildable tasks were found.\n"
+ message = "%s unbuildable tasks were found.\n" % len(problem_tasks)
message = message + "These are usually caused by circular dependencies and any circular
dependency chains found will be printed below. Increase the debug level to see a list of unbuildable
tasks.\n\n"
message = message + "Identifying dependency loops (this may take a short while)...\n"
logger.error(message)
@@ -422,14 +420,14 @@ class RunQueueData:
to optimise the execution order.
"""
- runq_build = []
+ runq_build = {}
recursivetasks = {}
recursiveitasks = {}
recursivetasksselfref = set()
taskData = self.taskData
- if len(taskData.tasks_name) == 0:
+ if len(taskData.taskentries) == 0:
# Nothing to do
return 0
@@ -447,160 +445,159 @@ class RunQueueData:
# rdeptast, recrdeptask, idepends).
def add_build_dependencies(depids, tasknames, depends):
- for depid in depids:
+ for depname in depids:
# Won't be in build_targets if ASSUME_PROVIDED
- if depid not in taskData.build_targets:
+ if depname not in taskData.build_targets or not taskData.build_targets[depname]:
continue
- depdata = taskData.build_targets[depid][0]
+ depdata = taskData.build_targets[depname][0]
if depdata is None:
continue
for taskname in tasknames:
- taskid = taskData.gettask_id_fromfnid(depdata, taskname)
- if taskid is not None:
- depends.add(taskid)
+ t = depdata + ":" + taskname
+ if t in taskData.taskentries:
+ depends.add(t)
def add_runtime_dependencies(depids, tasknames, depends):
- for depid in depids:
- if depid not in taskData.run_targets:
+ for depname in depids:
+ if depname not in taskData.run_targets or not taskData.run_targets[depname]:
continue
- depdata = taskData.run_targets[depid][0]
+ depdata = taskData.run_targets[depname][0]
if depdata is None:
continue
for taskname in tasknames:
- taskid = taskData.gettask_id_fromfnid(depdata, taskname)
- if taskid is not None:
- depends.add(taskid)
+ t = depdata + ":" + taskname
+ if t in taskData.taskentries:
+ depends.add(t)
- def add_resolved_dependencies(depids, tasknames, depends):
- for depid in depids:
- for taskname in tasknames:
- taskid = taskData.gettask_id_fromfnid(depid, taskname)
- if taskid is not None:
- depends.add(taskid)
+ def add_resolved_dependencies(fn, tasknames, depends):
+ for taskname in tasknames:
+ tid = fn + ":" + taskname
+ if tid in self.runtaskentries:
+ depends.add(tid)
+
+ for tid in taskData.taskentries:
+
+ fn = fn_from_tid(tid)
+ taskname = taskname_from_tid(tid)
- for task in range(len(taskData.tasks_name)):
depends = set()
- fnid = taskData.tasks_fnid[task]
- fn = taskData.fn_index[fnid]
task_deps = self.dataCache.task_deps[fn]
- #logger.debug(2, "Processing %s:%s", fn, taskData.tasks_name[task])
+ self.runtaskentries[tid] = RunTaskEntry()
- if fnid not in taskData.failed_fnids:
+ #logger.debug(2, "Processing %s:%s", fn, taskname)
+
+ if fn not in taskData.failed_fns:
# Resolve task internal dependencies
#
# e.g. addtask before X after Y
- depends = set(taskData.tasks_tdepends[task])
+ depends.update(taskData.taskentries[tid].tdepends)
# Resolve 'deptask' dependencies
#
# e.g. do_sometask[deptask] = "do_someothertask"
# (makes sure sometask runs after someothertask of all DEPENDS)
- if 'deptask' in task_deps and taskData.tasks_name[task] in task_deps['deptask']:
- tasknames = task_deps['deptask'][taskData.tasks_name[task]].split()
- add_build_dependencies(taskData.depids[fnid], tasknames, depends)
+ if 'deptask' in task_deps and taskname in task_deps['deptask']:
+ tasknames = task_deps['deptask'][taskname].split()
+ add_build_dependencies(taskData.depids[fn], tasknames, depends)
# Resolve 'rdeptask' dependencies
#
# e.g. do_sometask[rdeptask] = "do_someothertask"
# (makes sure sometask runs after someothertask of all RDEPENDS)
- if 'rdeptask' in task_deps and taskData.tasks_name[task] in task_deps['rdeptask']:
- tasknames = task_deps['rdeptask'][taskData.tasks_name[task]].split()
- add_runtime_dependencies(taskData.rdepids[fnid], tasknames, depends)
+ if 'rdeptask' in task_deps and taskname in task_deps['rdeptask']:
+ tasknames = task_deps['rdeptask'][taskname].split()
+ add_runtime_dependencies(taskData.rdepids[fn], tasknames, depends)
# Resolve inter-task dependencies
#
# e.g. do_sometask[depends] = "targetname:do_someothertask"
# (makes sure sometask runs after targetname's someothertask)
- idepends = taskData.tasks_idepends[task]
- for (depid, idependtask) in idepends:
- if depid in taskData.build_targets and not depid in taskData.failed_deps:
+ idepends = taskData.taskentries[tid].idepends
+ for (depname, idependtask) in idepends:
+ if depname in taskData.build_targets and taskData.build_targets[depname] and not depname
in taskData.failed_deps:
# Won't be in build_targets if ASSUME_PROVIDED
- depdata = taskData.build_targets[depid][0]
+ depdata = taskData.build_targets[depname][0]
if depdata is not None:
- taskid = taskData.gettask_id_fromfnid(depdata, idependtask)
- if taskid is None:
- bb.msg.fatal("RunQueue", "Task %s in %s depends upon non-existent task %s in
%s" % (taskData.tasks_name[task], fn, idependtask, taskData.fn_index[depdata]))
- depends.add(taskid)
- irdepends = taskData.tasks_irdepends[task]
- for (depid, idependtask) in irdepends:
- if depid in taskData.run_targets:
+ t = depdata + ":" + idependtask
+ depends.add(t)
+ if t not in taskData.taskentries:
+ bb.msg.fatal("RunQueue", "Task %s in %s depends upon non-existent task %s in
%s" % (taskname, fn, idependtask, depdata))
+ irdepends = taskData.taskentries[tid].irdepends
+ for (depname, idependtask) in irdepends:
+ if depname in taskData.run_targets:
# Won't be in run_targets if ASSUME_PROVIDED
- depdata = taskData.run_targets[depid][0]
+ depdata = taskData.run_targets[depname][0]
if depdata is not None:
- taskid = taskData.gettask_id_fromfnid(depdata, idependtask)
- if taskid is None:
- bb.msg.fatal("RunQueue", "Task %s in %s rdepends upon non-existent task %s
in %s" % (taskData.tasks_name[task], fn, idependtask, taskData.fn_index[depdata]))
- depends.add(taskid)
+ t = depdata + ":" + idependtask
+ depends.add(t)
+ if t not in taskData.taskentries:
+ bb.msg.fatal("RunQueue", "Task %s in %s rdepends upon non-existent task %s
in %s" % (taskname, fn, idependtask, depdata))
# Resolve recursive 'recrdeptask' dependencies (Part A)
#
# e.g. do_sometask[recrdeptask] = "do_someothertask"
# (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask
dependencies, recursively)
# We cover the recursive part of the dependencies below
- if 'recrdeptask' in task_deps and taskData.tasks_name[task] in task_deps['recrdeptask']:
- tasknames = task_deps['recrdeptask'][taskData.tasks_name[task]].split()
- recursivetasks[task] = tasknames
- add_build_dependencies(taskData.depids[fnid], tasknames, depends)
- add_runtime_dependencies(taskData.rdepids[fnid], tasknames, depends)
- if taskData.tasks_name[task] in tasknames:
- recursivetasksselfref.add(task)
-
- if 'recideptask' in task_deps and taskData.tasks_name[task] in task_deps['recideptask']:
- recursiveitasks[task] = []
- for t in task_deps['recideptask'][taskData.tasks_name[task]].split():
- newdep = taskData.gettask_id_fromfnid(fnid, t)
- recursiveitasks[task].append(newdep)
-
- self.runq_fnid.append(taskData.tasks_fnid[task])
- self.runq_task.append(taskData.tasks_name[task])
- self.runq_depends.append(depends)
- self.runq_revdeps.append(set())
- self.runq_hash.append("")
-
- runq_build.append(0)
+ if 'recrdeptask' in task_deps and taskname in task_deps['recrdeptask']:
+ tasknames = task_deps['recrdeptask'][taskname].split()
+ recursivetasks[tid] = tasknames
+ add_build_dependencies(taskData.depids[fn], tasknames, depends)
+ add_runtime_dependencies(taskData.rdepids[fn], tasknames, depends)
+ if taskname in tasknames:
+ recursivetasksselfref.add(tid)
+
+ if 'recideptask' in task_deps and taskname in task_deps['recideptask']:
+ recursiveitasks[tid] = []
+ for t in task_deps['recideptask'][taskname].split():
+ newdep = fn + ":" + t
+ recursiveitasks[tid].append(newdep)
+
+ self.runtaskentries[tid].depends = depends
# Resolve recursive 'recrdeptask' dependencies (Part B)
#
# e.g. do_sometask[recrdeptask] = "do_someothertask"
# (makes sure sometask runs after someothertask of all DEPENDS, RDEPENDS and intertask dependencies,
recursively)
- # We need to do this separately since we need all of self.runq_depends to be complete before this is
processed
+ # We need to do this separately since we need all of runtaskentries[*].depends to be complete before
this is processed
extradeps = {}
- for task in recursivetasks:
- extradeps[task] = set(self.runq_depends[task])
- tasknames = recursivetasks[task]
+ for tid in recursivetasks:
+ extradeps[tid] = set(self.runtaskentries[tid].depends)
+
+ tasknames = recursivetasks[tid]
seendeps = set()
- seenfnid = []
def generate_recdeps(t):
newdeps = set()
- add_resolved_dependencies([taskData.tasks_fnid[t]], tasknames, newdeps)
- extradeps[task].update(newdeps)
+ add_resolved_dependencies(fn_from_tid(t), tasknames, newdeps)
+ extradeps[tid].update(newdeps)
seendeps.add(t)
newdeps.add(t)
for i in newdeps:
- for n in self.runq_depends[i]:
+ task = self.runtaskentries[i].task
+ for n in self.runtaskentries[i].depends:
if n not in seendeps:
- generate_recdeps(n)
- generate_recdeps(task)
+ generate_recdeps(n)
+ generate_recdeps(tid)
- if task in recursiveitasks:
- for dep in recursiveitasks[task]:
+ if tid in recursiveitasks:
+ for dep in recursiveitasks[tid]:
generate_recdeps(dep)
# Remove circular references so that do_a[recrdeptask] = "do_a do_b" can work
- for task in recursivetasks:
- extradeps[task].difference_update(recursivetasksselfref)
+ for tid in recursivetasks:
+ extradeps[tid].difference_update(recursivetasksselfref)
- for task in range(len(taskData.tasks_name)):
+ for tid in self.runtaskentries:
+ task = self.runtaskentries[tid].task
# Add in extra dependencies
- if task in extradeps:
- self.runq_depends[task] = extradeps[task]
+ if tid in extradeps:
+ self.runtaskentries[tid].depends = extradeps[tid]
# Remove all self references
- if task in self.runq_depends[task]:
- logger.debug(2, "Task %s (%s %s) contains self reference! %s", task,
taskData.fn_index[taskData.tasks_fnid[task]], taskData.tasks_name[task], self.runq_depends[task])
- self.runq_depends[task].remove(task)
+ if tid in self.runtaskentries[tid].depends:
+ logger.debug(2, "Task %s contains self reference!", tid)
+ self.runtaskentries[tid].depends.remove(tid)
# Step B - Mark all active tasks
#
@@ -610,33 +607,30 @@ class RunQueueData:
logger.verbose("Marking Active Tasks")
- def mark_active(listid, depth):
+ def mark_active(tid, depth):
"""
Mark an item as active along with its depends
(calls itself recursively)
"""
- if runq_build[listid] == 1:
+ if tid in runq_build:
return
- runq_build[listid] = 1
+ runq_build[tid] = 1
- depends = self.runq_depends[listid]
+ depends = self.runtaskentries[tid].depends
for depend in depends:
mark_active(depend, depth+1)
self.target_pairs = []
for target in self.targets:
- targetid = taskData.getbuild_id(target[0])
-
- if targetid not in taskData.build_targets:
+ if target[0] not in taskData.build_targets or not taskData.build_targets[target[0]]:
continue
- if targetid in taskData.failed_deps:
+ if target[0] in taskData.failed_deps:
continue
- fnid = taskData.build_targets[targetid][0]
- fn = taskData.fn_index[fnid]
+ fn = taskData.build_targets[target[0]][0]
task = target[1]
parents = False
if task.endswith('-'):
@@ -645,87 +639,72 @@ class RunQueueData:
self.target_pairs.append((fn, task))
- if fnid in taskData.failed_fnids:
+ if fn in taskData.failed_fns:
continue
- if task not in taskData.tasks_lookup[fnid]:
+ tid = fn + ":" + task
+ if tid not in taskData.taskentries:
import difflib
- close_matches = difflib.get_close_matches(task, taskData.tasks_lookup[fnid], cutoff=0.7)
+ tasks = []
+ for x in taskData.taskentries:
+ if x.startswith(fn + ":"):
+ tasks.append(taskname_from_tid(x))
+ close_matches = difflib.get_close_matches(task, tasks, cutoff=0.7)
if close_matches:
extra = ". Close matches:\n %s" % "\n ".join(close_matches)
else:
extra = ""
bb.msg.fatal("RunQueue", "Task %s does not exist for target %s%s" % (task, target[0], extra))
-
+
# For tasks called "XXXX-", ony run their dependencies
- listid = taskData.tasks_lookup[fnid][task]
if parents:
- for i in self.runq_depends[listid]:
+ for i in self.runtaskentries[tid].depends:
mark_active(i, 1)
else:
- mark_active(listid, 1)
+ mark_active(tid, 1)
# Step C - Prune all inactive tasks
#
# Once all active tasks are marked, prune the ones we don't need.
- maps = []
delcount = 0
- for listid in range(len(self.runq_fnid)):
- if runq_build[listid-delcount] == 1:
- maps.append(listid-delcount)
- else:
- del self.runq_fnid[listid-delcount]
- del self.runq_task[listid-delcount]
- del self.runq_depends[listid-delcount]
- del runq_build[listid-delcount]
- del self.runq_revdeps[listid-delcount]
- del self.runq_hash[listid-delcount]
- delcount = delcount + 1
- maps.append(-1)
+ for tid in list(self.runtaskentries.keys()):
+ if tid not in runq_build:
+ del self.runtaskentries[tid]
+ delcount += 1
#
# Step D - Sanity checks and computation
#
# Check to make sure we still have tasks to run
- if len(self.runq_fnid) == 0:
+ if len(self.runtaskentries) == 0:
if not taskData.abort:
bb.msg.fatal("RunQueue", "All buildable tasks have been run but the build is incomplete
(--continue mode). Errors for the tasks that failed will have been printed above.")
else:
bb.msg.fatal("RunQueue", "No active tasks and not in --continue mode?! Please report this
bug.")
- logger.verbose("Pruned %s inactive tasks, %s left", delcount, len(self.runq_fnid))
-
- # Remap the dependencies to account for the deleted tasks
- # Check we didn't delete a task we depend on
- for listid in range(len(self.runq_fnid)):
- newdeps = []
- origdeps = self.runq_depends[listid]
- for origdep in origdeps:
- if maps[origdep] == -1:
- bb.msg.fatal("RunQueue", "Invalid mapping - Should never happen!")
- newdeps.append(maps[origdep])
- self.runq_depends[listid] = set(newdeps)
+ logger.verbose("Pruned %s inactive tasks, %s left", delcount, len(self.runtaskentries))
logger.verbose("Assign Weightings")
# Generate a list of reverse dependencies to ease future calculations
- for listid in range(len(self.runq_fnid)):
- for dep in self.runq_depends[listid]:
- self.runq_revdeps[dep].add(listid)
+ for tid in self.runtaskentries:
+ for dep in self.runtaskentries[tid].depends:
+ self.runtaskentries[dep].revdeps.add(tid)
# Identify tasks at the end of dependency chains
# Error on circular dependency loops (length two)
endpoints = []
- for listid in range(len(self.runq_fnid)):
- revdeps = self.runq_revdeps[listid]
+ for tid in self.runtaskentries:
+ revdeps = self.runtaskentries[tid].revdeps
if len(revdeps) == 0:
- endpoints.append(listid)
+ endpoints.append(tid)
for dep in revdeps:
- if dep in self.runq_depends[listid]:
+ if dep in self.runtaskentries[tid].depends:
#self.dump_data(taskData)
- bb.msg.fatal("RunQueue", "Task %s (%s) has circular dependency on %s (%s)" %
(taskData.fn_index[self.runq_fnid[dep]], self.runq_task[dep], taskData.fn_index[self.runq_fnid[listid]],
self.runq_task[listid]))
+ bb.msg.fatal("RunQueue", "Task %s has circular dependency on %s" % (tid, dep))
+
logger.verbose("Compute totals (have %s endpoint(s))", len(endpoints))
@@ -736,8 +715,8 @@ class RunQueueData:
# Sanity Check - Check for multiple tasks building the same provider
prov_list = {}
seen_fn = []
- for task in range(len(self.runq_fnid)):
- fn = taskData.fn_index[self.runq_fnid[task]]
+ for tid in self.runtaskentries:
+ fn = fn_from_tid(tid)
if fn in seen_fn:
continue
seen_fn.append(fn)
@@ -766,15 +745,15 @@ class RunQueueData:
commondeps = None
for provfn in prov_list[prov]:
deps = set()
- for task, fnid in enumerate(self.runq_fnid):
- fn = taskData.fn_index[fnid]
+ for tid in self.runtaskentries:
+ fn = fn_from_tid(tid)
if fn != provfn:
continue
- for dep in self.runq_revdeps[task]:
- fn = taskData.fn_index[self.runq_fnid[dep]]
+ for dep in self.runtaskentries[tid].revdeps:
+ fn = fn_from_tid(dep)
if fn == provfn:
continue
- deps.add(self.get_short_user_idstring(dep))
+ deps.add(dep)
if not commondeps:
commondeps = set(deps)
else:
@@ -827,27 +806,26 @@ class RunQueueData:
# Create a whitelist usable by the stamp checks
stampfnwhitelist = []
for entry in self.stampwhitelist.split():
- entryid = self.taskData.getbuild_id(entry)
- if entryid not in self.taskData.build_targets:
+ if entry not in self.taskData.build_targets:
continue
- fnid = self.taskData.build_targets[entryid][0]
- fn = self.taskData.fn_index[fnid]
+ fn = self.taskData.build_targets[entry][0]
stampfnwhitelist.append(fn)
self.stampfnwhitelist = stampfnwhitelist
# Iterate over the task list looking for tasks with a 'setscene' function
- self.runq_setscene = []
+ self.runq_setscene_tids = []
if not self.cooker.configuration.nosetscene:
- for task in range(len(self.runq_fnid)):
- setscene = taskData.gettask_id(self.taskData.fn_index[self.runq_fnid[task]],
self.runq_task[task] + "_setscene", False)
- if not setscene:
+ for tid in self.runtaskentries:
+ setscenetid = tid + "_setscene"
+ if setscenetid not in taskData.taskentries:
continue
- self.runq_setscene.append(task)
+ task = self.runtaskentries[tid].task
+ self.runq_setscene_tids.append(tid)
def invalidate_task(fn, taskname, error_nostamp):
taskdep = self.dataCache.task_deps[fn]
- fnid = self.taskData.getfn_id(fn)
- if taskname not in taskData.tasks_lookup[fnid]:
+ tid = fn + ":" + taskname
+ if tid not in taskData.taskentries:
logger.warning("Task %s does not exist, invalidating this task will have no effect" %
taskname)
if 'nostamp' in taskdep and taskname in taskdep['nostamp']:
if error_nostamp:
@@ -882,43 +860,38 @@ class RunQueueData:
# Iterate over the task list and call into the siggen code
dealtwith = set()
- todeal = set(range(len(self.runq_fnid)))
+ todeal = set(self.runtaskentries)
while len(todeal) > 0:
- for task in todeal.copy():
- if len(self.runq_depends[task] - dealtwith) == 0:
- dealtwith.add(task)
- todeal.remove(task)
+ for tid in todeal.copy():
+ if len(self.runtaskentries[tid].depends - dealtwith) == 0:
+ dealtwith.add(tid)
+ todeal.remove(tid)
procdep = []
- for dep in self.runq_depends[task]:
- procdep.append(self.taskData.fn_index[self.runq_fnid[dep]] + "." +
self.runq_task[dep])
- self.runq_hash[task] =
bb.parse.siggen.get_taskhash(self.taskData.fn_index[self.runq_fnid[task]], self.runq_task[task], procdep,
self.dataCache)
+ for dep in self.runtaskentries[tid].depends:
+ procdep.append(fn_from_tid(dep) + "." + taskname_from_tid(dep))
+ self.runtaskentries[tid].hash = bb.parse.siggen.get_taskhash(fn_from_tid(tid),
taskname_from_tid(tid), procdep, self.dataCache)
+ task = self.runtaskentries[tid].task
bb.parse.siggen.writeout_file_checksum_cache()
- return len(self.runq_fnid)
+ return len(self.runtaskentries)
def dump_data(self, taskQueue):
"""
Dump some debug information on the internal data structures
"""
logger.debug(3, "run_tasks:")
- for task in range(len(self.rqdata.runq_task)):
- logger.debug(3, " (%s)%s - %s: %s Deps %s RevDeps %s", task,
- taskQueue.fn_index[self.rqdata.runq_fnid[task]],
- self.rqdata.runq_task[task],
- self.rqdata.runq_weight[task],
- self.rqdata.runq_depends[task],
- self.rqdata.runq_revdeps[task])
+ for tid in self.runtaskentries:
+ logger.debug(3, " %s: %s Deps %s RevDeps %s", tid,
+ self.runtaskentries[tid].weight,
+ self.runtaskentries[tid].depends,
+ self.runtaskentries[tid].revdeps)
logger.debug(3, "sorted_tasks:")
- for task1 in range(len(self.rqdata.runq_task)):
- if task1 in self.prio_map:
- task = self.prio_map[task1]
- logger.debug(3, " (%s)%s - %s: %s Deps %s RevDeps %s", task,
- taskQueue.fn_index[self.rqdata.runq_fnid[task]],
- self.rqdata.runq_task[task],
- self.rqdata.runq_weight[task],
- self.rqdata.runq_depends[task],
- self.rqdata.runq_revdeps[task])
+ for tid in self.prio_map:
+ logger.debug(3, " %s: %s Deps %s RevDeps %s", tid,
+ self.runtaskentries[tid].weight,
+ self.runtaskentries[tid].depends,
+ self.runtaskentries[tid].revdeps)
class RunQueue:
def __init__(self, cooker, cfgData, dataCache, taskData, targets):
@@ -929,7 +902,7 @@ class RunQueue:
self.stamppolicy = cfgData.getVar("BB_STAMP_POLICY", True) or "perfile"
self.hashvalidate = cfgData.getVar("BB_HASHCHECK_FUNCTION", True) or None
- self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION", True) or None
+ self.setsceneverify = cfgData.getVar("BB_SETSCENE_VERIFY_FUNCTION2", True) or None
self.depvalidate = cfgData.getVar("BB_SETSCENE_DEPVALID", True) or None
self.state = runQueuePrepare
@@ -961,13 +934,17 @@ class RunQueue:
bb.utils.nonblockingfd(worker.stdout)
workerpipe = runQueuePipe(worker.stdout, None, self.cfgData, self, rqexec)
+ runqhash = {}
+ for tid in self.rqdata.runtaskentries:
+ runqhash[tid] = self.rqdata.runtaskentries[tid].hash
+
workerdata = {
"taskdeps" : self.rqdata.dataCache.task_deps,
"fakerootenv" : self.rqdata.dataCache.fakerootenv,
"fakerootdirs" : self.rqdata.dataCache.fakerootdirs,
"fakerootnoenv" : self.rqdata.dataCache.fakerootnoenv,
"sigdata" : bb.parse.siggen.get_taskdata(),
- "runq_hash" : self.rqdata.runq_hash,
+ "runq_hash" : runqhash,
"logdefaultdebug" : bb.msg.loggerDefaultDebugLevel,
"logdefaultverbose" : bb.msg.loggerDefaultVerbose,
"logdefaultverboselogs" : bb.msg.loggerVerboseLogs,
@@ -1033,7 +1010,7 @@ class RunQueue:
fds.append(self.fakeworkerpipe.input)
return fds
- def check_stamp_task(self, task, taskname = None, recurse = False, cache = None):
+ def check_stamp_task(self, tid, taskname = None, recurse = False, cache = None):
def get_timestamp(f):
try:
if not os.access(f, os.F_OK):
@@ -1050,9 +1027,9 @@ class RunQueue:
if self.stamppolicy == "whitelist":
stampwhitelist = self.rqdata.stampfnwhitelist
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]]
+ fn = fn_from_tid(tid)
if taskname is None:
- taskname = self.rqdata.runq_task[task]
+ taskname = taskname_from_tid(tid)
stampfile = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
@@ -1074,10 +1051,10 @@ class RunQueue:
iscurrent = True
t1 = get_timestamp(stampfile)
- for dep in self.rqdata.runq_depends[task]:
+ for dep in self.rqdata.runtaskentries[tid].depends:
if iscurrent:
- fn2 = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[dep]]
- taskname2 = self.rqdata.runq_task[dep]
+ fn2 = fn_from_tid(dep)
+ taskname2 = taskname_from_tid(dep)
stampfile2 = bb.build.stampfile(taskname2, self.rqdata.dataCache, fn2)
stampfile3 = bb.build.stampfile(taskname2 + "_setscene", self.rqdata.dataCache, fn2)
t2 = get_timestamp(stampfile2)
@@ -1104,7 +1081,7 @@ class RunQueue:
iscurrent = self.check_stamp_task(dep, recurse=True, cache=cache)
cache[dep] = iscurrent
if recurse:
- cache[task] = iscurrent
+ cache[tid] = iscurrent
return iscurrent
def _execute_runqueue(self):
@@ -1171,9 +1148,9 @@ class RunQueue:
if self.state is runQueueFailed:
if not self.rqdata.taskData.tryaltconfigs:
- raise bb.runqueue.TaskFailure(self.rqexe.failed_fnids)
- for fnid in self.rqexe.failed_fnids:
- self.rqdata.taskData.fail_fnid(fnid)
+ raise bb.runqueue.TaskFailure(self.rqexe.failed_fns)
+ for fn in self.rqexe.failed_fns:
+ self.rqdata.taskData.fail_fn(fn)
self.rqdata.reset()
if self.state is runQueueComplete:
@@ -1220,11 +1197,11 @@ class RunQueue:
def dump_signatures(self, options):
done = set()
bb.note("Reparsing files to collect dependency data")
- for task in range(len(self.rqdata.runq_fnid)):
- if self.rqdata.runq_fnid[task] not in done:
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]]
+ for tid in self.rqdata.runtaskentries:
+ fn = fn_from_tid(tid)
+ if fn not in done:
the_data = bb.cache.Cache.loadDataFull(fn, self.cooker.collection.get_file_appends(fn),
self.cooker.data)
- done.add(self.rqdata.runq_fnid[task])
+ done.add(fn)
bb.parse.siggen.dump_sigs(self.rqdata.dataCache, options)
@@ -1242,20 +1219,20 @@ class RunQueue:
stamppresent = []
valid_new = set()
- for task in range(len(self.rqdata.runq_fnid)):
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]]
- taskname = self.rqdata.runq_task[task]
+ for tid in self.rqdata.runtaskentries:
+ fn = fn_from_tid(tid)
+ taskname = taskname_from_tid(tid)
taskdep = self.rqdata.dataCache.task_deps[fn]
if 'noexec' in taskdep and taskname in taskdep['noexec']:
- noexec.append(task)
+ noexec.append(tid)
continue
sq_fn.append(fn)
sq_hashfn.append(self.rqdata.dataCache.hashfn[fn])
- sq_hash.append(self.rqdata.runq_hash[task])
+ sq_hash.append(self.rqdata.runtaskentries[tid].hash)
sq_taskname.append(taskname)
- sq_task.append(task)
+ sq_task.append(tid)
locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn, "d"
: self.cooker.expanded_data }
try:
call = self.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d, siginfo=True)"
@@ -1270,13 +1247,13 @@ class RunQueue:
# Tasks which are both setscene and noexec never care about dependencies
# We therefore find tasks which are setscene and noexec and mark their
# unique dependencies as valid.
- for task in noexec:
- if task not in self.rqdata.runq_setscene:
+ for tid in noexec:
+ if tid not in self.rqdata.runq_setscene_tids:
continue
- for dep in self.rqdata.runq_depends[task]:
+ for dep in self.rqdata.runtaskentries[tid].depends:
hasnoexecparents = True
- for dep2 in self.rqdata.runq_revdeps[dep]:
- if dep2 in self.rqdata.runq_setscene and dep2 in noexec:
+ for dep2 in self.rqdata.runtaskentries[dep].revdeps:
+ if dep2 in self.rqdata.runq_setscene_tids and dep2 in noexec:
continue
hasnoexecparents = False
break
@@ -1284,30 +1261,30 @@ class RunQueue:
valid_new.add(dep)
invalidtasks = set()
- for task in range(len(self.rqdata.runq_fnid)):
- if task not in valid_new and task not in noexec:
- invalidtasks.add(task)
+ for tid in self.rqdata.runtaskentries:
+ if tid not in valid_new and tid not in noexec:
+ invalidtasks.add(tid)
found = set()
processed = set()
- for task in invalidtasks:
- toprocess = set([task])
+ for tid in invalidtasks:
+ toprocess = set([tid])
while toprocess:
next = set()
for t in toprocess:
- for dep in self.rqdata.runq_depends[t]:
+ for dep in self.rqdata.runtaskentries[t].depends:
if dep in invalidtasks:
- found.add(task)
+ found.add(tid)
if dep not in processed:
processed.add(dep)
next.add(dep)
toprocess = next
- if task in found:
+ if tid in found:
toprocess = set()
tasklist = []
- for task in invalidtasks.difference(found):
- tasklist.append(self.rqdata.get_user_idstring(task))
+ for tid in invalidtasks.difference(found):
+ tasklist.append(tid)
if tasklist:
bb.plain("The differences between the current build and any cached tasks start at the following
tasks:\n" + "\n".join(tasklist))
@@ -1331,11 +1308,11 @@ class RunQueue:
return recout
- for task in invalidtasks:
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]]
+ for tid in invalidtasks:
+ fn = fn_from_tid(tid)
pn = self.rqdata.dataCache.pkg_fn[fn]
- taskname = self.rqdata.runq_task[task]
- h = self.rqdata.runq_hash[task]
+ taskname = taskname_from_tid(tid)
+ h = self.rqdata.runtaskentries[tid].hash
matches = bb.siggen.find_siginfo(pn, taskname, [], self.cfgData)
match = None
for m in matches:
@@ -1367,7 +1344,7 @@ class RunQueueExecute:
self.build_stamps = {}
self.build_stamps2 = []
- self.failed_fnids = []
+ self.failed_fns = []
self.stampcache = {}
@@ -1401,7 +1378,7 @@ class RunQueueExecute:
except IOError:
# worker must have died?
pass
- if len(self.failed_fnids) != 0:
+ if len(self.failed_fns) != 0:
self.rq.state = runQueueFailed
return
@@ -1416,7 +1393,7 @@ class RunQueueExecute:
self.rq.read_workers()
return self.rq.active_fds()
- if len(self.failed_fnids) != 0:
+ if len(self.failed_fns) != 0:
self.rq.state = runQueueFailed
return True
@@ -1430,13 +1407,9 @@ class RunQueueExecute:
taskdata = {}
taskdeps.add(task)
for dep in taskdeps:
- if setscene:
- depid = self.rqdata.runq_setscene[dep]
- else:
- depid = dep
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[depid]]
+ fn = fn_from_tid(dep)
pn = self.rqdata.dataCache.pkg_fn[fn]
- taskname = self.rqdata.runq_task[depid]
+ taskname = taskname_from_tid(dep)
taskdata[dep] = [pn, taskname, fn]
call = self.rq.depvalidate + "(task, taskdata, notneeded, d)"
locs = { "task" : task, "taskdata" : taskdata, "notneeded" : self.scenequeue_notneeded, "d" :
self.cooker.expanded_data }
@@ -1456,30 +1429,30 @@ class RunQueueExecuteTasks(RunQueueExecute):
def __init__(self, rq):
RunQueueExecute.__init__(self, rq)
- self.stats = RunQueueStats(len(self.rqdata.runq_fnid))
+ self.stats = RunQueueStats(len(self.rqdata.runtaskentries))
self.stampcache = {}
initial_covered = self.rq.scenequeue_covered.copy()
# Mark initial buildable tasks
- for task in range(self.stats.total):
- if len(self.rqdata.runq_depends[task]) == 0:
- self.runq_buildable.add(task)
- if len(self.rqdata.runq_revdeps[task]) > 0 and
self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered):
- self.rq.scenequeue_covered.add(task)
+ for tid in self.rqdata.runtaskentries:
+ if len(self.rqdata.runtaskentries[tid].depends) == 0:
+ self.runq_buildable.add(tid)
+ if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and
self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered):
+ self.rq.scenequeue_covered.add(tid)
found = True
while found:
found = False
- for task in range(self.stats.total):
- if task in self.rq.scenequeue_covered:
+ for tid in self.rqdata.runtaskentries:
+ if tid in self.rq.scenequeue_covered:
continue
- logger.debug(1, 'Considering %s (%s): %s' % (task, self.rqdata.get_user_idstring(task),
str(self.rqdata.runq_revdeps[task])))
+ logger.debug(1, 'Considering %s: %s' % (tid, str(self.rqdata.runtaskentries[tid].revdeps)))
- if len(self.rqdata.runq_revdeps[task]) > 0 and
self.rqdata.runq_revdeps[task].issubset(self.rq.scenequeue_covered):
+ if len(self.rqdata.runtaskentries[tid].revdeps) > 0 and
self.rqdata.runtaskentries[tid].revdeps.issubset(self.rq.scenequeue_covered):
found = True
- self.rq.scenequeue_covered.add(task)
+ self.rq.scenequeue_covered.add(tid)
logger.debug(1, 'Skip list (pre setsceneverify) %s', sorted(self.rq.scenequeue_covered))
@@ -1487,35 +1460,33 @@ class RunQueueExecuteTasks(RunQueueExecute):
covered_remove = set()
if self.rq.setsceneverify:
invalidtasks = []
- for task in range(len(self.rqdata.runq_task)):
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]]
- taskname = self.rqdata.runq_task[task]
+ tasknames = {}
+ fns = {}
+ for tid in self.rqdata.runtaskentries:
+ fn = fn_from_tid(tid)
+ taskname = taskname_from_tid(tid)
taskdep = self.rqdata.dataCache.task_deps[fn]
-
+ fns[tid] = fn
+ tasknames[tid] = taskname
if 'noexec' in taskdep and taskname in taskdep['noexec']:
continue
- if self.rq.check_stamp_task(task, taskname + "_setscene", cache=self.stampcache):
- logger.debug(2, 'Setscene stamp current for task %s(%s)', task,
self.rqdata.get_user_idstring(task))
+ if self.rq.check_stamp_task(tid, taskname + "_setscene", cache=self.stampcache):
+ logger.debug(2, 'Setscene stamp current for task %s', tid)
continue
- if self.rq.check_stamp_task(task, taskname, recurse = True, cache=self.stampcache):
- logger.debug(2, 'Normal stamp current for task %s(%s)', task,
self.rqdata.get_user_idstring(task))
+ if self.rq.check_stamp_task(tid, taskname, recurse = True, cache=self.stampcache):
+ logger.debug(2, 'Normal stamp current for task %s', tid)
continue
- invalidtasks.append(task)
+ invalidtasks.append(tid)
- call = self.rq.setsceneverify + "(covered, tasknames, fnids, fns, d, invalidtasks=invalidtasks)"
- call2 = self.rq.setsceneverify + "(covered, tasknames, fnids, fns, d)"
- locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : self.rqdata.runq_task, "fnids" :
self.rqdata.runq_fnid, "fns" : self.rqdata.taskData.fn_index, "d" : self.cooker.expanded_data, "invalidtasks"
: invalidtasks }
- # Backwards compatibility with older versions without invalidtasks
- try:
- covered_remove = bb.utils.better_eval(call, locs)
- except TypeError:
- covered_remove = bb.utils.better_eval(call2, locs)
+ call = self.rq.setsceneverify + "(covered, tasknames, fns, d, invalidtasks=invalidtasks)"
+ locs = { "covered" : self.rq.scenequeue_covered, "tasknames" : tasknames, "fns" : fns, "d" :
self.cooker.expanded_data, "invalidtasks" : invalidtasks }
+ covered_remove = bb.utils.better_eval(call, locs)
- def removecoveredtask(task):
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]]
- taskname = self.rqdata.runq_task[task] + '_setscene'
+ def removecoveredtask(tid):
+ fn = fn_from_tid(tid)
+ taskname = taskname_from_tid(tid) + '_setscene'
bb.build.del_stamp(taskname, self.rqdata.dataCache, fn)
- self.rq.scenequeue_covered.remove(task)
+ self.rq.scenequeue_covered.remove(tid)
toremove = covered_remove
for task in toremove:
@@ -1524,7 +1495,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
covered_remove = []
for task in toremove:
removecoveredtask(task)
- for deptask in self.rqdata.runq_depends[task]:
+ for deptask in self.rqdata.runtaskentries[task].depends:
if deptask not in self.rq.scenequeue_covered:
continue
if deptask in toremove or deptask in covered_remove or deptask in initial_covered:
@@ -1580,20 +1551,20 @@ class RunQueueExecuteTasks(RunQueueExecute):
completed dependencies as buildable
"""
self.runq_complete.add(task)
- for revdep in self.rqdata.runq_revdeps[task]:
+ for revdep in self.rqdata.runtaskentries[task].revdeps:
if revdep in self.runq_running:
continue
if revdep in self.runq_buildable:
continue
alldeps = 1
- for dep in self.rqdata.runq_depends[revdep]:
+ for dep in self.rqdata.runtaskentries[revdep].depends:
if dep not in self.runq_complete:
alldeps = 0
if alldeps == 1:
self.setbuildable(revdep)
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[revdep]]
- taskname = self.rqdata.runq_task[revdep]
- logger.debug(1, "Marking task %s (%s, %s) as buildable", revdep, fn, taskname)
+ fn = fn_from_tid(revdep)
+ taskname = taskname_from_tid(revdep)
+ logger.debug(1, "Marking task %s as buildable", revdep)
def task_complete(self, task):
self.stats.taskCompleted()
@@ -1606,8 +1577,8 @@ class RunQueueExecuteTasks(RunQueueExecute):
Updates the state engine with the failure
"""
self.stats.taskFailed()
- fnid = self.rqdata.runq_fnid[task]
- self.failed_fnids.append(fnid)
+ fn = fn_from_tid(task)
+ self.failed_fns.append(fn)
bb.event.fire(runQueueTaskFailed(task, self.stats, exitcode, self.rq), self.cfgData)
if self.rqdata.taskData.abort:
self.rq.state = runQueueCleanUp
@@ -1626,7 +1597,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
"""
self.rq.read_workers()
-
+
if self.stats.total == 0:
# nothing to do
@@ -1634,18 +1605,17 @@ class RunQueueExecuteTasks(RunQueueExecute):
task = self.sched.next()
if task is not None:
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]]
- taskname = self.rqdata.runq_task[task]
+ fn = fn_from_tid(task)
+ taskname = taskname_from_tid(task)
if task in self.rq.scenequeue_covered:
- logger.debug(2, "Setscene covered task %s (%s)", task,
- self.rqdata.get_user_idstring(task))
+ logger.debug(2, "Setscene covered task %s", task)
self.task_skip(task, "covered")
return True
if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
- logger.debug(2, "Stamp current task %s (%s)", task,
- self.rqdata.get_user_idstring(task))
+ logger.debug(2, "Stamp current task %s", task)
+
self.task_skip(task, "existing")
return True
@@ -1672,7 +1642,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
try:
self.rq.start_fakeworker(self)
except OSError as exc:
- logger.critical("Failed to spawn fakeroot worker to run %s:%s: %s" % (fn, taskname,
str(exc)))
+ logger.critical("Failed to spawn fakeroot worker to run %s: %s" % (task, str(exc)))
self.rq.state = runQueueFailed
return True
self.rq.fakeworker.stdin.write(b"<runtask>" + pickle.dumps((fn, task, taskname, False,
self.cooker.collection.get_file_appends(fn), taskdepdata)) + b"</runtask>")
@@ -1682,7 +1652,7 @@ class RunQueueExecuteTasks(RunQueueExecute):
self.rq.worker.stdin.flush()
self.build_stamps[task] = bb.build.stampfile(taskname, self.rqdata.dataCache, fn)
- self.build_stamps2.append(self.build_stamps[task])
+ self.build_stamps2.append(self.build_stamps[task])
self.runq_running.add(task)
self.stats.taskActive()
if self.stats.active < self.number_tasks:
@@ -1692,12 +1662,12 @@ class RunQueueExecuteTasks(RunQueueExecute):
self.rq.read_workers()
return self.rq.active_fds()
- if len(self.failed_fnids) != 0:
+ if len(self.failed_fns) != 0:
self.rq.state = runQueueFailed
return True
# Sanity Checks
- for task in range(self.stats.total):
+ for task in self.rqdata.runtaskentries:
if task not in self.runq_buildable:
logger.error("Task %s never buildable!", task)
if task not in self.runq_running:
@@ -1710,15 +1680,15 @@ class RunQueueExecuteTasks(RunQueueExecute):
def build_taskdepdata(self, task):
taskdepdata = {}
- next = self.rqdata.runq_depends[task]
+ next = self.rqdata.runtaskentries[task].depends
next.add(task)
while next:
additional = []
for revdep in next:
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[revdep]]
+ fn = fn_from_tid(revdep)
pn = self.rqdata.dataCache.pkg_fn[fn]
- taskname = self.rqdata.runq_task[revdep]
- deps = self.rqdata.runq_depends[revdep]
+ taskname = taskname_from_tid(revdep)
+ deps = self.rqdata.runtaskentries[revdep].depends
provides = self.rqdata.dataCache.fn_provides[fn]
taskdepdata[revdep] = [pn, taskname, fn, deps, provides]
for revdep2 in deps:
@@ -1738,16 +1708,16 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.scenequeue_notneeded = set()
# If we don't have any setscene functions, skip this step
- if len(self.rqdata.runq_setscene) == 0:
+ if len(self.rqdata.runq_setscene_tids) == 0:
rq.scenequeue_covered = set()
rq.state = runQueueRunInit
return
- self.stats = RunQueueStats(len(self.rqdata.runq_setscene))
+ self.stats = RunQueueStats(len(self.rqdata.runq_setscene_tids))
- sq_revdeps = []
- sq_revdeps_new = []
- sq_revdeps_squash = []
+ sq_revdeps = {}
+ sq_revdeps_new = {}
+ sq_revdeps_squash = {}
self.sq_harddeps = {}
# We need to construct a dependency graph for the setscene functions. Intermediate
@@ -1757,18 +1727,21 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
# First process the chains up to the first setscene task.
endpoints = {}
- for task in range(len(self.rqdata.runq_fnid)):
- sq_revdeps.append(copy.copy(self.rqdata.runq_revdeps[task]))
- sq_revdeps_new.append(set())
- if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene:
- endpoints[task] = set()
+ for tid in self.rqdata.runtaskentries:
+ sq_revdeps[tid] = copy.copy(self.rqdata.runtaskentries[tid].revdeps)
+ sq_revdeps_new[tid] = set()
+ if (len(sq_revdeps[tid]) == 0) and tid not in self.rqdata.runq_setscene_tids:
+ #bb.warn("Added endpoint %s" % (tid))
+ endpoints[tid] = set()
# Secondly process the chains between setscene tasks.
- for task in self.rqdata.runq_setscene:
- for dep in self.rqdata.runq_depends[task]:
+ for tid in self.rqdata.runq_setscene_tids:
+ #bb.warn("Added endpoint 2 %s" % (tid))
+ for dep in self.rqdata.runtaskentries[tid].depends:
if dep not in endpoints:
endpoints[dep] = set()
- endpoints[dep].add(task)
+ #bb.warn(" Added endpoint 3 %s" % (dep))
+ endpoints[dep].add(tid)
def process_endpoints(endpoints):
newendpoints = {}
@@ -1779,15 +1752,15 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
if sq_revdeps_new[point]:
tasks |= sq_revdeps_new[point]
sq_revdeps_new[point] = set()
- if point in self.rqdata.runq_setscene:
+ if point in self.rqdata.runq_setscene_tids:
sq_revdeps_new[point] = tasks
tasks = set()
- for dep in self.rqdata.runq_depends[point]:
+ for dep in self.rqdata.runtaskentries[point].depends:
if point in sq_revdeps[dep]:
sq_revdeps[dep].remove(point)
if tasks:
sq_revdeps_new[dep] |= tasks
- if (len(sq_revdeps[dep]) == 0 or len(sq_revdeps_new[dep]) != 0) and dep not in
self.rqdata.runq_setscene:
+ if (len(sq_revdeps[dep]) == 0 or len(sq_revdeps_new[dep]) != 0) and dep not in
self.rqdata.runq_setscene_tids:
newendpoints[dep] = task
if len(newendpoints) != 0:
process_endpoints(newendpoints)
@@ -1797,8 +1770,8 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
# Build a list of setscene tasks which are "unskippable"
# These are direct endpoints referenced by the build
endpoints2 = {}
- sq_revdeps2 = []
- sq_revdeps_new2 = []
+ sq_revdeps2 = {}
+ sq_revdeps_new2 = {}
def process_endpoints2(endpoints):
newendpoints = {}
for point, task in endpoints.items():
@@ -1808,84 +1781,85 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
if sq_revdeps_new2[point]:
tasks |= sq_revdeps_new2[point]
sq_revdeps_new2[point] = set()
- if point in self.rqdata.runq_setscene:
+ if point in self.rqdata.runq_setscene_tids:
sq_revdeps_new2[point] = tasks
- for dep in self.rqdata.runq_depends[point]:
+ for dep in self.rqdata.runtaskentries[point].depends:
if point in sq_revdeps2[dep]:
sq_revdeps2[dep].remove(point)
if tasks:
sq_revdeps_new2[dep] |= tasks
- if (len(sq_revdeps2[dep]) == 0 or len(sq_revdeps_new2[dep]) != 0) and dep not in
self.rqdata.runq_setscene:
+ if (len(sq_revdeps2[dep]) == 0 or len(sq_revdeps_new2[dep]) != 0) and dep not in
self.rqdata.runq_setscene_tids:
newendpoints[dep] = tasks
if len(newendpoints) != 0:
process_endpoints2(newendpoints)
- for task in range(len(self.rqdata.runq_fnid)):
- sq_revdeps2.append(copy.copy(self.rqdata.runq_revdeps[task]))
- sq_revdeps_new2.append(set())
- if (len(self.rqdata.runq_revdeps[task]) == 0) and task not in self.rqdata.runq_setscene:
- endpoints2[task] = set()
+ for tid in self.rqdata.runtaskentries:
+ sq_revdeps2[tid] = copy.copy(self.rqdata.runtaskentries[tid].revdeps)
+ sq_revdeps_new2[tid] = set()
+ if (len(sq_revdeps2[tid]) == 0) and tid not in self.rqdata.runq_setscene_tids:
+ endpoints2[tid] = set()
process_endpoints2(endpoints2)
self.unskippable = []
- for task in self.rqdata.runq_setscene:
- if sq_revdeps_new2[task]:
- self.unskippable.append(self.rqdata.runq_setscene.index(task))
+ for tid in self.rqdata.runq_setscene_tids:
+ if sq_revdeps_new2[tid]:
+ self.unskippable.append(tid)
- for task in range(len(self.rqdata.runq_fnid)):
- if task in self.rqdata.runq_setscene:
+ for tid in self.rqdata.runtaskentries:
+ if tid in self.rqdata.runq_setscene_tids:
deps = set()
- for dep in sq_revdeps_new[task]:
- deps.add(self.rqdata.runq_setscene.index(dep))
- sq_revdeps_squash.append(deps)
- elif len(sq_revdeps_new[task]) != 0:
+ for dep in sq_revdeps_new[tid]:
+ deps.add(dep)
+ sq_revdeps_squash[tid] = deps
+ elif len(sq_revdeps_new[tid]) != 0:
bb.msg.fatal("RunQueue", "Something went badly wrong during scenequeue generation, aborting.
Please report this problem.")
# Resolve setscene inter-task dependencies
# e.g. do_sometask_setscene[depends] = "targetname:do_someothertask_setscene"
# Note that anything explicitly depended upon will have its reverse dependencies removed to avoid
circular dependencies
- for task in self.rqdata.runq_setscene:
- realid =
self.rqdata.taskData.gettask_id(self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[task]],
self.rqdata.runq_task[task] + "_setscene", False)
- idepends = self.rqdata.taskData.tasks_idepends[realid]
- for (depid, idependtask) in idepends:
- if depid not in self.rqdata.taskData.build_targets:
+ for tid in self.rqdata.runq_setscene_tids:
+ realtid = tid + "_setscene"
+ idepends = self.rqdata.taskData.taskentries[realtid].idepends
+ for (depname, idependtask) in idepends:
+
+ if depname not in self.rqdata.taskData.build_targets:
continue
- depdata = self.rqdata.taskData.build_targets[depid][0]
- if depdata is None:
+ depfn = self.rqdata.taskData.build_targets[depname][0]
+ if depfn is None:
continue
- dep = self.rqdata.taskData.fn_index[depdata]
- taskid = self.rqdata.get_task_id(self.rqdata.taskData.getfn_id(dep),
idependtask.replace("_setscene", ""))
- if taskid is None:
- bb.msg.fatal("RunQueue", "Task %s_setscene depends upon non-existent task %s:%s" %
(self.rqdata.get_user_idstring(task), dep, idependtask))
+ deptid = depfn + ":" + idependtask.replace("_setscene", "")
+ if deptid not in self.rqdata.runtaskentries:
+ bb.msg.fatal("RunQueue", "Task %s depends upon non-existent task %s:%s" % (realtid,
depfn, idependtask))
- if not self.rqdata.runq_setscene.index(taskid) in self.sq_harddeps:
- self.sq_harddeps[self.rqdata.runq_setscene.index(taskid)] = set()
-
self.sq_harddeps[self.rqdata.runq_setscene.index(taskid)].add(self.rqdata.runq_setscene.index(task))
+ if not deptid in self.sq_harddeps:
+ self.sq_harddeps[deptid] = set()
+ self.sq_harddeps[deptid].add(tid)
-
sq_revdeps_squash[self.rqdata.runq_setscene.index(task)].add(self.rqdata.runq_setscene.index(taskid))
+ sq_revdeps_squash[tid].add(deptid)
# Have to zero this to avoid circular dependencies
- sq_revdeps_squash[self.rqdata.runq_setscene.index(taskid)] = set()
+ sq_revdeps_squash[deptid] = set()
for task in self.sq_harddeps:
for dep in self.sq_harddeps[task]:
sq_revdeps_squash[dep].add(task)
- #for task in range(len(sq_revdeps_squash)):
- # realtask = self.rqdata.runq_setscene[task]
- # bb.warn("Task %s: %s_setscene is %s " % (task, self.rqdata.get_user_idstring(realtask) ,
sq_revdeps_squash[task]))
+ #for tid in sq_revdeps_squash:
+ # for dep in sq_revdeps_squash[tid]:
+ # data = data + "\n %s" % dep
+ # bb.warn("Task %s_setscene: is %s " % (tid, data
- self.sq_deps = []
+ self.sq_deps = {}
self.sq_revdeps = sq_revdeps_squash
self.sq_revdeps2 = copy.deepcopy(self.sq_revdeps)
- for task in range(len(self.sq_revdeps)):
- self.sq_deps.append(set())
- for task in range(len(self.sq_revdeps)):
- for dep in self.sq_revdeps[task]:
- self.sq_deps[dep].add(task)
+ for tid in self.sq_revdeps:
+ self.sq_deps[tid] = set()
+ for tid in self.sq_revdeps:
+ for dep in self.sq_revdeps[tid]:
+ self.sq_deps[dep].add(tid)
- for task in range(len(self.sq_revdeps)):
- if len(self.sq_revdeps[task]) == 0:
- self.runq_buildable.add(task)
+ for tid in self.sq_revdeps:
+ if len(self.sq_revdeps[tid]) == 0:
+ self.runq_buildable.add(tid)
self.outrightfail = []
if self.rq.hashvalidate:
@@ -1896,35 +1870,35 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
sq_task = []
noexec = []
stamppresent = []
- for task in range(len(self.sq_revdeps)):
- realtask = self.rqdata.runq_setscene[task]
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]]
- taskname = self.rqdata.runq_task[realtask]
+ for tid in self.sq_revdeps:
+ fn = fn_from_tid(tid)
+ taskname = taskname_from_tid(tid)
+
taskdep = self.rqdata.dataCache.task_deps[fn]
if 'noexec' in taskdep and taskname in taskdep['noexec']:
- noexec.append(task)
- self.task_skip(task)
+ noexec.append(tid)
+ self.task_skip(tid)
bb.build.make_stamp(taskname + "_setscene", self.rqdata.dataCache, fn)
continue
- if self.rq.check_stamp_task(realtask, taskname + "_setscene", cache=self.stampcache):
- logger.debug(2, 'Setscene stamp current for task %s(%s)', task,
self.rqdata.get_user_idstring(realtask))
- stamppresent.append(task)
- self.task_skip(task)
+ if self.rq.check_stamp_task(tid, taskname + "_setscene", cache=self.stampcache):
+ logger.debug(2, 'Setscene stamp current for task %s', tid)
+ stamppresent.append(tid)
+ self.task_skip(tid)
continue
- if self.rq.check_stamp_task(realtask, taskname, recurse = True, cache=self.stampcache):
- logger.debug(2, 'Normal stamp current for task %s(%s)', task,
self.rqdata.get_user_idstring(realtask))
- stamppresent.append(task)
- self.task_skip(task)
+ if self.rq.check_stamp_task(tid, taskname, recurse = True, cache=self.stampcache):
+ logger.debug(2, 'Normal stamp current for task %s', tid)
+ stamppresent.append(tid)
+ self.task_skip(tid)
continue
sq_fn.append(fn)
sq_hashfn.append(self.rqdata.dataCache.hashfn[fn])
- sq_hash.append(self.rqdata.runq_hash[realtask])
+ sq_hash.append(self.rqdata.runtaskentries[tid].hash)
sq_taskname.append(taskname)
- sq_task.append(task)
+ sq_task.append(tid)
call = self.rq.hashvalidate + "(sq_fn, sq_task, sq_hash, sq_hashfn, d)"
locs = { "sq_fn" : sq_fn, "sq_task" : sq_taskname, "sq_hash" : sq_hash, "sq_hashfn" : sq_hashfn,
"d" : self.cooker.expanded_data }
valid = bb.utils.better_eval(call, locs)
@@ -1933,12 +1907,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
for v in valid:
valid_new.append(sq_task[v])
- for task in range(len(self.sq_revdeps)):
- if task not in valid_new and task not in noexec:
- realtask = self.rqdata.runq_setscene[task]
- logger.debug(2, 'No package found, so skipping setscene task %s',
- self.rqdata.get_user_idstring(realtask))
- self.outrightfail.append(task)
+ for tid in self.sq_revdeps:
+ if tid not in valid_new and tid not in noexec:
+ logger.debug(2, 'No package found, so skipping setscene task %s', tid)
+ self.outrightfail.append(tid)
logger.info('Executing SetScene Tasks')
@@ -1947,9 +1919,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
def scenequeue_updatecounters(self, task, fail = False):
for dep in self.sq_deps[task]:
if fail and task in self.sq_harddeps and dep in self.sq_harddeps[task]:
- realtask = self.rqdata.runq_setscene[task]
- realdep = self.rqdata.runq_setscene[dep]
- logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" %
(self.rqdata.get_user_idstring(realtask), self.rqdata.get_user_idstring(realdep)))
+ logger.debug(2, "%s was unavailable and is a hard dependency of %s so skipping" % (task,
dep))
self.scenequeue_updatecounters(dep, fail)
continue
if task not in self.sq_revdeps2[dep]:
@@ -1966,10 +1936,7 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
completed dependencies as buildable
"""
- index = self.rqdata.runq_setscene[task]
- logger.debug(1, 'Found task %s which could be accelerated',
- self.rqdata.get_user_idstring(index))
-
+ logger.debug(1, 'Found task %s which could be accelerated', task)
self.scenequeue_covered.add(task)
self.scenequeue_updatecounters(task)
@@ -1989,7 +1956,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.runq_buildable.add(task)
self.stats.taskCompleted()
self.stats.taskSkipped()
- index = self.rqdata.runq_setscene[task]
self.scenequeue_notcovered.add(task)
self.scenequeue_updatecounters(task, True)
@@ -2010,20 +1976,19 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
task = None
if self.stats.active < self.number_tasks:
# Find the next setscene to run
- for nexttask in range(self.stats.total):
+ for nexttask in self.rqdata.runq_setscene_tids:
if nexttask in self.runq_buildable and nexttask not in self.runq_running:
if nexttask in self.unskippable:
- logger.debug(2, "Setscene task %s is unskippable" %
self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask]))
+ logger.debug(2, "Setscene task %s is unskippable" % nexttask)
if nexttask not in self.unskippable and len(self.sq_revdeps[nexttask]) > 0 and
self.sq_revdeps[nexttask].issubset(self.scenequeue_covered) and self.check_dependencies(nexttask,
self.sq_revdeps[nexttask], True):
- realtask = self.rqdata.runq_setscene[nexttask]
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]]
+ fn = fn_from_tid(nexttask)
foundtarget = False
for target in self.rqdata.target_pairs:
- if target[0] == fn and target[1] == self.rqdata.runq_task[realtask]:
+ if target[0] == fn and target[1] == taskname_from_tid(nexttask):
foundtarget = True
break
if not foundtarget:
- logger.debug(2, "Skipping setscene for task %s" %
self.rqdata.get_user_idstring(self.rqdata.runq_setscene[nexttask]))
+ logger.debug(2, "Skipping setscene for task %s" % nexttask)
self.task_skip(nexttask)
self.scenequeue_notneeded.add(nexttask)
return True
@@ -2033,25 +1998,21 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
task = nexttask
break
if task is not None:
- realtask = self.rqdata.runq_setscene[task]
- fn = self.rqdata.taskData.fn_index[self.rqdata.runq_fnid[realtask]]
-
- taskname = self.rqdata.runq_task[realtask] + "_setscene"
- if self.rq.check_stamp_task(realtask, self.rqdata.runq_task[realtask], recurse = True,
cache=self.stampcache):
- logger.debug(2, 'Stamp for underlying task %s(%s) is current, so skipping setscene variant',
- task, self.rqdata.get_user_idstring(realtask))
+ fn = fn_from_tid(task)
+ taskname = taskname_from_tid(task) + "_setscene"
+ if self.rq.check_stamp_task(task, taskname_from_tid(task), recurse = True,
cache=self.stampcache):
+ logger.debug(2, 'Stamp for underlying task %s is current, so skipping setscene variant',
task)
self.task_failoutright(task)
return True
if self.cooker.configuration.force:
for target in self.rqdata.target_pairs:
- if target[0] == fn and target[1] == self.rqdata.runq_task[realtask]:
+ if target[0] == fn and target[1] == taskname_from_tid(task):
self.task_failoutright(task)
return True
- if self.rq.check_stamp_task(realtask, taskname, cache=self.stampcache):
- logger.debug(2, 'Setscene stamp current task %s(%s), so skip it and its dependencies',
- task, self.rqdata.get_user_idstring(realtask))
+ if self.rq.check_stamp_task(task, taskname, cache=self.stampcache):
+ logger.debug(2, 'Setscene stamp current task %s, so skip it and its dependencies', task)
self.task_skip(task)
return True
@@ -2062,10 +2023,10 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
if 'fakeroot' in taskdep and taskname in taskdep['fakeroot'] and not
self.cooker.configuration.dry_run:
if not self.rq.fakeworker:
self.rq.start_fakeworker(self)
- self.rq.fakeworker.stdin.write(b"<runtask>" + pickle.dumps((fn, realtask, taskname, True,
self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>")
+ self.rq.fakeworker.stdin.write(b"<runtask>" + pickle.dumps((fn, task, taskname, True,
self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>")
self.rq.fakeworker.stdin.flush()
else:
- self.rq.worker.stdin.write(b"<runtask>" + pickle.dumps((fn, realtask, taskname, True,
self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>")
+ self.rq.worker.stdin.write(b"<runtask>" + pickle.dumps((fn, task, taskname, True,
self.cooker.collection.get_file_appends(fn), None)) + b"</runtask>")
self.rq.worker.stdin.flush()
self.runq_running.add(task)
@@ -2077,17 +2038,17 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
self.rq.read_workers()
return self.rq.active_fds()
- #for task in range(self.stats.total):
- # if task not in self.runq_running:
- # buildable = task in self.runq_buildable
- # revdeps = self.sq_revdeps[task]
- # bb.warn("Found we didn't run %s %s %s %s" % (task, buildable, str(revdeps),
self.rqdata.get_user_idstring(self.rqdata.runq_setscene[task])))
+ #for tid in self.sq_revdeps:
+ # if tid not in self.runq_running:
+ # buildable = tid in self.runq_buildable
+ # revdeps = self.sq_revdeps[tid]
+ # bb.warn("Found we didn't run %s %s %s" % (tid, buildable, str(revdeps)))
# Convert scenequeue_covered task numbers into full taskgraph ids
oldcovered = self.scenequeue_covered
self.rq.scenequeue_covered = set()
for task in oldcovered:
- self.rq.scenequeue_covered.add(self.rqdata.runq_setscene[task])
+ self.rq.scenequeue_covered.add(task)
logger.debug(1, 'We can skip tasks %s', sorted(self.rq.scenequeue_covered))
@@ -2099,8 +2060,6 @@ class RunQueueExecuteScenequeue(RunQueueExecute):
return True
def runqueue_process_waitpid(self, task, status):
- task = self.rq.rqdata.runq_setscene.index(task)
-
RunQueueExecute.runqueue_process_waitpid(self, task, status)
class TaskFailure(Exception):
@@ -2127,9 +2086,9 @@ class runQueueEvent(bb.event.Event):
"""
def __init__(self, task, stats, rq):
self.taskid = task
- self.taskstring = rq.rqdata.get_user_idstring(task)
- self.taskname = rq.rqdata.get_task_name(task)
- self.taskfile = rq.rqdata.get_task_file(task)
+ self.taskstring = task
+ self.taskname = taskname_from_tid(task)
+ self.taskfile = fn_from_tid(task)
self.taskhash = rq.rqdata.get_task_hash(task)
self.stats = stats.copy()
bb.event.Event.__init__(self)
@@ -2140,11 +2099,10 @@ class sceneQueueEvent(runQueueEvent):
"""
def __init__(self, task, stats, rq, noexec=False):
runQueueEvent.__init__(self, task, stats, rq)
- realtask = rq.rqdata.runq_setscene[task]
- self.taskstring = rq.rqdata.get_user_idstring(realtask, "_setscene")
- self.taskname = rq.rqdata.get_task_name(realtask) + "_setscene"
- self.taskfile = rq.rqdata.get_task_file(realtask)
- self.taskhash = rq.rqdata.get_task_hash(realtask)
+ self.taskstring = task + "_setscene"
+ self.taskname = taskname_from_tid(task) + "_setscene"
+ self.taskfile = fn_from_tid(task)
+ self.taskhash = rq.rqdata.get_task_hash(task)
class runQueueTaskStarted(runQueueEvent):
"""
diff --git a/bitbake/lib/bb/taskdata.py b/bitbake/lib/bb/taskdata.py
index 65628c8..d8bdbca 100644
--- a/bitbake/lib/bb/taskdata.py
+++ b/bitbake/lib/bb/taskdata.py
@@ -37,27 +37,24 @@ def re_match_strings(target, strings):
return any(name == target or re.match(name, target)
for name in strings)
+class TaskEntry:
+ def __init__(self):
+ self.tdepends = []
+ self.idepends = []
+ self.irdepends = []
+
class TaskData:
"""
BitBake Task Data implementation
"""
def __init__(self, abort = True, tryaltconfigs = False, skiplist = None, allowincomplete = False):
- self.build_names_index = []
- self.run_names_index = []
- self.fn_index = []
-
self.build_targets = {}
self.run_targets = {}
self.external_targets = []
- self.tasks_fnid = []
- self.tasks_name = []
- self.tasks_tdepends = []
- self.tasks_idepends = []
- self.tasks_irdepends = []
- # Cache to speed up task ID lookups
- self.tasks_lookup = {}
+ self.seenfns = []
+ self.taskentries = {}
self.depids = {}
self.rdepids = {}
@@ -66,7 +63,7 @@ class TaskData:
self.failed_deps = []
self.failed_rdeps = []
- self.failed_fnids = []
+ self.failed_fns = []
self.abort = abort
self.tryaltconfigs = tryaltconfigs
@@ -74,88 +71,6 @@ class TaskData:
self.skiplist = skiplist
- def getbuild_id(self, name):
- """
- Return an ID number for the build target name.
- If it doesn't exist, create one.
- """
- if not name in self.build_names_index:
- self.build_names_index.append(name)
- return len(self.build_names_index) - 1
-
- return self.build_names_index.index(name)
-
- def getrun_id(self, name):
- """
- Return an ID number for the run target name.
- If it doesn't exist, create one.
- """
- if not name in self.run_names_index:
- self.run_names_index.append(name)
- return len(self.run_names_index) - 1
-
- return self.run_names_index.index(name)
-
- def getfn_id(self, name):
- """
- Return an ID number for the filename.
- If it doesn't exist, create one.
- """
- if not name in self.fn_index:
- self.fn_index.append(name)
- return len(self.fn_index) - 1
-
- return self.fn_index.index(name)
-
- def gettask_ids(self, fnid):
- """
- Return an array of the ID numbers matching a given fnid.
- """
- ids = []
- if fnid in self.tasks_lookup:
- for task in self.tasks_lookup[fnid]:
- ids.append(self.tasks_lookup[fnid][task])
- return ids
-
- def gettask_id_fromfnid(self, fnid, task):
- """
- Return an ID number for the task matching fnid and task.
- """
- if fnid in self.tasks_lookup:
- if task in self.tasks_lookup[fnid]:
- return self.tasks_lookup[fnid][task]
-
- return None
-
- def gettask_id(self, fn, task, create = True):
- """
- Return an ID number for the task matching fn and task.
- If it doesn't exist, create one by default.
- Optionally return None instead.
- """
- fnid = self.getfn_id(fn)
-
- if fnid in self.tasks_lookup:
- if task in self.tasks_lookup[fnid]:
- return self.tasks_lookup[fnid][task]
-
- if not create:
- return None
-
- self.tasks_name.append(task)
- self.tasks_fnid.append(fnid)
- self.tasks_tdepends.append([])
- self.tasks_idepends.append([])
- self.tasks_irdepends.append([])
-
- listid = len(self.tasks_name) - 1
-
- if fnid not in self.tasks_lookup:
- self.tasks_lookup[fnid] = {}
- self.tasks_lookup[fnid][task] = listid
-
- return listid
-
def add_tasks(self, fn, dataCache):
"""
Add tasks for a given fn to the database
@@ -163,29 +78,31 @@ class TaskData:
task_deps = dataCache.task_deps[fn]
- fnid = self.getfn_id(fn)
-
- if fnid in self.failed_fnids:
+ if fn in self.failed_fns:
bb.msg.fatal("TaskData", "Trying to re-add a failed file? Something is broken...")
# Check if we've already seen this fn
- if fnid in self.tasks_fnid:
+ if fn in self.seenfns:
return
+ self.seenfns.append(fn)
+
self.add_extra_deps(fn, dataCache)
for task in task_deps['tasks']:
+ tid = "%s:%s" % (fn, task)
+ self.taskentries[tid] = TaskEntry()
+
# Work out task dependencies
parentids = []
for dep in task_deps['parents'][task]:
if dep not in task_deps['tasks']:
bb.debug(2, "Not adding dependeny of %s on %s since %s does not exist" % (task, dep,
dep))
continue
- parentid = self.gettask_id(fn, dep)
+ parentid = "%s:%s" % (fn, dep)
parentids.append(parentid)
- taskid = self.gettask_id(fn, task)
- self.tasks_tdepends[taskid].extend(parentids)
+ self.taskentries[tid].tdepends.extend(parentids)
# Touch all intertask dependencies
if 'depends' in task_deps and task in task_deps['depends']:
@@ -194,29 +111,30 @@ class TaskData:
if dep:
if ":" not in dep:
bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':'
character\n. Task 'depends' should be specified in the form 'packagename:task'" % (fn, dep))
- ids.append(((self.getbuild_id(dep.split(":")[0])), dep.split(":")[1]))
- self.tasks_idepends[taskid].extend(ids)
+ ids.append(((dep.split(":")[0]), dep.split(":")[1]))
+ self.seen_build_target(dep.split(":")[0])
+ self.taskentries[tid].idepends.extend(ids)
if 'rdepends' in task_deps and task in task_deps['rdepends']:
ids = []
for dep in task_deps['rdepends'][task].split():
if dep:
if ":" not in dep:
bb.msg.fatal("TaskData", "Error for %s, dependency %s does not contain ':'
character\n. Task 'rdepends' should be specified in the form 'packagename:task'" % (fn, dep))
- ids.append(((self.getrun_id(dep.split(":")[0])), dep.split(":")[1]))
- self.tasks_irdepends[taskid].extend(ids)
-
+ ids.append(((dep.split(":")[0]), dep.split(":")[1]))
+ self.seen_run_target(dep.split(":")[0])
+ self.taskentries[tid].irdepends.extend(ids)
# Work out build dependencies
- if not fnid in self.depids:
- dependids = {}
+ if not fn in self.depids:
+ dependids = set()
for depend in dataCache.deps[fn]:
- dependids[self.getbuild_id(depend)] = None
- self.depids[fnid] = dependids.keys()
+ dependids.add(depend)
+ self.depids[fn] = list(dependids)
logger.debug(2, "Added dependencies %s for %s", str(dataCache.deps[fn]), fn)
# Work out runtime dependencies
- if not fnid in self.rdepids:
- rdependids = {}
+ if not fn in self.rdepids:
+ rdependids = set()
rdepends = dataCache.rundeps[fn]
rrecs = dataCache.runrecs[fn]
rdependlist = []
@@ -224,24 +142,26 @@ class TaskData:
for package in rdepends:
for rdepend in rdepends[package]:
rdependlist.append(rdepend)
- rdependids[self.getrun_id(rdepend)] = None
+ rdependids.add(rdepend)
for package in rrecs:
for rdepend in rrecs[package]:
rreclist.append(rdepend)
- rdependids[self.getrun_id(rdepend)] = None
+ rdependids.add(rdepend)
if rdependlist:
logger.debug(2, "Added runtime dependencies %s for %s", str(rdependlist), fn)
if rreclist:
logger.debug(2, "Added runtime recommendations %s for %s", str(rreclist), fn)
- self.rdepids[fnid] = rdependids.keys()
+ self.rdepids[fn] = list(rdependids)
- for dep in self.depids[fnid]:
+ for dep in self.depids[fn]:
+ self.seen_build_target(dep)
if dep in self.failed_deps:
- self.fail_fnid(fnid)
+ self.fail_fn(fn)
return
- for dep in self.rdepids[fnid]:
+ for dep in self.rdepids[fn]:
+ self.seen_run_target(dep)
if dep in self.failed_rdeps:
- self.fail_fnid(fnid)
+ self.fail_fn(fn)
return
def add_extra_deps(self, fn, dataCache):
@@ -263,9 +183,7 @@ class TaskData:
"""
Have we a build target matching this name?
"""
- targetid = self.getbuild_id(target)
-
- if targetid in self.build_targets:
+ if target in self.build_targets and self.build_targets[target]:
return True
return False
@@ -273,50 +191,54 @@ class TaskData:
"""
Have we a runtime target matching this name?
"""
- targetid = self.getrun_id(target)
-
- if targetid in self.run_targets:
+ if target in self.run_targets and self.run_targets[target]:
return True
return False
+ def seen_build_target(self, name):
+ """
+ Maintain a list of build targets
+ """
+ if name not in self.build_targets:
+ self.build_targets[name] = []
+
def add_build_target(self, fn, item):
"""
Add a build target.
If already present, append the provider fn to the list
"""
- targetid = self.getbuild_id(item)
- fnid = self.getfn_id(fn)
-
- if targetid in self.build_targets:
- if fnid in self.build_targets[targetid]:
+ if item in self.build_targets:
+ if fn in self.build_targets[item]:
return
- self.build_targets[targetid].append(fnid)
+ self.build_targets[item].append(fn)
return
- self.build_targets[targetid] = [fnid]
+ self.build_targets[item] = [fn]
+
+ def seen_run_target(self, name):
+ """
+ Maintain a list of runtime build targets
+ """
+ if name not in self.run_targets:
+ self.run_targets[name] = []
def add_runtime_target(self, fn, item):
"""
Add a runtime target.
If already present, append the provider fn to the list
"""
- targetid = self.getrun_id(item)
- fnid = self.getfn_id(fn)
-
- if targetid in self.run_targets:
- if fnid in self.run_targets[targetid]:
+ if item in self.run_targets:
+ if fn in self.run_targets[item]:
return
- self.run_targets[targetid].append(fnid)
+ self.run_targets[item].append(fn)
return
- self.run_targets[targetid] = [fnid]
+ self.run_targets[item] = [fn]
- def mark_external_target(self, item):
+ def mark_external_target(self, target):
"""
Mark a build target as being externally requested
"""
- targetid = self.getbuild_id(item)
-
- if targetid not in self.external_targets:
- self.external_targets.append(targetid)
+ if target not in self.external_targets:
+ self.external_targets.append(target)
def get_unresolved_build_targets(self, dataCache):
"""
@@ -324,12 +246,12 @@ class TaskData:
are unknown.
"""
unresolved = []
- for target in self.build_names_index:
+ for target in self.build_targets:
if re_match_strings(target, dataCache.ignored_dependencies):
continue
- if self.build_names_index.index(target) in self.failed_deps:
+ if target in self.failed_deps:
continue
- if not self.have_build_target(target):
+ if not self.build_targets[target]:
unresolved.append(target)
return unresolved
@@ -339,12 +261,12 @@ class TaskData:
are unknown.
"""
unresolved = []
- for target in self.run_names_index:
+ for target in self.run_targets:
if re_match_strings(target, dataCache.ignored_dependencies):
continue
- if self.run_names_index.index(target) in self.failed_rdeps:
+ if target in self.failed_rdeps:
continue
- if not self.have_runtime_target(target):
+ if not self.run_targets[target]:
unresolved.append(target)
return unresolved
@@ -352,50 +274,26 @@ class TaskData:
"""
Return a list of providers of item
"""
- targetid = self.getbuild_id(item)
+ return self.build_targets[item]
- return self.build_targets[targetid]
-
- def get_dependees(self, itemid):
+ def get_dependees(self, item):
"""
Return a list of targets which depend on item
"""
dependees = []
- for fnid in self.depids:
- if itemid in self.depids[fnid]:
- dependees.append(fnid)
- return dependees
-
- def get_dependees_str(self, item):
- """
- Return a list of targets which depend on item as a user readable string
- """
- itemid = self.getbuild_id(item)
- dependees = []
- for fnid in self.depids:
- if itemid in self.depids[fnid]:
- dependees.append(self.fn_index[fnid])
+ for fn in self.depids:
+ if item in self.depids[fn]:
+ dependees.append(fn)
return dependees
- def get_rdependees(self, itemid):
+ def get_rdependees(self, item):
"""
Return a list of targets which depend on runtime item
"""
dependees = []
- for fnid in self.rdepids:
- if itemid in self.rdepids[fnid]:
- dependees.append(fnid)
- return dependees
-
- def get_rdependees_str(self, item):
- """
- Return a list of targets which depend on runtime item as a user readable string
- """
- itemid = self.getrun_id(item)
- dependees = []
- for fnid in self.rdepids:
- if itemid in self.rdepids[fnid]:
- dependees.append(self.fn_index[fnid])
+ for fn in self.rdepids:
+ if item in self.rdepids[fn]:
+ dependees.append(fn)
return dependees
def get_reasons(self, item, runtime=False):
@@ -431,7 +329,7 @@ class TaskData:
except bb.providers.NoProvider:
if self.abort:
raise
- self.remove_buildtarget(self.getbuild_id(item))
+ self.remove_buildtarget(item)
self.mark_external_target(item)
@@ -453,7 +351,7 @@ class TaskData:
new = dataCache.pkg_fn[fn] + " RPROVIDES " + item
if new not in close_matches:
close_matches.append(new)
- bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item),
reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
+ bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item),
reasons=self.get_reasons(item), close_matches=close_matches), cfgData)
raise bb.providers.NoProvider(item)
if self.have_build_target(item):
@@ -462,10 +360,10 @@ class TaskData:
all_p = dataCache.providers[item]
eligible, foundUnique = bb.providers.filterProviders(all_p, item, cfgData, dataCache)
- eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids]
+ eligible = [p for p in eligible if not p in self.failed_fns]
if not eligible:
- bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees_str(item), reasons=["No
eligible PROVIDERs exist for '%s'" % item]), cfgData)
+ bb.event.fire(bb.event.NoProvider(item, dependees=self.get_dependees(item), reasons=["No
eligible PROVIDERs exist for '%s'" % item]), cfgData)
raise bb.providers.NoProvider(item)
if len(eligible) > 1 and foundUnique == False:
@@ -477,8 +375,7 @@ class TaskData:
self.consider_msgs_cache.append(item)
for fn in eligible:
- fnid = self.getfn_id(fn)
- if fnid in self.failed_fnids:
+ if fn in self.failed_fns:
continue
logger.debug(2, "adding %s to satisfy %s", fn, item)
self.add_build_target(fn, item)
@@ -502,14 +399,14 @@ class TaskData:
all_p = bb.providers.getRuntimeProviders(dataCache, item)
if not all_p:
- bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item),
reasons=self.get_reasons(item, True)), cfgData)
+ bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item),
reasons=self.get_reasons(item, True)), cfgData)
raise bb.providers.NoRProvider(item)
eligible, numberPreferred = bb.providers.filterProvidersRunTime(all_p, item, cfgData, dataCache)
- eligible = [p for p in eligible if not self.getfn_id(p) in self.failed_fnids]
+ eligible = [p for p in eligible if not p in self.failed_fns]
if not eligible:
- bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees_str(item),
reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
+ bb.event.fire(bb.event.NoProvider(item, runtime=True, dependees=self.get_rdependees(item),
reasons=["No eligible RPROVIDERs exist for '%s'" % item]), cfgData)
raise bb.providers.NoRProvider(item)
if len(eligible) > 1 and numberPreferred == 0:
@@ -531,82 +428,80 @@ class TaskData:
# run through the list until we find one that we can build
for fn in eligible:
- fnid = self.getfn_id(fn)
- if fnid in self.failed_fnids:
+ if fn in self.failed_fns:
continue
logger.debug(2, "adding '%s' to satisfy runtime '%s'", fn, item)
self.add_runtime_target(fn, item)
self.add_tasks(fn, dataCache)
- def fail_fnid(self, fnid, missing_list=None):
+ def fail_fn(self, fn, missing_list=None):
"""
Mark a file as failed (unbuildable)
Remove any references from build and runtime provider lists
missing_list, A list of missing requirements for this target
"""
- if fnid in self.failed_fnids:
+ if fn in self.failed_fns:
return
if not missing_list:
missing_list = []
- logger.debug(1, "File '%s' is unbuildable, removing...", self.fn_index[fnid])
- self.failed_fnids.append(fnid)
+ logger.debug(1, "File '%s' is unbuildable, removing...", fn)
+ self.failed_fns.append(fn)
for target in self.build_targets:
- if fnid in self.build_targets[target]:
- self.build_targets[target].remove(fnid)
+ if fn in self.build_targets[target]:
+ self.build_targets[target].remove(fn)
if len(self.build_targets[target]) == 0:
self.remove_buildtarget(target, missing_list)
for target in self.run_targets:
- if fnid in self.run_targets[target]:
- self.run_targets[target].remove(fnid)
+ if fn in self.run_targets[target]:
+ self.run_targets[target].remove(fn)
if len(self.run_targets[target]) == 0:
self.remove_runtarget(target, missing_list)
- def remove_buildtarget(self, targetid, missing_list=None):
+ def remove_buildtarget(self, target, missing_list=None):
"""
Mark a build target as failed (unbuildable)
Trigger removal of any files that have this as a dependency
"""
if not missing_list:
- missing_list = [self.build_names_index[targetid]]
+ missing_list = [target]
else:
- missing_list = [self.build_names_index[targetid]] + missing_list
- logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain
was: %s", self.build_names_index[targetid], missing_list)
- self.failed_deps.append(targetid)
- dependees = self.get_dependees(targetid)
- for fnid in dependees:
- self.fail_fnid(fnid, missing_list)
- for taskid in range(len(self.tasks_idepends)):
- idepends = self.tasks_idepends[taskid]
- for (idependid, idependtask) in idepends:
- if idependid == targetid:
- self.fail_fnid(self.tasks_fnid[taskid], missing_list)
-
- if self.abort and targetid in self.external_targets:
- target = self.build_names_index[targetid]
+ missing_list = [target] + missing_list
+ logger.verbose("Target '%s' is unbuildable, removing...\nMissing or unbuildable dependency chain
was: %s", target, missing_list)
+ self.failed_deps.append(target)
+ dependees = self.get_dependees(target)
+ for fn in dependees:
+ self.fail_fn(fn, missing_list)
+ for tid in self.taskentries:
+ for (idepend, idependtask) in self.taskentries[tid].idepends:
+ if idepend == target:
+ fn = tid.rsplit(":",1)[0]
+ self.fail_fn(fn, missing_list)
+
+ if self.abort and target in self.external_targets:
logger.error("Required build target '%s' has no buildable providers.\nMissing or unbuildable
dependency chain was: %s", target, missing_list)
raise bb.providers.NoProvider(target)
- def remove_runtarget(self, targetid, missing_list=None):
+ def remove_runtarget(self, target, missing_list=None):
"""
Mark a run target as failed (unbuildable)
Trigger removal of any files that have this as a dependency
"""
if not missing_list:
- missing_list = [self.run_names_index[targetid]]
+ missing_list = [target]
else:
- missing_list = [self.run_names_index[targetid]] + missing_list
-
- logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency
chain was: %s", self.run_names_index[targetid], missing_list)
- self.failed_rdeps.append(targetid)
- dependees = self.get_rdependees(targetid)
- for fnid in dependees:
- self.fail_fnid(fnid, missing_list)
- for taskid in range(len(self.tasks_irdepends)):
- irdepends = self.tasks_irdepends[taskid]
- for (idependid, idependtask) in irdepends:
- if idependid == targetid:
- self.fail_fnid(self.tasks_fnid[taskid], missing_list)
+ missing_list = [target] + missing_list
+
+ logger.info("Runtime target '%s' is unbuildable, removing...\nMissing or unbuildable dependency
chain was: %s", target, missing_list)
+ self.failed_rdeps.append(target)
+ dependees = self.get_rdependees(target)
+ for fn in dependees:
+ self.fail_fn(fn, missing_list)
+ for tid in self.taskentries:
+ for (idepend, idependtask) in self.taskentries[tid].irdepends:
+ if idepend == target:
+ fn = tid.rsplit(":",1)[0]
+ self.fail_fn(fn, missing_list)
def add_unresolved(self, cfgData, dataCache):
"""
@@ -620,17 +515,16 @@ class TaskData:
self.add_provider_internal(cfgData, dataCache, target)
added = added + 1
except bb.providers.NoProvider:
- targetid = self.getbuild_id(target)
- if self.abort and targetid in self.external_targets and not self.allowincomplete:
+ if self.abort and target in self.external_targets and not self.allowincomplete:
raise
if not self.allowincomplete:
- self.remove_buildtarget(targetid)
+ self.remove_buildtarget(target)
for target in self.get_unresolved_run_targets(dataCache):
try:
self.add_rprovider(cfgData, dataCache, target)
added = added + 1
except (bb.providers.NoRProvider, bb.providers.MultipleRProvider):
- self.remove_runtarget(self.getrun_id(target))
+ self.remove_runtarget(target)
logger.debug(1, "Resolved " + str(added) + " extra dependencies")
if added == 0:
break
@@ -638,13 +532,13 @@ class TaskData:
def get_providermap(self, prefix=None):
provmap = {}
- for name in self.build_names_index:
+ for name in self.build_targets:
if prefix and not name.startswith(prefix):
continue
if self.have_build_target(name):
provider = self.get_provider(name)
if provider:
- provmap[name] = self.fn_index[provider[0]]
+ provmap[name] = provider[0]
return provmap
def dump_data(self):
@@ -652,39 +546,37 @@ class TaskData:
Dump some debug information on the internal data structures
"""
logger.debug(3, "build_names:")
- logger.debug(3, ", ".join(self.build_names_index))
+ logger.debug(3, ", ".join(self.build_targets))
logger.debug(3, "run_names:")
- logger.debug(3, ", ".join(self.run_names_index))
+ logger.debug(3, ", ".join(self.run_targets))
logger.debug(3, "build_targets:")
- for buildid in range(len(self.build_names_index)):
- target = self.build_names_index[buildid]
+ for target in self.build_targets:
targets = "None"
- if buildid in self.build_targets:
- targets = self.build_targets[buildid]
- logger.debug(3, " (%s)%s: %s", buildid, target, targets)
+ if target in self.build_targets:
+ targets = self.build_targets[target]
+ logger.debug(3, " %s: %s", target, targets)
logger.debug(3, "run_targets:")
- for runid in range(len(self.run_names_index)):
- target = self.run_names_index[runid]
+ for target in self.run_targets:
targets = "None"
- if runid in self.run_targets:
- targets = self.run_targets[runid]
- logger.debug(3, " (%s)%s: %s", runid, target, targets)
+ if target in self.run_targets:
+ targets = self.run_targets[target]
+ logger.debug(3, " %s: %s", target, targets)
logger.debug(3, "tasks:")
- for task in range(len(self.tasks_name)):
- logger.debug(3, " (%s)%s - %s: %s",
- task,
- self.fn_index[self.tasks_fnid[task]],
- self.tasks_name[task],
- self.tasks_tdepends[task])
+ for tid in self.taskentries:
+ logger.debug(3, " %s: %s %s %s",
+ tid,
+ self.taskentries[tid].idepends,
+ self.taskentries[tid].irdepends,
+ self.taskentries[tid].tdepends)
logger.debug(3, "dependency ids (per fn):")
- for fnid in self.depids:
- logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.depids[fnid])
+ for fn in self.depids:
+ logger.debug(3, " %s: %s", fn, self.depids[fn])
logger.debug(3, "runtime dependency ids (per fn):")
- for fnid in self.rdepids:
- logger.debug(3, " %s %s: %s", fnid, self.fn_index[fnid], self.rdepids[fnid])
+ for fn in self.rdepids:
+ logger.debug(3, " %s: %s", fn, self.rdepids[fn])
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
