[gnome-ostree] task: Add the concept of "TaskAfter"
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-ostree] task: Add the concept of "TaskAfter"
- Date: Tue, 26 Feb 2013 00:55:12 +0000 (UTC)
commit 0ced95e2910a298bfd2d78563f9ad16282753587
Author: Colin Walters <walters verbum org>
Date: Sun Feb 24 13:06:13 2013 -0500
task: Add the concept of "TaskAfter"
This way we can automatically run build when resolve completes, both
bdiff and builddisks when build completes, and smoketest when
builddisks completes.
src/js/builtins/autobuilder.js | 100 ++++++++++++---------------------------
src/js/builtins/make.js | 17 +++++--
src/js/builtins/run_task.js | 4 +-
src/js/jsondb.js | 7 +++
src/js/task.js | 88 +++++++++++++++++++++++++++-------
src/js/tasks/task-bdiff.js | 5 +-
src/js/tasks/task-build.js | 5 +-
src/js/tasks/task-builddisks.js | 5 +-
src/js/tasks/task-resolve.js | 18 +++++--
src/js/tasks/task-smoketest.js | 3 +-
10 files changed, 142 insertions(+), 110 deletions(-)
---
diff --git a/src/js/builtins/autobuilder.js b/src/js/builtins/autobuilder.js
index b22710b..4230572 100644
--- a/src/js/builtins/autobuilder.js
+++ b/src/js/builtins/autobuilder.js
@@ -50,9 +50,8 @@ const Autobuilder = new Lang.Class({
this.parser.addArgument('--autoupdate-self', { action: 'storeTrue' });
this.parser.addArgument('--stage');
- this._stages = ['resolve', 'build', 'builddisks', 'smoke'];
-
this._buildNeeded = true;
+ this._initialResolveNeeded = true;
this._fullResolveNeeded = true;
this._resolveTimeout = 0;
this._sourceSnapshotPath = null;
@@ -64,17 +63,6 @@ const Autobuilder = new Lang.Class({
this._initSnapshot(null, null, cancellable);
this._autoupdate_self = args.autoupdate_self;
- if (!args.stage)
- args.stage = 'build';
- this._stageIndex = this._stages.indexOf(args.stage);
- if (this._stageIndex < 0)
- throw new Error("Unknown stage " + args.stage);
- this._do_builddisks = this._stageIndex >= this._stages.indexOf('builddisks');
- this._do_smoke = this._stageIndex >= this._stages.indexOf('smoke');
-
- this._resolveTaskName = 'resolve'
- this._buildTaskName = 'build'
- this._bdiffTaskName = 'bdiff';
this._manifestPath = Gio.File.new_for_path('manifest.json');
@@ -90,15 +78,18 @@ const Autobuilder = new Lang.Class({
this._taskmaster = new Task.TaskMaster(this.workdir.get_child('tasks'),
{ onEmpty: Lang.bind(this, this._onTasksComplete) });
+ this._taskmaster.connect('task-executing', Lang.bind(this, this._onTaskExecuting));
this._taskmaster.connect('task-complete', Lang.bind(this, this._onTaskCompleted));
this._sourceSnapshotPath = this._src_db.getLatestPath();
+ /* Start an initial, non-fetching resolve */
+ this._runResolve();
+ /* Flag immediately that we need a full resolve */
+ this._fullResolveNeeded = true;
+ /* And set a timeout for 10 minutes for the next full resolve */
this._resolveTimeout = GLib.timeout_add_seconds(GLib.PRIORITY_DEFAULT,
60 * 10, Lang.bind(this, this._triggerFullResolve));
- this._runResolve();
- if (this._sourceSnapshotPath != null)
- this._runBuild();
this._updateStatus();
@@ -108,11 +99,20 @@ const Autobuilder = new Lang.Class({
_onTasksComplete: function() {
},
+ _onTaskExecuting: function(taskmaster, task) {
+ let workdir = task._workdir;
+ print("Task " + task.name + " executing in " + workdir.get_path());
+ this._updateStatus();
+ },
+
_onTaskCompleted: function(taskmaster, task, success, error) {
- if (task.name == this._resolveTaskName) {
- this._onResolveExited(task, success, error);
- } else if (task.name == this._buildTaskName) {
- this._onBuildExited(task, success, error);
+ if (task.name == 'resolve')
+ this._runResolve();
+ if (success) {
+ print("Task " + task.name + " complete: " + task._workdir.get_path());
+ } else {
+ this._failed = true;
+ print("Task " + task.name + " failed: " + task._workdir.get_path());
}
this._updateStatus();
},
@@ -161,68 +161,28 @@ const Autobuilder = new Lang.Class({
_runResolve: function() {
let cancellable = null;
- if (!(this._queuedForceResolve.length > 0 || this._fullResolveNeeded))
+ if (!(this._initialResolveNeeded ||
+ this._queuedForceResolve.length > 0 ||
+ this._fullResolveNeeded))
return;
- if (this._taskmaster.isTaskQueued(this._resolveTaskName))
+ if (this._taskmaster.isTaskQueued('resolve'))
return;
if (this._autoupdate_self)
ProcUtil.runSync(['git', 'pull', '-r'], cancellable)
- if (this._fullResolveNeeded) {
+ if (this._initialResolveNeeded) {
+ this._initialResolveNeeded = false;
+ this._taskmaster.pushTask('resolve', { });
+ } else if (this._fullResolveNeeded) {
this._fullResolveNeeded = false;
- this._taskmaster.pushTask(this._resolveTaskName,
- { fetchAll: true });
+ this._taskmaster.pushTask('resolve', { fetchAll: true });
} else {
- this._taskmaster.pushTask(this._resolveTaskName,
- { fetchComponents: this._queuedForceResolve });
+ this._taskmaster.pushTask('resolve', { fetchComponents: this._queuedForceResolve });
}
this._queuedForceResolve = [];
this._updateStatus();
- },
-
- _onResolveExited: function(resolveTask, success, msg) {
- print(Format.vprintf("resolve exited; success=%s msg=%s", [success, msg]))
- this._prevSourceSnapshotPath = this._sourceSnapshotPath;
- this._sourceSnapshotPath = this._src_db.getLatestPath();
- let changed = (this._prevSourceSnapshotPath == null ||
- !this._prevSourceSnapshotPath.equal(this._sourceSnapshotPath));
- if (changed)
- print(Format.vprintf("New version is %s", [this._sourceSnapshotPath.get_path()]))
- if (!this._buildNeeded)
- this._buildNeeded = changed;
- this._runBuild();
- this._runResolve();
- this._updateStatus();
- },
-
- _onBuildExited: function(buildTaskset, success, msg) {
- print(Format.vprintf("build exited; success=%s msg=%s", [success, msg]))
- if (this._buildNeeded)
- this._runBuild()
-
- this._updateStatus();
- },
-
- _runBuild: function() {
- let cancellable = null;
- if (this._taskmaster.isTaskQueued(this._buildTaskName))
- return;
- if (!this._buildNeeded)
- return;
-
- this._buildNeeded = false;
- this._taskmaster.pushTask(this._buildTaskName);
- this._updateStatus();
- },
-
- _runBdiff: function() {
- if (this._taskmaster.isTaskQueued(this._bdiffTaskName))
- return;
-
- this._taskmaster.pushTask(this._bdiffTaskName);
- this._updateStatus();
}
});
diff --git a/src/js/builtins/make.js b/src/js/builtins/make.js
index 93dcfae..3f91b2f 100644
--- a/src/js/builtins/make.js
+++ b/src/js/builtins/make.js
@@ -40,6 +40,8 @@ const Make = new Lang.Class({
_init: function() {
this.parent();
+ this.parser.addArgument(['-n', '--only'], { action: 'storeTrue',
+ help: "Don't process tasks after this" });
this.parser.addArgument('taskname');
this.parser.addArgument('parameters', { nargs: '*' });
},
@@ -48,8 +50,10 @@ const Make = new Lang.Class({
this._initWorkdir(null, cancellable);
this._loop = loop;
this._failed = false;
+ this._oneOnly = args.only;
let taskmaster = new Task.TaskMaster(this.workdir.get_child('tasks'),
- { onEmpty: Lang.bind(this, this._onTasksComplete) });
+ { onEmpty: Lang.bind(this, this._onTasksComplete),
+ processAfter: !args.only });
this._taskmaster = taskmaster;
taskmaster.connect('task-executing', Lang.bind(this, this._onTaskExecuting));
taskmaster.connect('task-complete', Lang.bind(this, this._onTaskCompleted));
@@ -74,13 +78,16 @@ const Make = new Lang.Class({
let workdir = task._workdir;
print("Task " + task.name + " executing in " + workdir.get_path());
let output = workdir.get_child('output.txt');
- let context = new GSystem.SubprocessContext({ argv: ['tail', '-f', output.get_path() ] });
- this._tail = new GSystem.Subprocess({ context: context });
- this._tail.init(null);
+ if (this._oneOnly) {
+ let context = new GSystem.SubprocessContext({ argv: ['tail', '-f', output.get_path() ] });
+ this._tail = new GSystem.Subprocess({ context: context });
+ this._tail.init(null);
+ }
},
_onTaskCompleted: function(taskmaster, task, success, error) {
- this._tail.request_exit();
+ if (this._oneOnly)
+ this._tail.request_exit();
if (success) {
print("Task " + task.name + " complete: " + task._workdir.get_path());
} else {
diff --git a/src/js/builtins/run_task.js b/src/js/builtins/run_task.js
index d76168f..d37ce40 100644
--- a/src/js/builtins/run_task.js
+++ b/src/js/builtins/run_task.js
@@ -46,9 +46,9 @@ const RunTask = new Lang.Class({
execute: function(args, loop, cancellable) {
let taskset = Task.TaskSet.prototype.getInstance();
- let [taskDef, vars] = taskset.getTask(args.taskName);
+ let taskDef = taskset.getTask(args.taskName);
let params = JSON.parse(args.parameters);
- let instance = new taskDef(null, args.taskName, vars, params);
+ let instance = new taskDef(null, args.taskName, [], params);
instance.execute(cancellable);
}
});
diff --git a/src/js/jsondb.js b/src/js/jsondb.js
index ece07d5..71013da 100644
--- a/src/js/jsondb.js
+++ b/src/js/jsondb.js
@@ -76,6 +76,13 @@ const JsonDB = new Lang.Class({
return this._path.get_child(all[0][3]);
},
+ getLatestVersion: function() {
+ let path = this.getLatestPath();
+ if (path == null)
+ return null;
+ return this.parseVersionStr(path.get_basename());
+ },
+
getPreviousPath: function(path) {
let name = path.get_basename();
let [target_major, target_minor] = this._parseVersion(name);
diff --git a/src/js/task.js b/src/js/task.js
index 757e139..d16516c 100644
--- a/src/js/task.js
+++ b/src/js/task.js
@@ -62,22 +62,31 @@ const TaskSet = new Lang.Class({
params = Params.parse(params, { allowNone: false })
for (let i = 0; i < this._tasks.length; i++) {
let taskDef = this._tasks[i];
- let pattern = taskDef.prototype.TaskPattern;
- let re = pattern[0];
- let match = re.exec(taskName);
- if (!match)
- continue;
- let vars = {};
- for (let i = 1; i < pattern.length; i++) {
- vars[pattern[i]] = match[i];
- }
- return [taskDef, vars];
+ let curName = taskDef.prototype.TaskName
+ if (curName == taskName)
+ return taskDef;
}
if (!params.allowNone)
throw new Error("No task definition matches " + taskName);
return null;
},
+ getTasksAfter: function(taskName) {
+ let ret = [];
+ for (let i = 0; i < this._tasks.length; i++) {
+ let taskDef = this._tasks[i];
+ let after = taskDef.prototype.TaskAfter;
+ for (let j = 0; j < after.length; j++) {
+ let a = after[j];
+ if (a == taskName) {
+ ret.push(taskDef);
+ break;
+ }
+ }
+ }
+ return ret;
+ },
+
getInstance: function() {
if (!_tasksetInstance)
_tasksetInstance = new TaskSet();
@@ -89,8 +98,10 @@ const TaskMaster = new Lang.Class({
Name: 'TaskMaster',
_init: function(path, params) {
- params = Params.parse(params, {onEmpty: null});
+ params = Params.parse(params, { onEmpty: null,
+ processAfter: true });
this.path = path;
+ this._processAfter = params.processAfter;
this.maxConcurrent = GLib.get_num_processors();
this._onEmpty = params.onEmpty;
this.cancellable = null;
@@ -102,23 +113,36 @@ const TaskMaster = new Lang.Class({
this._caughtError = false;
this._taskset = TaskSet.prototype.getInstance();
+
+ this._taskVersions = {};
+ },
+
+ _pushTaskDef: function(taskDef, parameters) {
+ let name = taskDef.prototype.TaskName;
+ if (!this._isTaskPending(name)) {
+ let instance = new taskDef(this, name, [], parameters);
+ instance.onComplete = Lang.bind(this, this._onComplete, instance);
+ this._pendingTasksList.push(instance);
+ this._queueRecalculate();
+ }
},
pushTask: function(taskName, parameters) {
- let [taskDef, vars] = this._taskset.getTask(taskName);
- let instance = new taskDef(this, taskName, vars, parameters);
- instance.onComplete = Lang.bind(this, this._onComplete, instance);
- this._pendingTasksList.push(instance);
- this._queueRecalculate();
+ let taskDef = this._taskset.getTask(taskName);
+ this._pushTaskDef(taskDef, parameters);
},
- isTaskQueued: function(taskName) {
+ _isTaskPending: function(taskName) {
for (let i = 0; i < this._pendingTasksList.length; i++) {
let pending = this._pendingTasksList[i];
if (pending.name == taskName)
return true;
}
- return this.isTaskExecuting(taskName);
+ return false;
+ },
+
+ isTaskQueued: function(taskName) {
+ return this._isTaskPending(taskName) || this.isTaskExecuting(taskName);
},
isTaskExecuting: function(taskName) {
@@ -162,7 +186,6 @@ const TaskMaster = new Lang.Class({
},
_onComplete: function(success, error, task) {
- this.emit('task-complete', task, success, error);
let idx = -1;
for (let i = 0; i < this._executing.length; i++) {
let executingTask = this._executing[i];
@@ -174,6 +197,24 @@ const TaskMaster = new Lang.Class({
if (idx == -1)
throw new Error("TaskMaster: Internal error - Failed to find completed task:" + task.TaskName);
this._executing.splice(idx, 1);
+ this.emit('task-complete', task, success, error);
+ if (this._processAfter) {
+ let changed = true;
+ let version = task.queryVersion();
+ if (version !== null) {
+ let oldVersion = this._taskVersions[task.name];
+ if (oldVersion == version)
+ changed = false;
+ else if (oldVersion != null)
+ print("task " + task.name + " new version: " + version);
+ }
+ if (changed) {
+ let tasksAfter = this._taskset.getTasksAfter(task.name);
+ for (let i = 0; i < tasksAfter.length; i++) {
+ this._pushTaskDef(tasksAfter[i], {});
+ }
+ }
+ }
this._queueRecalculate();
},
@@ -182,6 +223,10 @@ const TaskMaster = new Lang.Class({
this._pendingTasksList.length > 0 &&
!this.isTaskExecuting(this._pendingTasksList[0].name)) {
let task = this._pendingTasksList.shift();
+ let version = task.queryVersion();
+ if (version !== null) {
+ this._taskVersions[task.name] = version;
+ }
task._executeInSubprocessInternal(this.cancellable);
this.emit('task-executing', task);
this._executing.push(task);
@@ -194,6 +239,7 @@ const TaskDef = new Lang.Class({
Name: 'TaskDef',
TaskPattern: null,
+ TaskAfter: [],
PreserveStdout: true,
RetainFailed: 1,
@@ -260,6 +306,10 @@ const TaskDef = new Lang.Class({
}
},
+ queryVersion: function() {
+ return null;
+ },
+
execute: function(cancellable) {
throw new Error("Not implemented");
},
diff --git a/src/js/tasks/task-bdiff.js b/src/js/tasks/task-bdiff.js
index 91993e3..a42303c 100644
--- a/src/js/tasks/task-bdiff.js
+++ b/src/js/tasks/task-bdiff.js
@@ -37,9 +37,8 @@ const TaskBdiff = new Lang.Class({
Name: "TaskBdiff",
Extends: Task.TaskDef,
- TaskPattern: [/bdiff$/],
-
- TaskAfterPrefix: '/build/',
+ TaskName: "bdiff",
+ TaskAfter: ['build'],
_gitLogToJson: function(repoDir, specification) {
let log = ProcUtil.runSyncGetOutputLines(['git', 'log', '--format=email', specification],
diff --git a/src/js/tasks/task-build.js b/src/js/tasks/task-build.js
index 44c14b5..f7f6b54 100644
--- a/src/js/tasks/task-build.js
+++ b/src/js/tasks/task-build.js
@@ -46,9 +46,8 @@ const TaskBuild = new Lang.Class({
Name: "TaskBuild",
Extends: Task.TaskDef,
- TaskPattern: [/build$/],
-
- TaskAfterPrefix: '/resolve/',
+ TaskName: "build",
+ TaskAfter: ['resolve'],
_resolveRefs: function(refs) {
if (refs.length == 0)
diff --git a/src/js/tasks/task-builddisks.js b/src/js/tasks/task-builddisks.js
index 39e2330..661907a 100644
--- a/src/js/tasks/task-builddisks.js
+++ b/src/js/tasks/task-builddisks.js
@@ -39,9 +39,8 @@ const TaskBuildDisks = new Lang.Class({
Name: 'TaskBuildDisks',
Extends: Task.TaskDef,
- TaskPattern: [/builddisks$/],
-
- TaskAfterPrefix: '/build/',
+ TaskName: "builddisks",
+ TaskAfter: ['build'],
// Legacy
_VERSION_RE: /^(\d+)\.(\d+)$/,
diff --git a/src/js/tasks/task-resolve.js b/src/js/tasks/task-resolve.js
index 7c06d69..f43ac46 100644
--- a/src/js/tasks/task-resolve.js
+++ b/src/js/tasks/task-resolve.js
@@ -36,12 +36,24 @@ const TaskResolve = new Lang.Class({
Name: "TaskResolve",
Extends: Task.TaskDef,
- TaskPattern: [/resolve$/],
+ TaskName: "resolve",
DefaultParameters: {fetchAll: false,
fetchComponents: [],
timeoutSec: 10},
+ _getDb: function() {
+ if (this._db == null) {
+ let snapshotdir = this.workdir.get_child('snapshots');
+ this._db = new JsonDB.JsonDB(snapshotdir);
+ }
+ return this._db;
+ },
+
+ queryVersion: function() {
+ return this._getDb().getLatestVersion();
+ },
+
execute: function(cancellable) {
let manifestPath = this.workdir.get_child('manifest.json');
let data = JsonUtil.loadJson(manifestPath, cancellable);
@@ -70,9 +82,7 @@ const TaskResolve = new Lang.Class({
component['revision'] = revision;
}
- let snapshotdir = this.workdir.get_child('snapshots');
- this._src_db = new JsonDB.JsonDB(snapshotdir);
- let [path, modified] = this._src_db.store(this._snapshot.data, cancellable);
+ let [path, modified] = this._getDb().store(this._snapshot.data, cancellable);
if (modified) {
print("New source snapshot: " + path.get_path());
} else {
diff --git a/src/js/tasks/task-smoketest.js b/src/js/tasks/task-smoketest.js
index 525178c..1a15c8a 100644
--- a/src/js/tasks/task-smoketest.js
+++ b/src/js/tasks/task-smoketest.js
@@ -220,7 +220,8 @@ const TaskSmoketest = new Lang.Class({
Name: 'TaskSmoketest',
Extends: Task.TaskDef,
- TaskPattern: [/smoketest$/],
+ TaskName: "smoketest",
+ TaskAfter: ['builddisks'],
execute: function(cancellable) {
let imageDir = this.workdir.get_child('images');
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
