[gnome-ostree/wip/dyntask-2] wip
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-ostree/wip/dyntask-2] wip
- Date: Fri, 1 Feb 2013 02:14:20 +0000 (UTC)
commit 953cb4c541adf5c3b5a6b103ccc39e204b9adba6
Author: Colin Walters <walters verbum org>
Date: Thu Jan 24 09:13:51 2013 -0500
wip
Makefile-ostbuild.am | 5 +-
src/ostbuild/js/buildutil.js | 27 ++
src/ostbuild/js/builtins/make.js | 65 ++++
src/ostbuild/js/builtins/run_task.js | 57 ++++
src/ostbuild/js/dyntask.js | 314 +++++++++++++++----
src/ostbuild/js/jsonutil.js | 38 +++
src/ostbuild/js/main.js | 2 +
src/ostbuild/js/subtask.js | 34 +-
.../js/{builtins/build.js => tasks/task-build.js} | 101 +++----
src/ostbuild/js/tasks/task-checksum.js | 123 --------
src/ostbuild/ostbuild.in | 2 +-
11 files changed, 501 insertions(+), 267 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index ef0ef1f..7ee25f1 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -60,21 +60,22 @@ jsostbuild_DATA= \
jsostbuiltinsdir=$(jsostbuilddir)/builtins
jsostbuiltins_DATA= \
src/ostbuild/js/builtins/autobuilder.js \
- src/ostbuild/js/builtins/build.js \
src/ostbuild/js/builtins/build_disks.js \
src/ostbuild/js/builtins/checkout.js \
src/ostbuild/js/builtins/git_mirror.js \
+ src/ostbuild/js/builtins/make.js \
src/ostbuild/js/builtins/qa_make_disk.js \
src/ostbuild/js/builtins/qa_pull_deploy.js \
src/ostbuild/js/builtins/qa_smoketest.js \
src/ostbuild/js/builtins/prefix.js \
src/ostbuild/js/builtins/resolve.js \
+ src/ostbuild/js/builtins/run_task.js \
src/ostbuild/js/builtins/shell.js \
$(NULL)
jsosttasksdir=$(jsostbuilddir)/tasks
jsosttasks_DATA= \
- src/ostbuild/js/tasks/task-checksum.js \
+ src/ostbuild/js/tasks/task-build.js \
$(NULL)
endif
diff --git a/src/ostbuild/js/buildutil.js b/src/ostbuild/js/buildutil.js
index 0f3d1c7..7246e5c 100644
--- a/src/ostbuild/js/buildutil.js
+++ b/src/ostbuild/js/buildutil.js
@@ -127,3 +127,30 @@ function getBaseUserChrootArgs() {
let path = findUserChrootPath();
return [path.get_path(), '--unshare-pid', '--unshare-ipc', '--unshare-net'];
}
+
+function compareVersions(a, b) {
+ let adot = a.indexOf('.');
+ while (adot != -1) {
+ let bdot = b.indexOf('.');
+ if (bdot == -1)
+ return 1;
+ let aSub = parseInt(a.substr(0, adot));
+ let bSub = parseInt(b.substr(0, bdot));
+ if (aSub > bSub)
+ return 1;
+ else if (aSub < bSub)
+ return -1;
+ a = a.substr(adot + 1);
+ b = b.substr(bdot + 1);
+ adot = a.indexOf('.');
+ }
+ if (b.indexOf('.') != -1)
+ return -1;
+ let aSub = parseInt(a);
+ let bSub = parseInt(b);
+ if (aSub > bSub)
+ return 1;
+ else if (aSub < bSub)
+ return -1;
+ return 0;
+}
diff --git a/src/ostbuild/js/builtins/make.js b/src/ostbuild/js/builtins/make.js
new file mode 100644
index 0000000..2f39876
--- /dev/null
+++ b/src/ostbuild/js/builtins/make.js
@@ -0,0 +1,65 @@
+// Copyright (C) 2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const Builtin = imports.builtin;
+const DynTask = imports.dyntask;
+const JsonDB = imports.jsondb;
+const ProcUtil = imports.procutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const Config = imports.config;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const Make = new Lang.Class({
+ Name: 'Make',
+ Extends: Builtin.Builtin,
+
+ DESCRIPTION: "Execute tasks",
+
+ _init: function() {
+ this.parent();
+ this.parser.addArgument('tasknames', { nargs: '*' });
+ },
+
+ execute: function(args, loop, cancellable) {
+ this._loop = loop;
+ this._err = null;
+ let taskmaster = new DynTask.TaskMaster(this.workdir.get_child('tasks'),
+ { onEmpty: Lang.bind(this, this._onTasksComplete) });
+ taskmaster.pushTasks(args.tasknames);
+ loop.run();
+ if (this._err)
+ throw new Error("Error: " + this._err);
+ else
+ print("Success!")
+ },
+
+ _onTasksComplete: function(success, err) {
+ if (!success)
+ this._err = err;
+ this._loop.quit();
+ }
+});
diff --git a/src/ostbuild/js/builtins/run_task.js b/src/ostbuild/js/builtins/run_task.js
new file mode 100644
index 0000000..7b26118
--- /dev/null
+++ b/src/ostbuild/js/builtins/run_task.js
@@ -0,0 +1,57 @@
+// Copyright (C) 2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const Builtin = imports.builtin;
+const DynTask = imports.dyntask;
+const JsonDB = imports.jsondb;
+const ProcUtil = imports.procutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const Config = imports.config;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const RunTask = new Lang.Class({
+ Name: 'RunTask',
+ Extends: Builtin.Builtin,
+
+ DESCRIPTION: "Internal helper to execute a task",
+
+ _init: function() {
+ this.parent();
+ this.parser.addArgument('taskName');
+ this.parser.addArgument('outfd');
+ },
+
+ execute: function(args, loop, cancellable) {
+ let outstream = Gio.UnixOutputStream.new(parseInt(args.outfd), true);
+ let taskset = DynTask.TaskSet.prototype.getInstance();
+ let [taskDef, inputs] = taskset.getTask(args.taskName);
+ let instance = new taskDef(null, args.taskName, inputs);
+ let result = instance.executeSync(cancellable);
+ JsonUtil.writeJsonToStream(outstream, result || {}, cancellable);
+ outstream.close(cancellable);
+ }
+});
diff --git a/src/ostbuild/js/dyntask.js b/src/ostbuild/js/dyntask.js
index a99cab3..fafa130 100644
--- a/src/ostbuild/js/dyntask.js
+++ b/src/ostbuild/js/dyntask.js
@@ -22,28 +22,18 @@ const Lang = imports.lang;
const GSystem = imports.gi.GSystem;
const Params = imports.params;
+const JsonUtil = imports.jsonutil;
+const ProcUtil = imports.procutil;
+const BuildUtil = imports.buildutil;
const VERSION_RE = /(\d+)\.(\d+)/;
-const TaskMaster = new Lang.Class({
- Name: 'TaskMaster',
-
- _init: function(path, params) {
- params = Params.parse(params, {maxConcurrent: 4,
- onEmpty: null});
- this.path = path;
- this.maxConcurrent = params.maxConcurrent;
- this._onEmpty = params.onEmpty;
- this.cancellable = null;
- this._idleRecalculateId = 0;
- this._taskSerial = 1;
+var _tasksetInstance = null;
+const TaskSet = new Lang.Class({
+ Name: 'TaskSet',
+
+ _init: function() {
this._tasks = [];
- this._executing = [];
- this._pendingTasksList = [];
- this._seenTasks = {};
- this._completeTasks = {};
- this._taskErrors = {};
-
let taskdir = Gio.File.new_for_path(GLib.getenv('OSTBUILD_DATADIR')).resolve_relative_path('js/tasks');
let denum = taskdir.enumerate_children('standard::*', 0, null);
let finfo;
@@ -54,8 +44,7 @@ const TaskMaster = new Lang.Class({
if (defname.indexOf('Task') !== 0)
continue;
let cls = taskMod[defname];
- let instance = new cls;
- this.register(instance);
+ this.register(cls);
}
}
},
@@ -64,52 +53,85 @@ const TaskMaster = new Lang.Class({
this._tasks.push(taskdef);
},
- _pushRecurse: function(taskName, seen) {
- if (seen[taskName])
- return null;
- let result = null;
+ getAllTasks: function() {
+ return this._tasks;
+ },
+
+ getTask: function(taskName, params) {
+ params = Params.parse(params, { allowNone: false })
for (let i = 0; i < this._tasks.length; i++) {
let taskDef = this._tasks[i];
- let pattern = taskDef.getPattern();
+ let pattern = taskDef.prototype.TaskPattern;
let re = pattern[0];
let match = re.exec(taskName);
if (!match)
continue;
-
- let serial = this._taskSerial;
- this._taskSerial++;
let vars = {};
for (let i = 1; i < pattern.length; i++) {
vars[pattern[i]] = match[i];
}
- let specifiedDependencies = taskDef.getDepends(vars);;
- let waitingDependencies = {};
- for (let j = 0; j < specifiedDependencies.length; j++) {
- let depName = specifiedDependencies[j];
- if (!this._completeTasks[depName]) {
- let depTask = this._pushRecurse(depName, seen);
- waitingDependencies[depName] = depTask;
- }
- }
- result = {name: taskName,
- def: taskDef,
- vars: vars,
- dependencies: specifiedDependencies,
- waitingDependencies: waitingDependencies,
- serial: serial,
- result: null };
- this._pendingTasksList.push(result);
- seen[taskName] = true;
- break;
+ return [taskDef, vars];
}
- if (!result)
+ if (!params.allowNone)
throw new Error("No task definition matches " + taskName);
+ return null;
+ },
+
+ getInstance: function() {
+ if (!_tasksetInstance)
+ _tasksetInstance = new TaskSet();
+ return _tasksetInstance;
+ }
+});
+
+const TaskMaster = new Lang.Class({
+ Name: 'TaskMaster',
+
+ _init: function(path, params) {
+ params = Params.parse(params, {maxConcurrent: 4,
+ onEmpty: null});
+ this.path = path;
+ this.maxConcurrent = params.maxConcurrent;
+ this._onEmpty = params.onEmpty;
+ this.cancellable = null;
+ this._idleRecalculateId = 0;
+ this._executing = [];
+ this._pendingTasksList = [];
+ this._seenTasks = {};
+ this._completeTasks = {};
+ this._taskErrors = {};
+ this._caughtError = false;
+
+ this._taskset = TaskSet.prototype.getInstance();
+ },
+
+ _pushRecurse: function(taskName, seen) {
+ if (seen[taskName])
+ return null;
+ let [taskDef, inputs] = this._taskset.getTask(taskName);
+ let specifiedDependencies = taskDef.prototype.getDepends(inputs);
+ let waitingDependencies = {};
+ for (let j = 0; j < specifiedDependencies.length; j++) {
+ let depName = specifiedDependencies[j];
+ if (!this._completeTasks[depName]) {
+ let depTask = this._pushRecurse(depName, seen);
+ waitingDependencies[depName] = depTask;
+ }
+ }
+ let instance = new taskDef(this, taskName, inputs);
+ instance.onComplete = Lang.bind(this, this._onComplete, instance);
+ instance.dependencies = specifiedDependencies;
+ instance.waitingDependencies = waitingDependencies;
+ this._pendingTasksList.push(instance);
+ seen[taskName] = true;
this._queueRecalculate();
- return result;
+ return instance;
},
- push: function(taskName) {
- return this._pushRecurse(taskName, {});
+ pushTasks: function(taskNames) {
+ let seen = {};
+ for (let i = 0; i < taskNames.length; i++)
+ this._pushRecurse(taskNames[i], seen);
},
_queueRecalculate: function() {
@@ -137,7 +159,7 @@ const TaskMaster = new Lang.Class({
if (this._executing.length == 0 &&
this._pendingTasksList.length == 0) {
- this._onEmpty();
+ this._onEmpty(true, null);
return;
} else if (this._pendingTasksList.length == 0) {
return;
@@ -156,20 +178,24 @@ const TaskMaster = new Lang.Class({
_onComplete: function(result, error, task) {
if (error) {
print("TaskMaster: While executing " + task.name + ": " + error);
- this._taskErrors[task.name] = error;
+ if (!this._caughtError) {
+ this._caughtError = true;
+ this._onEmpty(false, error);
+ }
+ return;
} else {
print("TaskMaster: Completed: " + task.name + " : " + JSON.stringify(result));
}
let idx = -1;
for (let i = 0; i < this._executing.length; i++) {
let executingTask = this._executing[i];
- if (executingTask.serial != task.serial)
+ if (executingTask !== task)
continue;
idx = i;
break;
}
if (idx == -1)
- throw new Error("TaskMaster: Internal error - Failed to find completed task serial:" + task.serial);
+ throw new Error("TaskMaster: Internal error - Failed to find completed task:" + task.name);
task.result = result;
this._completeTasks[task.name] = task;
this._executing.splice(idx, 1);
@@ -197,12 +223,8 @@ const TaskMaster = new Lang.Class({
!this._hasDeps(this._pendingTasksList[0])) {
let task = this._pendingTasksList.shift();
print("TaskMaster: running: " + task.name);
- let depResults = [];
- for (let i = 0; i < task.dependencies.length; i++) {
- let depName = task.dependencies[i];
- depResults.push(this._completeTasks[depName].result);
- }
- task.def.execute(task.vars, depResults, this.cancellable, Lang.bind(this, this._onComplete, task));
+ let cls = task.def;
+ task.execute(this.cancellable);
this._executing.push(task);
}
}
@@ -211,22 +233,182 @@ const TaskMaster = new Lang.Class({
const TaskDef = new Lang.Class({
Name: 'TaskDef',
- _init: function() {
- },
+ TaskPattern: null,
- getPattern: function() {
- throw new Error("Not implemented");
+ _init: function(taskmaster, name, inputs) {
+ this.taskmaster = taskmaster;
+ this.name = name;
+ this.inputs = inputs;
},
getDepends: function(inputs) {
return [];
},
- execute: function(inputs, dependResults, cancellable, onComplete) {
+ execute: function(cancellable) {
throw new Error("Not implemented");
}
});
+const SubTaskDef = new Lang.Class({
+ Name: 'SubTaskDef',
+ Extends: TaskDef,
+
+ PreserveStdout: true,
+ RetainFailed: 1,
+ RetainSuccess: 5,
+
+ _VERSION_RE: /(\d+\d\d\d\d)\.(\d+)/,
+
+ _loadVersionsFrom: function(dir, cancellable) {
+ let e = dir.enumerate_children('standard::*', Gio.FileQueryInfoFlags.NONE, cancellable);
+ let info;
+ let results = [];
+ while ((info = e.next_file(cancellable)) != null) {
+ let name = info.get_name();
+ let match = this._VERSION_RE.exec(name);
+ if (!match)
+ continue;
+ results.push(name);
+ }
+ results.sort(BuildUtil.compareVersions);
+ return results;
+ },
+
+ _cleanOldVersions: function(dir, retain, cancellable) {
+ let versions = this._loadVersionsFrom(dir, cancellable);
+ while (versions.length > retain) {
+ let child = dir.get_child(versions.shift());
+ GSystem.shutil_rm_rf(child, cancellable);
+ }
+ },
+
+ executeSync: function(cancellable) {
+ throw new Error("Not implemented");
+ },
+
+ execute: function(cancellable) {
+ this._asyncOutstanding = 0;
+ this._cancellable = cancellable;
+
+ this._subtaskdir = this.taskmaster.path.resolve_relative_path(this.name.substr(1));
+ GSystem.file_ensure_directory(this._subtaskdir, true, cancellable);
+
+ let allVersions = [];
+
+ this._successDir = this._subtaskdir.get_child('successful');
+ GSystem.file_ensure_directory(this._successDir, true, cancellable);
+ let successVersions = this._loadVersionsFrom(this._successDir, cancellable);
+ for (let i = 0; i < successVersions.length; i++) {
+ allVersions.push([true, successVersions[i]]);
+ }
+
+ this._failedDir = this._subtaskdir.get_child('failed');
+ GSystem.file_ensure_directory(this._failedDir, true, cancellable);
+ let failedVersions = this._loadVersionsFrom(this._failedDir, cancellable);
+ for (let i = 0; i < failedVersions.length; i++) {
+ allVersions.push([false, failedVersions[i]]);
+ }
+
+ allVersions.sort(function (a, b) {
+ let [successA, versionA] = a;
+ let [successB, versionB] = b;
+ return BuildUtil.compareVersions(versionA, versionB);
+ });
+
+ let currentTime = GLib.DateTime.new_now_utc();
+
+ let currentYmd = Format.vprintf('%d%02d%02d', [currentTime.get_year(),
+ currentTime.get_month(),
+ currentTime.get_day_of_month()]);
+ let version = null;
+ if (allVersions.length > 0) {
+ let [lastSuccess, lastVersion] = allVersions[allVersions.length-1];
+ let match = this._VERSION_RE.exec(lastVersion);
+ if (!match) throw new Error();
+ let lastYmd = match[1];
+ let lastSerial = match[2];
+ if (lastYmd == currentYmd) {
+ version = currentYmd + '.' + (parseInt(lastSerial) + 1);
+ }
+ }
+ if (version === null) {
+ version = currentYmd + '.0';
+ }
+
+ this._workdir = this._subtaskdir.get_child(version);
+ GSystem.shutil_rm_rf(this._workdir, cancellable);
+ GSystem.file_ensure_directory(this._workdir, true, cancellable);
+
+ let baseArgv = ['ostbuild', 'run-task', this.name];
+ let context = new GSystem.SubprocessContext({ argv: baseArgv });
+ context.set_cwd(this._workdir.get_path());
+ context.set_stdin_disposition(GSystem.SubprocessStreamDisposition.PIPE);
+ if (this.PreserveStdout) {
+ let outPath = this._workdir.get_child('output.txt');
+ context.set_stdout_file_path(outPath.get_path());
+ context.set_stderr_disposition(GSystem.SubprocessStreamDisposition.STDERR_MERGE);
+ } else {
+ context.set_stdout_disposition(GSystem.SubprocessStreamDisposition.NULL);
+ let errPath = this._workdir.get_child('errors.txt');
+ context.set_stderr_file_path(errPath.get_path());
+ }
+ let [success, resultpipe, fdno] = context.open_pipe_read(cancellable);
+ context.argv_append(''+fdno);
+ this._proc = new GSystem.Subprocess({ context: context });
+ this._proc.init(cancellable);
+
+ this._proc.wait(cancellable, Lang.bind(this, this._onChildExited));
+ this._asyncOutstanding += 1;
+
+ this._result = null;
+ this._error = null;
+ JsonUtil.loadJsonFromStreamAsync(resultpipe, cancellable, Lang.bind(this, this._onResultRead));
+ this._asyncOutstanding += 1;
+ },
+
+ _onAsyncOpComplete: function(error) {
+ this._asyncOutstanding--;
+ if (error && !this._error)
+ this._error = error;
+ if (this._asyncOutstanding != 0)
+ return;
+ if (this._error) {
+ this.onComplete(null, this._error);
+ } else {
+ this.onComplete(this._result, null);
+ }
+ },
+
+ _onChildExited: function(proc, result) {
+ let [success, errmsg] = ProcUtil.asyncWaitCheckFinish(proc, result);
+ let target;
+ if (!success) {
+ target = this._failedDir.get_child(this._workdir.get_basename());
+ GSystem.file_rename(this._workdir, target, null);
+ this._cleanOldVersions(this._failedDir, this.RetainFailed, null);
+ this._onAsyncOpComplete(new Error(errmsg));
+ } else {
+ target = this._successDir.get_child(this._workdir.get_basename());
+ GSystem.file_rename(this._workdir, target, null);
+ this._cleanOldVersions(this._successDir, this.RetainSuccess, null);
+ this._onAsyncOpComplete(null);
+ }
+ print("Stored results of " + this.name + " in " + target.get_path());
+ },
+
+ _onResultRead: function(result, error) {
+ if (!error) {
+ let childResult = result['result'];
+ if (childResult)
+ this._result = childResult;
+ this._onAsyncOpComplete(null);
+ } else {
+ this._onAsyncOpComplete(error);
+ }
+ }
+});
+
function demo(argv) {
var loop = GLib.MainLoop.new(null, true);
let ecode = 1;
diff --git a/src/ostbuild/js/jsonutil.js b/src/ostbuild/js/jsonutil.js
index 04a82ca..24deb88 100644
--- a/src/ostbuild/js/jsonutil.js
+++ b/src/ostbuild/js/jsonutil.js
@@ -27,6 +27,44 @@ function writeJsonToStream(stream, data, cancellable) {
stream.write_bytes(new GLib.Bytes(buf), cancellable);
}
+function writeJsonToStreamAsync(stream, data, cancellable, onComplete) {
+ let buf = JSON.stringify(data, null, " ");
+ stream.write_bytes_async(new GLib.Bytes(buf), GLib.PRIORITY_DEFAULT,
+ cancellable, function(stream, result) {
+ let err = null;
+ try {
+ stream.write_bytes_finish(result);
+ } catch (e) {
+ err = e;
+ }
+ onComplete(err != null, err);
+ });
+}
+
+function loadJsonFromStream(stream, cancellable) {
+ let membuf = Gio.MemoryOutputStream.new_resizable();
+ membuf.splice(stream, Gio.OutputStreamSpliceFlags.CLOSE_TARGET, cancellable);
+ let bytes = membuf.steal_as_bytes();
+ return JSON.parse(bytes.toArray().toString());
+}
+
+function loadJsonFromStreamAsync(stream, cancellable, onComplete) {
+ let membuf = Gio.MemoryOutputStream.new_resizable();
+ membuf.splice_async(stream, Gio.OutputStreamSpliceFlags.CLOSE_TARGET, GLib.PRIORITY_DEFAULT,
+ cancellable, function(stream, result) {
+ let err = null;
+ let res = null;
+ try {
+ stream.splice_finish(result);
+ let bytes = membuf.steal_as_bytes();
+ res = JSON.parse(bytes.toArray().toString());
+ } catch (e) {
+ err = e;
+ }
+ onComplete(res, err);
+ });
+}
+
function writeJsonFileAtomic(path, data, cancellable) {
let s = path.replace(null, false, Gio.FileCreateFlags.REPLACE_DESTINATION, cancellable);
writeJsonToStream(s, data, cancellable);
diff --git a/src/ostbuild/js/main.js b/src/ostbuild/js/main.js
index ae92800..0f1e2e2 100755
--- a/src/ostbuild/js/main.js
+++ b/src/ostbuild/js/main.js
@@ -26,7 +26,9 @@ const BUILTINS = ['autobuilder',
'resolve',
'build',
'build-disks',
+ 'make',
'shell',
+ 'run-task',
'qa-make-disk',
'qa-pull-deploy',
'qa-smoketest'];
diff --git a/src/ostbuild/js/subtask.js b/src/ostbuild/js/subtask.js
index ddab482..dca1b7a 100644
--- a/src/ostbuild/js/subtask.js
+++ b/src/ostbuild/js/subtask.js
@@ -92,6 +92,23 @@ const TaskSet = new Lang.Class({
this._load();
},
+ _load: function() {
+ var e = this.path.enumerate_children('standard::*', Gio.FileQueryInfoFlags.NONE, null);
+ let info;
+ let history = [];
+ while ((info = e.next_file(null)) != null) {
+ let name = info.get_name();
+ let childPath = this.path.get_child(name);
+ let match = VERSION_RE.exec(name);
+ if (!match)
+ continue;
+ history.push(new TaskHistoryEntry(childPath))
+ }
+ history.sort(TaskHistoryEntry.prototype.compareTo);
+ this._history = history;
+ this._cleanOldEntries();
+ },
+
_cleanOldEntries: function() {
while (this._history.length > this._maxVersions) {
let task = this._history.shift();
@@ -99,23 +116,6 @@ const TaskSet = new Lang.Class({
}
},
- _load: function() {
- var e = this.path.enumerate_children('standard::*', Gio.FileQueryInfoFlags.NONE, null);
- let info;
- let history = [];
- while ((info = e.next_file(null)) != null) {
- let name = info.get_name();
- let childPath = this.path.get_child(name);
- let match = VERSION_RE.exec(name);
- if (!match)
- continue;
- history.push(new TaskHistoryEntry(childPath))
- }
- history.sort(TaskHistoryEntry.prototype.compareTo);
- this._history = history;
- this._cleanOldEntries();
- },
-
_onProcessComplete: function(proc, result) {
if (!this._running) throw new Error();
diff --git a/src/ostbuild/js/builtins/build.js b/src/ostbuild/js/tasks/task-build.js
similarity index 92%
rename from src/ostbuild/js/builtins/build.js
rename to src/ostbuild/js/tasks/task-build.js
index 7c54a8a..5040dbf 100644
--- a/src/ostbuild/js/builtins/build.js
+++ b/src/ostbuild/js/tasks/task-build.js
@@ -23,7 +23,7 @@ const Format = imports.format;
const GSystem = imports.gi.GSystem;
const Builtin = imports.builtin;
-const SubTask = imports.subtask;
+const DynTask = imports.dyntask;
const JsonDB = imports.jsondb;
const ProcUtil = imports.procutil;
const StreamUtil = imports.streamutil;
@@ -35,13 +35,13 @@ const Vcs = imports.vcs;
const ArgParse = imports.argparse;
const OPT_COMMON_CFLAGS = {'i686': '-O2 -g -m32 -march=i686 -mtune=atom -fasynchronous-unwind-tables',
- 'x86_64': '-O2 -g -m64 -mtune=generic'}
+ 'x86_64': '-O2 -g -m64 -mtune=generic'};
-const Build = new Lang.Class({
- Name: "Build",
- Extends: Builtin.Builtin,
+const TaskBuild = new Lang.Class({
+ Name: "TaskBuild",
+ Extends: DynTask.SubTaskDef,
- DESCRIPTION: "Build multiple components and generate trees",
+ TaskPattern: [/build\/(.*)$/, 'PREFIX'],
_resolveRefs: function(refs) {
if (refs.length == 0)
@@ -290,6 +290,7 @@ const Build = new Lang.Class({
let prefix = this._snapshot.data['prefix'];
let buildname = Format.vprintf('%s/%s/%s', [prefix, basename, architecture]);
+ let unixBuildname = buildname.replace(/\//g, '_');
let buildRef = 'components/' + buildname;
let currentVcsVersion = component['revision'];
@@ -320,16 +321,13 @@ const Build = new Lang.Class({
let patchdir;
if (expandedComponent['patches']) {
let patchesRevision = expandedComponent['patches']['revision'];
- if (this.args.patches_path) {
- patchdir = Gio.File.new_for_path(this.args.patches_path);
- } else if (this._cachedPatchdirRevision == patchesRevision) {
+ if (this._cachedPatchdirRevision == patchesRevision) {
patchdir = this.patchdir;
} else {
patchdir = Vcs.checkoutPatches(this.mirrordir,
this.patchdir,
expandedComponent,
- cancellable,
- {patchesPath: this.args.patches_path});
+ cancellable);
this._cachedPatchdirRevision = patchesRevision;
}
if ((previousMetadata != null) &&
@@ -366,33 +364,31 @@ const Build = new Lang.Class({
}
}
- let taskdir = this.workdir.get_child('tasks');
- let buildTaskset = new SubTask.TaskSet(taskdir.get_child(buildname));
+ let cwd = Gio.File.new_for_path('.');
+ let buildWorkdir = cwd.get_child('tmp-' + unixBuildname);
+ GSystem.file_ensure_directory(buildWorkdir, true, cancellable);
- let workdir = buildTaskset.prepare();
-
- let tempMetadataPath = workdir.get_child('_ostbuild-meta.json');
+ let tempMetadataPath = buildWorkdir.get_child('_ostbuild-meta.json');
JsonUtil.writeJsonFileAtomic(tempMetadataPath, expandedComponent, cancellable);
- let componentSrc = workdir.get_child(basename);
+ let componentSrc = buildWorkdir.get_child(basename);
let childArgs = ['ostbuild', 'checkout', '--snapshot=' + this._snapshot.path.get_path(),
'--checkoutdir=' + componentSrc.get_path(),
'--metadata-path=' + tempMetadataPath.get_path(),
'--overwrite', basename];
- if (this.args.patches_path)
- childArgs.push('--patches-path=' + this.args.patches_path);
- else if (patchdir)
+ if (patchdir) {
childArgs.push('--patches-path=' + patchdir.get_path());
- ProcUtil.runSync(childArgs, cancellable, { logInitiation: true });
+ }
+ ProcUtil.runSync(childArgs, cancellable);
GSystem.file_unlink(tempMetadataPath, cancellable);
- let componentResultdir = workdir.get_child('results');
+ let componentResultdir = buildWorkdir.get_child('results');
GSystem.file_ensure_directory(componentResultdir, true, cancellable);
- let rootdir = this._composeBuildroot(workdir, basename, architecture, cancellable);
+ let rootdir = this._composeBuildroot(buildWorkdir, basename, architecture, cancellable);
- let tmpdir=workdir.get_child('tmp');
+ let tmpdir=buildWorkdir.get_child('tmp');
GSystem.file_ensure_directory(tmpdir, true, cancellable);
let srcCompileOnePath = this.libdir.get_child('ostree-build-compile-one');
@@ -424,26 +420,11 @@ const Build = new Lang.Class({
let context = new GSystem.SubprocessContext({ argv: childArgs });
context.set_environment(ProcUtil.objectToEnvironment(envCopy));
-
- let mainContext = new GLib.MainContext();
- mainContext.push_thread_default();
- let loop = GLib.MainLoop.new(mainContext, true);
- let t;
- try {
- t = buildTaskset.start(context, cancellable, Lang.bind(this, this._onBuildComplete, loop));
- print("Started child process " + context.argv.map(GLib.shell_quote).join(' '));
- loop.run();
- } finally {
- mainContext.pop_thread_default();
- }
- let buildSuccess = this._currentBuildSucceded;
- let msg = this._currentBuildSuccessMsg;
-
- if (!buildSuccess) {
- this._analyzeBuildFailure(t, architecture, component, componentSrc,
- currentVcsVersion, previousVcsVersion, cancellable);
- throw new Error("Build failure in component " + buildname + " : " + msg);
- }
+
+ let proc = new GSystem.Subprocess({ context: context });
+ proc.init(cancellable);
+ print("Started child process " + context.argv.map(GLib.shell_quote).join(' '));
+ proc.wait_sync_check(cancellable);
let recordedMetaPath = componentResultdir.get_child('_ostbuild-meta.json');
JsonUtil.writeJsonFileAtomic(recordedMetaPath, expandedComponent, cancellable);
@@ -472,7 +453,7 @@ const Build = new Lang.Class({
if (statoverridePath != null)
GSystem.file_unlink(statoverridePath, cancellable);
- GSystem.shutil_rm_rf(tmpdir, cancellable);
+ GSystem.shutil_rm_rf(buildWorkdir, cancellable);
let ostreeRevision = this._saveComponentBuild(buildname, expandedComponent, cancellable);
@@ -623,8 +604,7 @@ const Build = new Lang.Class({
Lang.copyProperties(BuildUtil.BUILD_ENV, env);
env['DL_DIR'] = downloads.get_path();
env['SSTATE_DIR'] = sstateDir.get_path();
- ProcUtil.runSync(cmd, cancellable, { logInitiation: true,
- env:ProcUtil.objectToEnvironment(env) });
+ ProcUtil.runSync(cmd, cancellable, {env:ProcUtil.objectToEnvironment(env)});
let componentTypes = ['runtime', 'devel'];
for (let i = 0; i < componentTypes.length; i++) {
@@ -642,26 +622,31 @@ const Build = new Lang.Class({
builtRevisionPath.replace_contents(basemeta['revision'], null, false, Gio.FileCreateFlags.REPLACE_DESTINATION, cancellable);
},
- _init: function() {
- this.parent();
- this.parser.addArgument('--prefix');
- this.parser.addArgument('--snapshot');
- this.parser.addArgument('--patches-path');
+ executeSync: function(cancellable) {
+ let prefix = this.inputs.PREFIX;
this.forceBuildComponents = {};
this.cachedPatchdirRevision = null;
- },
-
- execute: function(args, loop, cancellable) {
- this._initSnapshot(args.prefix, args.snapshot, cancellable);
- this.args = args;
+
+ this.prefix = prefix;
+ this.config = Config.get();
+ this.workdir = Gio.File.new_for_path(this.config.getGlobal('workdir'));
+ this.patchdir = this.workdir.get_child('patches');
+ this.mirrordir = Gio.File.new_for_path(this.config.getGlobal('mirrordir'));
+ this.libdir = Gio.File.new_for_path(GLib.getenv('OSTBUILD_LIBDIR'));
+ this.repo = this.workdir.get_child('repo');
+ let snapshotDir = this.workdir.get_child('snapshots');
+ let srcdb = new JsonDB.JsonDB(snapshotDir.get_child(prefix));
+ let snapshotPath = srcdb.getLatestPath();
+ let data = srcdb.loadFromPath(snapshotPath, cancellable);
+ this._snapshot = new Snapshot.Snapshot(data, snapshotPath);
let components = this._snapshot.data['components'];
let buildresultDir = this.workdir.get_child('builds').get_child(this.prefix);
let builddb = new JsonDB.JsonDB(buildresultDir);
- let targetSourceVersion = builddb.parseVersionStr(this._snapshot.path.get_basename())
+ let targetSourceVersion = builddb.parseVersionStr(this._snapshot.path.get_basename());
let haveLocalComponent = false;
for (let i = 0; i < components.length; i++) {
diff --git a/src/ostbuild/ostbuild.in b/src/ostbuild/ostbuild.in
index 01d75bc..258d9d0 100755
--- a/src/ostbuild/ostbuild.in
+++ b/src/ostbuild/ostbuild.in
@@ -25,4 +25,4 @@ export GIO_USE_VFS=local
export OSTBUILD_DATADIR= pkgdatadir@
export OSTBUILD_LIBDIR= pkglibdir@
-exec gjs -I "${jsdir}" "${jsdir}/main.js" "$@"
+exec $OSTBUILD_GDB gjs -I "${jsdir}" "${jsdir}/main.js" "$@"
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
