[gnome-continuous] Revert "Implement new model for ostbuild"
- From: Jasper St. Pierre <jstpierre src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-continuous] Revert "Implement new model for ostbuild"
- Date: Mon, 30 Sep 2013 18:52:55 +0000 (UTC)
commit 10eef2b20ca868e8932f395308e57cad33479b00
Author: Jasper St. Pierre <jstpierre mecheye net>
Date: Mon Sep 30 14:52:10 2013 -0400
Revert "Implement new model for ostbuild"
This reverts commit bc8949373209a5f534c651a289069b687a9d5bc6.
Whoops. I totally did not mean to push this one to master.
Makefile-ostbuild.am | 1 +
src/js/buildutil.js | 2 +-
src/js/builtin.js | 14 +++-
src/js/builtins/autobuilder.js | 61 +-------------
src/js/builtins/make.js | 6 +-
src/js/jsondb.js | 166 +++++++++++++++++++++++++++++++++++++++
src/js/snapshot.js | 6 --
src/js/task.js | 161 ++++++++++++++++++++++++++------------
src/js/tasks/task-bdiff.js | 32 +++++---
src/js/tasks/task-build.js | 30 ++++++-
src/js/tasks/task-builddisks.js | 48 ++++++++++-
src/js/tasks/task-resolve.js | 29 +++-----
12 files changed, 399 insertions(+), 157 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 2ee96e2..392ed8b 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -64,6 +64,7 @@ jsostbuild_DATA= \
src/js/builtin.js \
src/js/fileutil.js \
src/js/task.js \
+ src/js/jsondb.js \
src/js/jsonutil.js \
src/js/jsutil.js \
src/js/main.js \
diff --git a/src/js/buildutil.js b/src/js/buildutil.js
index 9fb8a13..ae78ab8 100644
--- a/src/js/buildutil.js
+++ b/src/js/buildutil.js
@@ -104,7 +104,7 @@ function atomicSymlinkSwap(linkPath, newTarget, cancellable) {
let parent = linkPath.get_parent();
let tmpLinkPath = parent.get_child('current-new.tmp');
GSystem.shutil_rm_rf(tmpLinkPath, cancellable);
- let relpath = GSystem.file_get_relpath(parent, newTarget);
+ let relpath = parent.get_relative_path(newTarget);
tmpLinkPath.make_symbolic_link(relpath, cancellable);
GSystem.file_rename(tmpLinkPath, linkPath, cancellable);
}
diff --git a/src/js/builtin.js b/src/js/builtin.js
index d8e52fc..cc14040 100644
--- a/src/js/builtin.js
+++ b/src/js/builtin.js
@@ -24,6 +24,7 @@ const GSystem = imports.gi.GSystem;
const Params = imports.params;
const JsonUtil = imports.jsonutil;
const ArgParse = imports.argparse;
+const JsonDB = imports.jsondb;
const Snapshot = imports.snapshot;
const BuildUtil = imports.buildutil;
@@ -56,8 +57,17 @@ const Builtin = new Lang.Class({
_initSnapshot: function(workdir, snapshotPath, cancellable) {
this._initWorkdir(workdir, cancellable);
- let path = Gio.File.new_for_path(snapshotPath);
- this._snapshot = Snapshot.fromFile(path, cancellable);
+ let snapshotDir = this.workdir.get_child('snapshots');
+ let path, data;
+ if (snapshotPath !== null) {
+ path = Gio.File.new_for_path(snapshotPath);
+ data = JsonUtil.loadJson(path, cancellable);
+ } else {
+ let db = new JsonDB.JsonDB(snapshotDir);
+ path = db.getLatestPath();
+ data = db.loadFromPath(path, cancellable);
+ }
+ this._snapshot = new Snapshot.Snapshot(data, path);
},
main: function(argv, loop, cancellable) {
diff --git a/src/js/builtins/autobuilder.js b/src/js/builtins/autobuilder.js
index 2a8bc2b..511f04a 100644
--- a/src/js/builtins/autobuilder.js
+++ b/src/js/builtins/autobuilder.js
@@ -19,12 +19,9 @@ const GLib = imports.gi.GLib;
const Gio = imports.gi.Gio;
const Lang = imports.lang;
-const GSystem = imports.gi.GSystem;
-
const Builtin = imports.builtin;
const Task = imports.task;
const ProcUtil = imports.procutil;
-const VersionedDir = imports.versioneddir;
var AutoBuilderIface = <interface name="org.gnome.OSTreeBuild.AutoBuilder">
<method name="queueResolve">
@@ -38,9 +35,7 @@ const Autobuilder = new Lang.Class({
Extends: Builtin.Builtin,
DESCRIPTION: "Automatically fetch git repositories and build",
-
- _VERSION_RE: /^(\d+\d\d\d\d)\.(\d+)$/,
-
+
_init: function() {
this.parent();
@@ -56,8 +51,6 @@ const Autobuilder = new Lang.Class({
execute: function(args, loop, cancellable) {
this._initWorkdir(null, cancellable);
- this._buildsDir = new VersionedDir.VersionedDir(this.workdir.get_child('builds'), this._VERSION_RE);
-
if (args.autoupdate_self)
this._autoupdate_self = Gio.File.new_for_path(args.autoupdate_self);
@@ -68,7 +61,7 @@ const Autobuilder = new Lang.Class({
this._impl = Gio.DBusExportedObject.wrapJSObject(AutoBuilderIface, this);
this._impl.export(Gio.DBus.session, '/org/gnome/OSTreeBuild/AutoBuilder');
- this._taskmaster = new Task.TaskMaster(this.workdir,
+ this._taskmaster = new Task.TaskMaster(this.workdir.get_child('tasks'),
{ onEmpty: Lang.bind(this, this._onTasksComplete) });
this._taskmaster.connect('task-executing', Lang.bind(this, this._onTaskExecuting));
this._taskmaster.connect('task-complete', Lang.bind(this, this._onTaskCompleted));
@@ -95,9 +88,6 @@ const Autobuilder = new Lang.Class({
},
_onTaskCompleted: function(taskmaster, task, success, error) {
- if (!task.changed)
- GSystem.shutil_rm_rf(task.buildPath, cancellable);
-
if (task.name == 'resolve')
this._runResolve();
if (success) {
@@ -144,46 +134,6 @@ const Autobuilder = new Lang.Class({
return true;
},
- _getLastVersion: function(cancellable) {
- let allVersions = this._buildsDir.loadVersions(cancellable);
- if (allVersions.length > 0)
- return allVersions[allVersions.length-1];
- else
- return null;
- },
-
- _getNextBuildDirectory: function(cancellable) {
- let currentTime = GLib.DateTime.new_now_utc();
- let currentYmd = Format.vprintf('%d%02d%02d', [currentTime.get_year(),
- currentTime.get_month(),
- currentTime.get_day_of_month()]);
-
- let version = null;
- let lastVersion = this._getLastVersion(cancellable);
- if (lastVersion) {
- let match = this._VERSION_RE.exec(lastVersion);
- if (!match) throw new Error();
- let lastYmd = match[1];
- let lastSerial = match[2];
- if (lastYmd == currentYmd) {
- version = currentYmd + '.' + (parseInt(lastSerial) + 1);
- }
- }
- if (version === null) {
- version = currentYmd + '.0';
- }
-
- let buildPath = this._buildsDir.path.get_child(version);
- GSystem.file_ensure_directory(buildPath, true, cancellable);
-
- if (lastVersion) {
- let lastBuildPath = this._buildsDir.path.get_child(lastVersion);
- BuildUtil.atomicSymlinkSwap(buildPath.get_child('last-build'), lastBuildPath);
- }
-
- return buildPath;
- },
-
_runResolve: function() {
let cancellable = null;
@@ -199,15 +149,14 @@ const Autobuilder = new Lang.Class({
ProcUtil.runSync(['git', 'pull', '-r'], cancellable,
{ cwd: this._autoupdate_self })
- let buildPath = this._getNextBuildDirectory(cancellable);
if (this._initialResolveNeeded) {
this._initialResolveNeeded = false;
- this._taskmaster.startBuild('resolve', buildPath, { });
+ this._taskmaster.pushTask('resolve', { });
} else if (this._fullResolveNeeded) {
this._fullResolveNeeded = false;
- this._taskmaster.startBuild('resolve', buildPath, { fetchAll: true });
+ this._taskmaster.pushTask('resolve', { fetchAll: true });
} else {
- this._taskmaster.startBuild('resolve', buildPath, { fetchSrcUrls: this._resolveSrcUrls });
+ this._taskmaster.pushTask('resolve', { fetchSrcUrls: this._resolveSrcUrls });
}
this._resolveSrcUrls = [];
diff --git a/src/js/builtins/make.js b/src/js/builtins/make.js
index 256179c..db35fc3 100644
--- a/src/js/builtins/make.js
+++ b/src/js/builtins/make.js
@@ -37,7 +37,6 @@ const Make = new Lang.Class({
this.parser.addArgument(['-x', '--skip'], { action: 'append',
help: "Don't process tasks after this" });
this.parser.addArgument('taskname');
- this.parser.addArgument('buildPath');
this.parser.addArgument('parameters', { nargs: '*' });
},
@@ -48,7 +47,7 @@ const Make = new Lang.Class({
this._cancellable = cancellable;
this._tasksComplete = false;
this._oneOnly = args.only;
- let taskmaster = new Task.TaskMaster(this.workdir,
+ let taskmaster = new Task.TaskMaster(this.workdir.get_child('tasks'),
{ onEmpty: Lang.bind(this, this._onTasksComplete),
processAfter: !args.only,
skip: args.skip });
@@ -56,8 +55,7 @@ const Make = new Lang.Class({
taskmaster.connect('task-executing', Lang.bind(this, this._onTaskExecuting));
taskmaster.connect('task-complete', Lang.bind(this, this._onTaskCompleted));
let params = this._parseParameters(args.parameters);
- let buildPath = Gio.File.new_for_path(args.buildPath);
- taskmaster.startBuild(buildPath, args.taskname, params);
+ taskmaster.pushTask(args.taskname, params);
loop.run();
if (!this._failed)
print("Success!")
diff --git a/src/js/jsondb.js b/src/js/jsondb.js
new file mode 100644
index 0000000..d435d8d
--- /dev/null
+++ b/src/js/jsondb.js
@@ -0,0 +1,166 @@
+// Copyright (C) 2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const JsonUtil = imports.jsonutil;
+const GSystem = imports.gi.GSystem;
+
+const JsonDB = new Lang.Class({
+ Name: 'JsonDB',
+
+ _init: function(path) {
+ this._path = path;
+ GSystem.file_ensure_directory(this._path, true, null);
+ this._re = /^(\d+)\.(\d+)-([0-9a-f]+).json$/;
+ this._maxVersions = 5;
+ },
+
+ _parseVersion: function(basename) {
+ let match = this._re.exec(basename);
+ if (!match)
+ throw new Error("No JSONDB version in " + basename);
+ return [parseInt(match[1]), parseInt(match[2])];
+ },
+
+ parseVersionStr: function(basename) {
+ let [major, minor] = this._parseVersion(basename);
+ return Format.vprintf('%d.%d', [major, minor]);
+ },
+
+ _getAll: function() {
+ let cancellable = null;
+ var result = [];
+ var e = this._path.enumerate_children('standard::*', Gio.FileQueryInfoFlags.NONE, cancellable);
+ let info;
+ while ((info = e.next_file(null)) != null) {
+ let name = info.get_name();
+ let match = this._re.exec(name);
+ if (!match)
+ continue;
+ result.push([parseInt(match[1]), parseInt(match[2]),
+ match[3], name]);
+ }
+ e.close(cancellable);
+ result.sort(function(a, b) {
+ var aMajor = a[0]; var bMajor = b[0];
+ var aMinor = a[1]; var bMinor = b[1];
+ if (aMajor < bMajor) return 1;
+ else if (aMajor > bMajor) return -1;
+ else if (aMinor < bMinor) return 1;
+ else if (aMinor > bMinor) return -1;
+ else return 0;
+ });
+ return result;
+ },
+
+ getLatestPath: function() {
+ let all = this._getAll();
+ if (all.length == 0)
+ return null;
+ return this._path.get_child(all[0][3]);
+ },
+
+ getLatestVersion: function() {
+ let path = this.getLatestPath();
+ if (path == null)
+ return null;
+ return this.parseVersionStr(path.get_basename());
+ },
+
+ getPreviousPath: function(path) {
+ let name = path.get_basename();
+ let [target_major, target_minor] = this._parseVersion(name);
+ let files = this._getAll();
+ let prev = null;
+ let found = false;
+ for (let i = files.length - 1; i >= 0; i--) {
+ let [major, minor, csum, fname] = files[i];
+ if (target_major == major && target_minor == minor) {
+ found = true;
+ break;
+ }
+ prev = fname;
+ }
+ if (found && prev)
+ return this._path.get_child(prev);
+ return null;
+ },
+
+ loadFromPath: function(path, cancellable) {
+ return JsonUtil.loadJson(this._path.get_child(path.get_basename()), cancellable);
+ },
+
+ _updateIndex: function(cancellable) {
+ let files = this._getAll();
+ let fnames = [];
+ for (let i = 0; i < files.length; i++) {
+ fnames.push(files[i][3]);
+ }
+ let index = { files: fnames };
+ JsonUtil.writeJsonFileAtomic(this._path.get_child('index.json'), index, cancellable);
+ },
+
+ store: function(obj, cancellable) {
+ let files = this._getAll();
+ let latest = null;
+ if (files.length > 0) {
+ latest = files[0];
+ }
+
+ let currentTime = GLib.DateTime.new_now_utc();
+ let currentYmd = Format.vprintf('%d%02d%02d', [currentTime.get_year(),
+ currentTime.get_month(),
+ currentTime.get_day_of_month()]);
+
+ let buf = JsonUtil.serializeJson(obj);
+ let csum = GLib.compute_checksum_for_string(GLib.ChecksumType.SHA256, buf, -1);
+
+ if (latest && csum == latest[2]) {
+ return [this._path.get_child(latest[3]), false];
+ }
+
+ let version = null;
+ if (latest) {
+ let lastYmd = latest[0];
+ if (lastYmd == currentYmd)
+ version = currentYmd + '.' + (latest[1]+1);
+ }
+ if (version == null) {
+ version = currentYmd + '.0';
+ }
+
+ let targetName = Format.vprintf('%s-%s.json', [version, csum]);
+ let targetPath = this._path.get_child(targetName);
+ targetPath.replace_contents(buf, null, false, Gio.FileCreateFlags.REPLACE_DESTINATION, cancellable);
+
+ if (files.length + 1 > this._maxVersions) {
+ for (let i = Math.max(files.length-(this._maxVersions-1), 0);
+ i < files.length;
+ i++) {
+ GSystem.file_unlink(this._path.get_child(files[i][3]), cancellable);
+ }
+ }
+
+ this._updateIndex(cancellable);
+
+ return [targetPath, true];
+ }
+});
diff --git a/src/js/snapshot.js b/src/js/snapshot.js
index 4af4ecd..281ee6e 100644
--- a/src/js/snapshot.js
+++ b/src/js/snapshot.js
@@ -18,7 +18,6 @@
const Lang = imports.lang;
-const JsonUtil = imports.jsonutil;
const Params = imports.params;
function _componentDict(snapshot) {
@@ -66,11 +65,6 @@ function snapshotDiff(a, b) {
return [added, modified, removed];
}
-function fromFile(path, cancellable) {
- let data = JsonUtil.loadJson(path, cancellable);
- return new Snapshot(data, path);
-}
-
const Snapshot = new Lang.Class({
Name: 'Snapshot',
diff --git a/src/js/task.js b/src/js/task.js
index 9a78c0b..7ce83dc 100644
--- a/src/js/task.js
+++ b/src/js/task.js
@@ -25,8 +25,10 @@ const GSystem = imports.gi.GSystem;
const OSTree = imports.gi.OSTree;
const Params = imports.params;
const JsonUtil = imports.jsonutil;
+const JsonDB = imports.jsondb;
const ProcUtil = imports.procutil;
const BuildUtil = imports.buildutil;
+const VersionedDir = imports.versioneddir;
const DefaultTaskDef = {
TaskName: '',
@@ -132,10 +134,6 @@ const TaskMaster = new Lang.Class({
this._skipTasks = {};
for (let i = 0; i < params.skip.length; i++)
this._skipTasks[params.skip[i]] = true;
-
- this.tasksPath = this.path.get_child('tasks');
- GSystem.file_ensure_directory(this.tasksPath, true, null);
-
this.maxConcurrent = GLib.get_num_processors();
this._onEmpty = params.onEmpty;
this.cancellable = null;
@@ -152,28 +150,12 @@ const TaskMaster = new Lang.Class({
this._scheduledTaskTimeouts = {};
},
- _getTaskBuildPath: function(taskName) {
- let buildPath = this.tasksPath.resolve_relative_path(taskName);
- return GSystem.file_realpath(buildPath);
- },
-
- _setTaskBuildPath: function(taskName, buildPath) {
- let taskLink = this.tasksPath.get_child(taskName);
- BuildUtil.atomicSymlinkSwap(taskLink, buildPath, this.cancellable);
- return buildPath;
- },
-
_pushTaskDataImmediate: function(taskData) {
this._pendingTasksList.push(taskData);
this._queueRecalculate();
},
- startBuild: function(buildPath, taskName, parameters) {
- this._setTaskBuildPath(taskName, buildPath);
- this._pushTask(taskName, parameters);
- },
-
- _pushTask: function(name, parameters) {
+ pushTask: function(name, parameters) {
let taskDef = this._taskset.getTaskDef(name);
let taskData = new TaskData(taskDef, parameters);
if (!this._isTaskPending(name)) {
@@ -297,24 +279,19 @@ const TaskMaster = new Lang.Class({
if (idx == -1)
throw new Error("TaskMaster: Internal error - Failed to find completed task:" +
runner.taskData.name);
this._executing.splice(idx, 1);
-
this.emit('task-complete', runner, success, error);
if (success && this._processAfter) {
- let taskName = runner.taskData.name;
- if (success && runner.changed) {
- let taskDef = runner.taskData.taskDef;
- let buildPath = this._getTaskBuildPath(taskName);
- let after = this._taskset.getTasksAfter(taskName);
- for (let i = 0; i < after.length; i++) {
+ if (runner.changed) {
+ let taskName = runner.taskData.name;
+ let taskDef = runner.taskData.taskDef;
+ let after = this._taskset.getTasksAfter(taskName);
+ for (let i = 0; i < after.length; i++) {
let afterTaskName = after[i];
- if (!this._skipTasks[afterTaskName]) {
- this._setTaskBuildPath(afterTaskName, buildPath);
+ if (!this._skipTasks[afterTaskName])
this.pushTask(afterTaskName, {});
- }
- }
+ }
}
- }
-
+ }
this._queueRecalculate();
},
@@ -345,7 +322,6 @@ const Task = new Lang.Class({
this.workdir = Gio.File.new_for_path(GLib.getenv('_OSTBUILD_WORKDIR'));
BuildUtil.checkIsWorkDirectory(this.workdir);
- this.builddir = Gio.File.new_for_path(GLib.getenv('_OSTBUILD_BUILDDIR'));
this.resultdir = this.workdir.get_child('results');
GSystem.file_ensure_directory(this.resultdir, true, null);
@@ -362,6 +338,11 @@ const Task = new Lang.Class({
this.ostreeRepo.open(null);
},
+ _getResultDb: function(taskname) {
+ let path = this.resultdir.resolve_relative_path(taskname);
+ return new JsonDB.JsonDB(path);
+ },
+
execute: function(cancellable) {
throw new Error("Not implemented");
},
@@ -370,36 +351,85 @@ const Task = new Lang.Class({
const TaskRunner = new Lang.Class({
Name: 'TaskRunner',
+ _VERSION_RE: /^(\d+\d\d\d\d)\.(\d+)$/,
+
_init: function(taskmaster, taskData, onComplete) {
this.taskmaster = taskmaster;
this.taskData = taskData;
this.onComplete = onComplete;
this.name = taskData.name;
- this.workdir = taskmaster.path;
+ this.workdir = taskmaster.path.get_parent();
BuildUtil.checkIsWorkDirectory(this.workdir);
},
+ _loadAllVersions: function(cancellable) {
+ let allVersions = [];
+
+ let successVersions = this._successDir.loadVersions(cancellable);
+ for (let i = 0; i < successVersions.length; i++) {
+ allVersions.push([true, successVersions[i]]);
+ }
+
+ let failedVersions = this._failedDir.loadVersions(cancellable);
+ for (let i = 0; i < failedVersions.length; i++) {
+ allVersions.push([false, failedVersions[i]]);
+ }
+
+ allVersions.sort(function (a, b) {
+ let [successA, versionA] = a;
+ let [successB, versionB] = b;
+ return BuildUtil.compareVersions(versionA, versionB);
+ });
+
+ return allVersions;
+ },
+
executeInSubprocess: function(cancellable) {
this._cancellable = cancellable;
this._startTimeMillis = GLib.get_monotonic_time() / 1000;
- // To prevent tasks from stomping on each other's toes, we put the task
- // cwd in its own task dir. If a task has any results it wants to pass
- // on between builds, it needs to write to _OSTBUILD_BUILDDIR.
- let buildPath = this.taskmaster.tasksPath.resolve_relative_path(this.name);
- buildPath = GSystem.file_realpath(buildPath);
+ this.dir = this.taskmaster.path.resolve_relative_path(this.name);
+ GSystem.file_ensure_directory(this.dir, true, cancellable);
+
+ this._topDir = new VersionedDir.VersionedDir(this.dir, this._VERSION_RE);
+ this._successDir = new VersionedDir.VersionedDir(this.dir.get_child('successful'),
+ this._VERSION_RE);
+ this._failedDir = new VersionedDir.VersionedDir(this.dir.get_child('failed'),
+ this._VERSION_RE);
+
+ let allVersions = this._loadAllVersions(cancellable);
+
+ let currentTime = GLib.DateTime.new_now_utc();
+
+ let currentYmd = Format.vprintf('%d%02d%02d', [currentTime.get_year(),
+ currentTime.get_month(),
+ currentTime.get_day_of_month()]);
+ let version = null;
+ if (allVersions.length > 0) {
+ let [lastSuccess, lastVersion] = allVersions[allVersions.length-1];
+ let match = this._VERSION_RE.exec(lastVersion);
+ if (!match) throw new Error();
+ let lastYmd = match[1];
+ let lastSerial = match[2];
+ if (lastYmd == currentYmd) {
+ version = currentYmd + '.' + (parseInt(lastSerial) + 1);
+ }
+ }
+ if (version === null) {
+ version = currentYmd + '.0';
+ }
- this._buildName = buildPath.get_basename();
- this._taskCwd = buildPath.get_child(this.name);
- GSystem.file_ensure_directory(this._taskCwd, false, cancellable);
+ this._version = version;
+ this._taskCwd = this.dir.get_child(version);
+ GSystem.shutil_rm_rf(this._taskCwd, cancellable);
+ GSystem.file_ensure_directory(this._taskCwd, true, cancellable);
let baseArgv = ['ostbuild', 'run-task', this.name, JSON.stringify(this.taskData.parameters)];
let context = new GSystem.SubprocessContext({ argv: baseArgv });
context.set_cwd(this._taskCwd.get_path());
let childEnv = GLib.get_environ();
- childEnv.push('_OSTBUILD_BUILDDIR=' + buildPath.get_path());
childEnv.push('_OSTBUILD_WORKDIR=' + this.workdir.get_path());
context.set_environment(childEnv);
if (this.taskData.taskDef.PreserveStdout) {
@@ -417,6 +447,20 @@ const TaskRunner = new Lang.Class({
this._proc.wait(cancellable, Lang.bind(this, this._onChildExited));
},
+ _updateIndex: function(cancellable) {
+ let allVersions = this._loadAllVersions(cancellable);
+
+ let fileList = [];
+ for (let i = 0; i < allVersions.length; i++) {
+ let [successful, version] = allVersions[i];
+ let fname = (successful ? 'successful/' : 'failed/') + version;
+ fileList.push(fname);
+ }
+
+ let index = { files: fileList };
+ JsonUtil.writeJsonFileAtomic(this.dir.get_child('index.json'), index, cancellable);
+ },
+
_onChildExited: function(proc, result) {
let cancellable = this._cancellable;
let [success, errmsg] = ProcUtil.asyncWaitCheckFinish(proc, result);
@@ -429,16 +473,24 @@ const TaskRunner = new Lang.Class({
this.changed = data['modified'];
}
- this.onComplete(success, errmsg);
-
- if (!this.changed)
- return;
+ if (!success) {
+ target = this._failedDir.path.get_child(this._version);
+ GSystem.file_rename(this._taskCwd, target, null);
+ this._taskCwd = target;
+ this._failedDir.cleanOldVersions(this.taskData.taskDef.RetainFailed, null);
+ this.onComplete(success, errmsg);
+ } else {
+ target = this._successDir.path.get_child(this._version);
+ GSystem.file_rename(this._taskCwd, target, null);
+ this._taskCwd = target;
+ this._successDir.cleanOldVersions(this.taskData.taskDef.RetainSuccess, null);
+ this.onComplete(success, null);
+ }
let elapsedMillis = GLib.get_monotonic_time() / 1000 - this._startTimeMillis;
let targetPath = this.workdir.get_relative_path(this._taskCwd);
-
let meta = { taskMetaVersion: 0,
- buildName: this._buildName,
+ taskVersion: this._version,
success: success,
errmsg: errmsg,
elapsedMillis: elapsedMillis,
@@ -450,5 +502,12 @@ const TaskRunner = new Lang.Class({
}
JsonUtil.writeJsonFileAtomic(this._taskCwd.get_child('meta.json'), meta, cancellable);
+
+ // Also remove any old interrupted versions
+ this._topDir.cleanOldVersions(0, null);
+
+ this._updateIndex(cancellable);
+
+ BuildUtil.atomicSymlinkSwap(this.dir.get_child('current'), target, cancellable);
}
});
diff --git a/src/js/tasks/task-bdiff.js b/src/js/tasks/task-bdiff.js
index d37c326..d1cffff 100644
--- a/src/js/tasks/task-bdiff.js
+++ b/src/js/tasks/task-bdiff.js
@@ -83,21 +83,32 @@ const TaskBdiff = new Lang.Class({
},
execute: function(cancellable) {
- let latestSnapshotPath = this.builddir.get_child('snapshot.json');
- let previousSnapshotPath = this.builddir.get_child('last-build/snapshot.json');
- if (!previousSnapshotPath.query_exists(cancellable))
- return;
+ let builddb = this._getResultDb('build');
+ let latestPath = builddb.getLatestPath();
+ if (!latestPath)
+ throw new Error("No builds!")
+ let latestBuildVersion = builddb.parseVersionStr(latestPath.get_basename());
- let latestBuildSnapshot = Snapshot.fromFile(latestSnapshotPath, cancellable);
- let previousBuildSnapshot = Snapshot.fromFile(previousSnapshotData, cancellable);
+ let previousPath = builddb.getPreviousPath(latestPath);
+ if (!previousPath)
+ throw new Error("No build previous to " + latestBuildVersion)
+
+ let latestBuildData = builddb.loadFromPath(latestPath, cancellable);
+ let latestBuildSnapshot = new Snapshot.Snapshot(latestBuildData['snapshot'], null);
+ let previousBuildData = builddb.loadFromPath(previousPath, cancellable);
+ let previousBuildSnapshot = new Snapshot.Snapshot(previousBuildData['snapshot'], null);
let added = [];
let modified = [];
let removed = [];
- let result = { added: added,
- modified: modified,
- removed: removed };
+ let result = {fromBuildVersion: builddb.parseVersionStr(previousPath.get_basename()),
+ toBuildVersion: builddb.parseVersionStr(latestPath.get_basename()),
+ fromSrcVersion: builddb.parseVersionStr(previousBuildData['snapshotName']),
+ toSrcVersion: builddb.parseVersionStr(latestBuildData['snapshotName']),
+ added: added,
+ modified: modified,
+ removed: removed};
let modifiedNames = [];
@@ -135,6 +146,7 @@ const TaskBdiff = new Lang.Class({
diffstat: diffstat });
}
- JsonUtil.writeJsonFileAtomic(this.builddir.get_child('bdiff.json'), result, cancellable);
+ let bdiffdb = this._getResultDb('bdiff');
+ bdiffdb.store(result, cancellable);
}
});
diff --git a/src/js/tasks/task-build.js b/src/js/tasks/task-build.js
index 198a17a..de96aba 100644
--- a/src/js/tasks/task-build.js
+++ b/src/js/tasks/task-build.js
@@ -30,6 +30,7 @@ const AsyncUtil = imports.asyncutil;
const ProcUtil = imports.procutil;
const StreamUtil = imports.streamutil;
const JsonUtil = imports.jsonutil;
+const JsonDB = imports.jsondb;
const Snapshot = imports.snapshot;
const BuildUtil = imports.buildutil;
const Vcs = imports.vcs;
@@ -1127,11 +1128,14 @@ const TaskBuild = new Lang.Class({
this.forceBuildComponents[this.parameters.forceComponents[i]] = true;
this.cachedPatchdirRevision = null;
- let snapshotPath = this.builddir.get_child('snapshot.json');
- let workingSnapshotPath = Gio.File.new_for_path('snapshot.json');
+ let snapshotDir = this.workdir.get_child('snapshots');
+ let srcdb = new JsonDB.JsonDB(snapshotDir);
+ let snapshotPath = srcdb.getLatestPath();
+ let workingSnapshotPath = Gio.File.new_for_path(snapshotPath.get_basename());
GSystem.file_linkcopy(snapshotPath, workingSnapshotPath, Gio.FileCopyFlags.OVERWRITE,
cancellable);
- this._snapshot = Snapshot.fromFile(workingSnapshotPath, cancellable);
+ let data = srcdb.loadFromPath(workingSnapshotPath, cancellable);
+ this._snapshot = new Snapshot.Snapshot(data, workingSnapshotPath);
let osname = this._snapshot.data['osname'];
this.osname = osname;
@@ -1141,6 +1145,10 @@ const TaskBuild = new Lang.Class({
let components = this._snapshot.data['components'];
+ let builddb = this._getResultDb('build');
+
+ let targetSourceVersion = builddb.parseVersionStr(this._snapshot.path.get_basename());
+
// Pick up overrides from $workdir/overrides/$name
for (let i = 0; i < components.length; i++) {
let component = components[i];
@@ -1162,6 +1170,19 @@ const TaskBuild = new Lang.Class({
haveLocalComponent = true;
}
+ let latestBuildPath = builddb.getLatestPath();
+ if (latestBuildPath != null) {
+ let lastBuiltSourceData = builddb.loadFromPath(latestBuildPath, cancellable);
+ let lastBuiltSourceVersion = builddb.parseVersionStr(lastBuiltSourceData['snapshotName']);
+ if (!haveLocalComponent && lastBuiltSourceVersion == targetSourceVersion) {
+ print("Already built source snapshot " + lastBuiltSourceVersion);
+ return;
+ } else {
+ print("Last successful build was " + lastBuiltSourceVersion);
+ }
+ }
+ print("building " + targetSourceVersion);
+
this._componentBuildCachePath = this.cachedir.get_child('component-builds.json');
if (this._componentBuildCachePath.query_exists(cancellable)) {
this._componentBuildCache = JsonUtil.loadJson(this._componentBuildCachePath, cancellable);
@@ -1417,6 +1438,7 @@ const TaskBuild = new Lang.Class({
this._writeStatus('built: ' + this._rebuiltComponents.join(' '), cancellable);
- JsonUtil.writeJsonFileAtomic(this.builddir.get_child('build.json'), buildData, cancellable);
+ let [path, modified] = builddb.store(buildData, cancellable);
+ print("Build complete: " + path.get_path());
}
});
diff --git a/src/js/tasks/task-builddisks.js b/src/js/tasks/task-builddisks.js
index f1218ea..10d638e 100644
--- a/src/js/tasks/task-builddisks.js
+++ b/src/js/tasks/task-builddisks.js
@@ -29,6 +29,7 @@ const Task = imports.task;
const ProcUtil = imports.procutil;
const BuildUtil = imports.buildutil;
const LibQA = imports.libqa;
+const VersionedDir = imports.versioneddir;
const JsonUtil = imports.jsonutil;
const JSUtil = imports.jsutil;
const GuestFish = imports.guestfish;
@@ -47,18 +48,39 @@ const TaskBuildDisks = new Lang.Class({
// Legacy
_VERSION_RE: /^(\d+)\.(\d+)$/,
+ _imageSubdir: 'images',
_inheritPreviousDisk: true,
_onlyTreeSuffixes: ['-runtime'],
execute: function(cancellable) {
- let buildData = JsonUtil.loadJson(this.builddir.get_child('build.json'), cancellable);
+ let baseImageDir = this.workdir.resolve_relative_path(this._imageSubdir);
+ let baseImageVersionedDir = new VersionedDir.VersionedDir(baseImageDir, this._VERSION_RE);
+ GSystem.file_ensure_directory(baseImageDir, true, cancellable);
+ let currentImageLink = baseImageDir.get_child('current');
+ let previousImageLink = baseImageDir.get_child('previous');
+
+ let builddb = this._getResultDb('build');
+
+ let latestPath = builddb.getLatestPath();
+ let buildVersion = builddb.parseVersionStr(latestPath.get_basename());
+ this._buildVersion = buildVersion;
+ let buildData = builddb.loadFromPath(latestPath, cancellable);
+
+ let targetImageDir = baseImageDir.get_child(buildVersion);
+
+ if (targetImageDir.query_exists(null)) {
+ print("Already created " + targetImageDir.get_path());
+ return;
+ }
- let prevImageDir = this.builddir.get_child('last-build/images');
- let targetImageDir = this.builddir.get_child('images');
let workImageDir = Gio.File.new_for_path('images');
GSystem.file_ensure_directory(workImageDir, true, cancellable);
+ let destPath = workImageDir.get_child('build-' + buildVersion + '.json');
+ GSystem.file_linkcopy(latestPath, destPath, Gio.FileCopyFlags.ALL_METADATA, cancellable);
+
let targets = buildData['targets'];
+
let osname = buildData['snapshot']['osname'];
let originRepoUrl = buildData['snapshot']['repo'];
@@ -76,7 +98,7 @@ const TaskBuildDisks = new Lang.Class({
let squashedName = osname + '-' + targetName.substr(targetName.lastIndexOf('/') + 1);
let diskName = squashedName + '.qcow2';
let diskPath = workImageDir.get_child(diskName);
- let prevPath = prevImageDir.get_child(diskName);
+ let prevPath = currentImageLink.get_child(diskName);
GSystem.shutil_rm_rf(diskPath, cancellable);
let doCloneDisk = this._inheritPreviousDisk && prevPath.query_exists(null);
if (doCloneDisk) {
@@ -106,6 +128,24 @@ const TaskBuildDisks = new Lang.Class({
}
GSystem.file_rename(workImageDir, targetImageDir, cancellable);
+
+ let currentInfo = null;
+ try {
+ currentInfo = currentImageLink.query_info('standard::symlink-target',
Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS, cancellable);
+ } catch (e) {
+ if (!e.matches(Gio.IOErrorEnum, Gio.IOErrorEnum.NOT_FOUND))
+ throw e;
+ }
+ if (currentInfo != null) {
+ let newPreviousTmppath = baseImageDir.get_child('previous-new.tmp');
+ let currentLinkTarget = currentInfo.get_symlink_target();
+ GSystem.shutil_rm_rf(newPreviousTmppath, cancellable);
+ newPreviousTmppath.make_symbolic_link(currentLinkTarget, cancellable);
+ GSystem.file_rename(newPreviousTmppath, previousImageLink, cancellable);
+ }
+ BuildUtil.atomicSymlinkSwap(baseImageDir.get_child('current'), targetImageDir, cancellable);
+
+ baseImageVersionedDir.cleanOldVersions(IMAGE_RETAIN_COUNT, cancellable);
},
_postDiskCreation: function(squashedName, diskPath, cancellable) {
diff --git a/src/js/tasks/task-resolve.js b/src/js/tasks/task-resolve.js
index 8b91e84..94d334c 100644
--- a/src/js/tasks/task-resolve.js
+++ b/src/js/tasks/task-resolve.js
@@ -18,6 +18,7 @@
const Gio = imports.gi.Gio;
const Lang = imports.lang;
+const JsonDB = imports.jsondb;
const Task = imports.task;
const ProcUtil = imports.procutil;
const JsonUtil = imports.jsonutil;
@@ -37,22 +38,12 @@ const TaskResolve = new Lang.Class({
fetchComponents: [],
timeoutSec: 10},
- _writeSnapshotToBuild: function(cancellable) {
- let data = this._snapshot.data;
- let buf = JsonUtil.serializeJson(data);
-
- let oldSnapshot = this.builddir.get_child('last-build/snapshot.json');
- if (oldSnapshot.query_exists(cancellable)) {
- let oldBytes = GSystem.file_map_readonly(snapshotPath, cancellable);
- let oldCsum = GLib.compute_checksum_for_bytes(GLib.ChecksumType.SHA256, oldBytes);
- let newCsum = GLib.compute_checksum_for_string(GLib.ChecksumType.SHA256, buf, -1);
- if (oldCsum == newCsum)
- return false;
- }
-
- let snapshot = this.builddir.get_child('snapshot.json');
- JsonUtil.writeJsonFileAtomic(snapshot, data, cancellable);
- return true;
+ _getDb: function() {
+ if (this._db == null) {
+ let snapshotdir = this.workdir.get_child('snapshots');
+ this._db = new JsonDB.JsonDB(snapshotdir);
+ }
+ return this._db;
},
execute: function(cancellable) {
@@ -86,11 +77,11 @@ const TaskResolve = new Lang.Class({
component['revision'] = revision;
}
- let modified = this._writeSnapshotToBuild(cancellable);
+ let [path, modified] = this._getDb().store(this._snapshot.data, cancellable);
if (modified) {
- print("New source snapshot");
+ print("New source snapshot: " + path.get_path());
} else {
- print("Source snapshot unchanged");
+ print("Source snapshot unchanged: " + path.get_path());
}
let modifiedPath = Gio.File.new_for_path('modified.json');
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
