[gnome-continuous/wip/new-model: 14/14] Implement new model for ostbuild
- From: Jasper St. Pierre <jstpierre src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-continuous/wip/new-model: 14/14] Implement new model for ostbuild
- Date: Mon, 30 Sep 2013 18:53:34 +0000 (UTC)
commit 01f6cf1edadb52495c273c717d869d08c97b4396
Author: Jasper St. Pierre <jstpierre mecheye net>
Date: Mon Sep 30 14:51:28 2013 -0400
Implement new model for ostbuild
At a high level, this kills off JsonDB and makes build IDs consistent
across all tasks.
The details are that this implements a new scheme for how tasks are run
and how the autobuilder and task manager manage tasks: each build has
its own directory where results are compiled, and tasks are run inside
these build directories.
The autobuilder makes build directories speculatively based on a trigger,
runs a resolve, and then kicks off the rest of the build process. If the
resolve returns that nothing has changed, then the build directory is
deleted so we don't waste directories.
The details of what each task is working is tracked by a series of symbolic
links in a new toplevel directory, 'tasks': when a task starts, it takes the
realpath of these, and when a task finishes, the task manager updates the
symlinks for the tasks that follow.
This symlink adjusting scheme ensures that if we have a heavy task, like
builddisks, we don't run it for every possible build, but only the latest
build finished when the current builddisks finishes.
Soon, I expect to add a status.json for a build that details the overall
state of the build (in-progress, success, failed) and the status of every
task that has run on it.
I also want to add symbolic links in a new toplevel directory, 'results',
that provide a convenient way to get the latest successful build or the
latest failed build, etc.
Makefile-ostbuild.am | 1 -
src/js/buildutil.js | 2 +-
src/js/builtin.js | 14 +---
src/js/builtins/autobuilder.js | 61 +++++++++++++-
src/js/builtins/make.js | 6 +-
src/js/jsondb.js | 166 ---------------------------------------
src/js/snapshot.js | 6 ++
src/js/task.js | 161 ++++++++++++--------------------------
src/js/tasks/task-bdiff.js | 32 +++-----
src/js/tasks/task-build.js | 30 +------
src/js/tasks/task-builddisks.js | 48 +----------
src/js/tasks/task-resolve.js | 29 +++++---
12 files changed, 157 insertions(+), 399 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 392ed8b..2ee96e2 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -64,7 +64,6 @@ jsostbuild_DATA= \
src/js/builtin.js \
src/js/fileutil.js \
src/js/task.js \
- src/js/jsondb.js \
src/js/jsonutil.js \
src/js/jsutil.js \
src/js/main.js \
diff --git a/src/js/buildutil.js b/src/js/buildutil.js
index ae78ab8..9fb8a13 100644
--- a/src/js/buildutil.js
+++ b/src/js/buildutil.js
@@ -104,7 +104,7 @@ function atomicSymlinkSwap(linkPath, newTarget, cancellable) {
let parent = linkPath.get_parent();
let tmpLinkPath = parent.get_child('current-new.tmp');
GSystem.shutil_rm_rf(tmpLinkPath, cancellable);
- let relpath = parent.get_relative_path(newTarget);
+ let relpath = GSystem.file_get_relpath(parent, newTarget);
tmpLinkPath.make_symbolic_link(relpath, cancellable);
GSystem.file_rename(tmpLinkPath, linkPath, cancellable);
}
diff --git a/src/js/builtin.js b/src/js/builtin.js
index cc14040..d8e52fc 100644
--- a/src/js/builtin.js
+++ b/src/js/builtin.js
@@ -24,7 +24,6 @@ const GSystem = imports.gi.GSystem;
const Params = imports.params;
const JsonUtil = imports.jsonutil;
const ArgParse = imports.argparse;
-const JsonDB = imports.jsondb;
const Snapshot = imports.snapshot;
const BuildUtil = imports.buildutil;
@@ -57,17 +56,8 @@ const Builtin = new Lang.Class({
_initSnapshot: function(workdir, snapshotPath, cancellable) {
this._initWorkdir(workdir, cancellable);
- let snapshotDir = this.workdir.get_child('snapshots');
- let path, data;
- if (snapshotPath !== null) {
- path = Gio.File.new_for_path(snapshotPath);
- data = JsonUtil.loadJson(path, cancellable);
- } else {
- let db = new JsonDB.JsonDB(snapshotDir);
- path = db.getLatestPath();
- data = db.loadFromPath(path, cancellable);
- }
- this._snapshot = new Snapshot.Snapshot(data, path);
+ let path = Gio.File.new_for_path(snapshotPath);
+ this._snapshot = Snapshot.fromFile(path, cancellable);
},
main: function(argv, loop, cancellable) {
diff --git a/src/js/builtins/autobuilder.js b/src/js/builtins/autobuilder.js
index 511f04a..2a8bc2b 100644
--- a/src/js/builtins/autobuilder.js
+++ b/src/js/builtins/autobuilder.js
@@ -19,9 +19,12 @@ const GLib = imports.gi.GLib;
const Gio = imports.gi.Gio;
const Lang = imports.lang;
+const GSystem = imports.gi.GSystem;
+
const Builtin = imports.builtin;
const Task = imports.task;
const ProcUtil = imports.procutil;
+const VersionedDir = imports.versioneddir;
var AutoBuilderIface = <interface name="org.gnome.OSTreeBuild.AutoBuilder">
<method name="queueResolve">
@@ -35,7 +38,9 @@ const Autobuilder = new Lang.Class({
Extends: Builtin.Builtin,
DESCRIPTION: "Automatically fetch git repositories and build",
-
+
+ _VERSION_RE: /^(\d+\d\d\d\d)\.(\d+)$/,
+
_init: function() {
this.parent();
@@ -51,6 +56,8 @@ const Autobuilder = new Lang.Class({
execute: function(args, loop, cancellable) {
this._initWorkdir(null, cancellable);
+ this._buildsDir = new VersionedDir.VersionedDir(this.workdir.get_child('builds'), this._VERSION_RE);
+
if (args.autoupdate_self)
this._autoupdate_self = Gio.File.new_for_path(args.autoupdate_self);
@@ -61,7 +68,7 @@ const Autobuilder = new Lang.Class({
this._impl = Gio.DBusExportedObject.wrapJSObject(AutoBuilderIface, this);
this._impl.export(Gio.DBus.session, '/org/gnome/OSTreeBuild/AutoBuilder');
- this._taskmaster = new Task.TaskMaster(this.workdir.get_child('tasks'),
+ this._taskmaster = new Task.TaskMaster(this.workdir,
{ onEmpty: Lang.bind(this, this._onTasksComplete) });
this._taskmaster.connect('task-executing', Lang.bind(this, this._onTaskExecuting));
this._taskmaster.connect('task-complete', Lang.bind(this, this._onTaskCompleted));
@@ -88,6 +95,9 @@ const Autobuilder = new Lang.Class({
},
_onTaskCompleted: function(taskmaster, task, success, error) {
+ if (!task.changed)
+ GSystem.shutil_rm_rf(task.buildPath, cancellable);
+
if (task.name == 'resolve')
this._runResolve();
if (success) {
@@ -134,6 +144,46 @@ const Autobuilder = new Lang.Class({
return true;
},
+ _getLastVersion: function(cancellable) {
+ let allVersions = this._buildsDir.loadVersions(cancellable);
+ if (allVersions.length > 0)
+ return allVersions[allVersions.length-1];
+ else
+ return null;
+ },
+
+ _getNextBuildDirectory: function(cancellable) {
+ let currentTime = GLib.DateTime.new_now_utc();
+ let currentYmd = Format.vprintf('%d%02d%02d', [currentTime.get_year(),
+ currentTime.get_month(),
+ currentTime.get_day_of_month()]);
+
+ let version = null;
+ let lastVersion = this._getLastVersion(cancellable);
+ if (lastVersion) {
+ let match = this._VERSION_RE.exec(lastVersion);
+ if (!match) throw new Error();
+ let lastYmd = match[1];
+ let lastSerial = match[2];
+ if (lastYmd == currentYmd) {
+ version = currentYmd + '.' + (parseInt(lastSerial) + 1);
+ }
+ }
+ if (version === null) {
+ version = currentYmd + '.0';
+ }
+
+ let buildPath = this._buildsDir.path.get_child(version);
+ GSystem.file_ensure_directory(buildPath, true, cancellable);
+
+ if (lastVersion) {
+ let lastBuildPath = this._buildsDir.path.get_child(lastVersion);
+ BuildUtil.atomicSymlinkSwap(buildPath.get_child('last-build'), lastBuildPath);
+ }
+
+ return buildPath;
+ },
+
_runResolve: function() {
let cancellable = null;
@@ -149,14 +199,15 @@ const Autobuilder = new Lang.Class({
ProcUtil.runSync(['git', 'pull', '-r'], cancellable,
{ cwd: this._autoupdate_self })
+ let buildPath = this._getNextBuildDirectory(cancellable);
if (this._initialResolveNeeded) {
this._initialResolveNeeded = false;
- this._taskmaster.pushTask('resolve', { });
+ this._taskmaster.startBuild('resolve', buildPath, { });
} else if (this._fullResolveNeeded) {
this._fullResolveNeeded = false;
- this._taskmaster.pushTask('resolve', { fetchAll: true });
+ this._taskmaster.startBuild('resolve', buildPath, { fetchAll: true });
} else {
- this._taskmaster.pushTask('resolve', { fetchSrcUrls: this._resolveSrcUrls });
+ this._taskmaster.startBuild('resolve', buildPath, { fetchSrcUrls: this._resolveSrcUrls });
}
this._resolveSrcUrls = [];
diff --git a/src/js/builtins/make.js b/src/js/builtins/make.js
index db35fc3..256179c 100644
--- a/src/js/builtins/make.js
+++ b/src/js/builtins/make.js
@@ -37,6 +37,7 @@ const Make = new Lang.Class({
this.parser.addArgument(['-x', '--skip'], { action: 'append',
help: "Don't process tasks after this" });
this.parser.addArgument('taskname');
+ this.parser.addArgument('buildPath');
this.parser.addArgument('parameters', { nargs: '*' });
},
@@ -47,7 +48,7 @@ const Make = new Lang.Class({
this._cancellable = cancellable;
this._tasksComplete = false;
this._oneOnly = args.only;
- let taskmaster = new Task.TaskMaster(this.workdir.get_child('tasks'),
+ let taskmaster = new Task.TaskMaster(this.workdir,
{ onEmpty: Lang.bind(this, this._onTasksComplete),
processAfter: !args.only,
skip: args.skip });
@@ -55,7 +56,8 @@ const Make = new Lang.Class({
taskmaster.connect('task-executing', Lang.bind(this, this._onTaskExecuting));
taskmaster.connect('task-complete', Lang.bind(this, this._onTaskCompleted));
let params = this._parseParameters(args.parameters);
- taskmaster.pushTask(args.taskname, params);
+ let buildPath = Gio.File.new_for_path(args.buildPath);
+ taskmaster.startBuild(buildPath, args.taskname, params);
loop.run();
if (!this._failed)
print("Success!")
diff --git a/src/js/snapshot.js b/src/js/snapshot.js
index 281ee6e..4af4ecd 100644
--- a/src/js/snapshot.js
+++ b/src/js/snapshot.js
@@ -18,6 +18,7 @@
const Lang = imports.lang;
+const JsonUtil = imports.jsonutil;
const Params = imports.params;
function _componentDict(snapshot) {
@@ -65,6 +66,11 @@ function snapshotDiff(a, b) {
return [added, modified, removed];
}
+function fromFile(path, cancellable) {
+ let data = JsonUtil.loadJson(path, cancellable);
+ return new Snapshot(data, path);
+}
+
const Snapshot = new Lang.Class({
Name: 'Snapshot',
diff --git a/src/js/task.js b/src/js/task.js
index 7ce83dc..9a78c0b 100644
--- a/src/js/task.js
+++ b/src/js/task.js
@@ -25,10 +25,8 @@ const GSystem = imports.gi.GSystem;
const OSTree = imports.gi.OSTree;
const Params = imports.params;
const JsonUtil = imports.jsonutil;
-const JsonDB = imports.jsondb;
const ProcUtil = imports.procutil;
const BuildUtil = imports.buildutil;
-const VersionedDir = imports.versioneddir;
const DefaultTaskDef = {
TaskName: '',
@@ -134,6 +132,10 @@ const TaskMaster = new Lang.Class({
this._skipTasks = {};
for (let i = 0; i < params.skip.length; i++)
this._skipTasks[params.skip[i]] = true;
+
+ this.tasksPath = this.path.get_child('tasks');
+ GSystem.file_ensure_directory(this.tasksPath, true, null);
+
this.maxConcurrent = GLib.get_num_processors();
this._onEmpty = params.onEmpty;
this.cancellable = null;
@@ -150,12 +152,28 @@ const TaskMaster = new Lang.Class({
this._scheduledTaskTimeouts = {};
},
+ _getTaskBuildPath: function(taskName) {
+ let buildPath = this.tasksPath.resolve_relative_path(taskName);
+ return GSystem.file_realpath(buildPath);
+ },
+
+ _setTaskBuildPath: function(taskName, buildPath) {
+ let taskLink = this.tasksPath.get_child(taskName);
+ BuildUtil.atomicSymlinkSwap(taskLink, buildPath, this.cancellable);
+ return buildPath;
+ },
+
_pushTaskDataImmediate: function(taskData) {
this._pendingTasksList.push(taskData);
this._queueRecalculate();
},
- pushTask: function(name, parameters) {
+ startBuild: function(buildPath, taskName, parameters) {
+ this._setTaskBuildPath(taskName, buildPath);
+ this._pushTask(taskName, parameters);
+ },
+
+ _pushTask: function(name, parameters) {
let taskDef = this._taskset.getTaskDef(name);
let taskData = new TaskData(taskDef, parameters);
if (!this._isTaskPending(name)) {
@@ -279,19 +297,24 @@ const TaskMaster = new Lang.Class({
if (idx == -1)
throw new Error("TaskMaster: Internal error - Failed to find completed task:" +
runner.taskData.name);
this._executing.splice(idx, 1);
+
this.emit('task-complete', runner, success, error);
if (success && this._processAfter) {
- if (runner.changed) {
- let taskName = runner.taskData.name;
- let taskDef = runner.taskData.taskDef;
- let after = this._taskset.getTasksAfter(taskName);
- for (let i = 0; i < after.length; i++) {
+ let taskName = runner.taskData.name;
+ if (success && runner.changed) {
+ let taskDef = runner.taskData.taskDef;
+ let buildPath = this._getTaskBuildPath(taskName);
+ let after = this._taskset.getTasksAfter(taskName);
+ for (let i = 0; i < after.length; i++) {
let afterTaskName = after[i];
- if (!this._skipTasks[afterTaskName])
+ if (!this._skipTasks[afterTaskName]) {
+ this._setTaskBuildPath(afterTaskName, buildPath);
this.pushTask(afterTaskName, {});
- }
+ }
+ }
}
- }
+ }
+
this._queueRecalculate();
},
@@ -322,6 +345,7 @@ const Task = new Lang.Class({
this.workdir = Gio.File.new_for_path(GLib.getenv('_OSTBUILD_WORKDIR'));
BuildUtil.checkIsWorkDirectory(this.workdir);
+ this.builddir = Gio.File.new_for_path(GLib.getenv('_OSTBUILD_BUILDDIR'));
this.resultdir = this.workdir.get_child('results');
GSystem.file_ensure_directory(this.resultdir, true, null);
@@ -338,11 +362,6 @@ const Task = new Lang.Class({
this.ostreeRepo.open(null);
},
- _getResultDb: function(taskname) {
- let path = this.resultdir.resolve_relative_path(taskname);
- return new JsonDB.JsonDB(path);
- },
-
execute: function(cancellable) {
throw new Error("Not implemented");
},
@@ -351,85 +370,36 @@ const Task = new Lang.Class({
const TaskRunner = new Lang.Class({
Name: 'TaskRunner',
- _VERSION_RE: /^(\d+\d\d\d\d)\.(\d+)$/,
-
_init: function(taskmaster, taskData, onComplete) {
this.taskmaster = taskmaster;
this.taskData = taskData;
this.onComplete = onComplete;
this.name = taskData.name;
- this.workdir = taskmaster.path.get_parent();
+ this.workdir = taskmaster.path;
BuildUtil.checkIsWorkDirectory(this.workdir);
},
- _loadAllVersions: function(cancellable) {
- let allVersions = [];
-
- let successVersions = this._successDir.loadVersions(cancellable);
- for (let i = 0; i < successVersions.length; i++) {
- allVersions.push([true, successVersions[i]]);
- }
-
- let failedVersions = this._failedDir.loadVersions(cancellable);
- for (let i = 0; i < failedVersions.length; i++) {
- allVersions.push([false, failedVersions[i]]);
- }
-
- allVersions.sort(function (a, b) {
- let [successA, versionA] = a;
- let [successB, versionB] = b;
- return BuildUtil.compareVersions(versionA, versionB);
- });
-
- return allVersions;
- },
-
executeInSubprocess: function(cancellable) {
this._cancellable = cancellable;
this._startTimeMillis = GLib.get_monotonic_time() / 1000;
- this.dir = this.taskmaster.path.resolve_relative_path(this.name);
- GSystem.file_ensure_directory(this.dir, true, cancellable);
-
- this._topDir = new VersionedDir.VersionedDir(this.dir, this._VERSION_RE);
- this._successDir = new VersionedDir.VersionedDir(this.dir.get_child('successful'),
- this._VERSION_RE);
- this._failedDir = new VersionedDir.VersionedDir(this.dir.get_child('failed'),
- this._VERSION_RE);
-
- let allVersions = this._loadAllVersions(cancellable);
-
- let currentTime = GLib.DateTime.new_now_utc();
-
- let currentYmd = Format.vprintf('%d%02d%02d', [currentTime.get_year(),
- currentTime.get_month(),
- currentTime.get_day_of_month()]);
- let version = null;
- if (allVersions.length > 0) {
- let [lastSuccess, lastVersion] = allVersions[allVersions.length-1];
- let match = this._VERSION_RE.exec(lastVersion);
- if (!match) throw new Error();
- let lastYmd = match[1];
- let lastSerial = match[2];
- if (lastYmd == currentYmd) {
- version = currentYmd + '.' + (parseInt(lastSerial) + 1);
- }
- }
- if (version === null) {
- version = currentYmd + '.0';
- }
+ // To prevent tasks from stomping on each other's toes, we put the task
+ // cwd in its own task dir. If a task has any results it wants to pass
+ // on between builds, it needs to write to _OSTBUILD_BUILDDIR.
+ let buildPath = this.taskmaster.tasksPath.resolve_relative_path(this.name);
+ buildPath = GSystem.file_realpath(buildPath);
- this._version = version;
- this._taskCwd = this.dir.get_child(version);
- GSystem.shutil_rm_rf(this._taskCwd, cancellable);
- GSystem.file_ensure_directory(this._taskCwd, true, cancellable);
+ this._buildName = buildPath.get_basename();
+ this._taskCwd = buildPath.get_child(this.name);
+ GSystem.file_ensure_directory(this._taskCwd, false, cancellable);
let baseArgv = ['ostbuild', 'run-task', this.name, JSON.stringify(this.taskData.parameters)];
let context = new GSystem.SubprocessContext({ argv: baseArgv });
context.set_cwd(this._taskCwd.get_path());
let childEnv = GLib.get_environ();
+ childEnv.push('_OSTBUILD_BUILDDIR=' + buildPath.get_path());
childEnv.push('_OSTBUILD_WORKDIR=' + this.workdir.get_path());
context.set_environment(childEnv);
if (this.taskData.taskDef.PreserveStdout) {
@@ -447,20 +417,6 @@ const TaskRunner = new Lang.Class({
this._proc.wait(cancellable, Lang.bind(this, this._onChildExited));
},
- _updateIndex: function(cancellable) {
- let allVersions = this._loadAllVersions(cancellable);
-
- let fileList = [];
- for (let i = 0; i < allVersions.length; i++) {
- let [successful, version] = allVersions[i];
- let fname = (successful ? 'successful/' : 'failed/') + version;
- fileList.push(fname);
- }
-
- let index = { files: fileList };
- JsonUtil.writeJsonFileAtomic(this.dir.get_child('index.json'), index, cancellable);
- },
-
_onChildExited: function(proc, result) {
let cancellable = this._cancellable;
let [success, errmsg] = ProcUtil.asyncWaitCheckFinish(proc, result);
@@ -473,24 +429,16 @@ const TaskRunner = new Lang.Class({
this.changed = data['modified'];
}
- if (!success) {
- target = this._failedDir.path.get_child(this._version);
- GSystem.file_rename(this._taskCwd, target, null);
- this._taskCwd = target;
- this._failedDir.cleanOldVersions(this.taskData.taskDef.RetainFailed, null);
- this.onComplete(success, errmsg);
- } else {
- target = this._successDir.path.get_child(this._version);
- GSystem.file_rename(this._taskCwd, target, null);
- this._taskCwd = target;
- this._successDir.cleanOldVersions(this.taskData.taskDef.RetainSuccess, null);
- this.onComplete(success, null);
- }
+ this.onComplete(success, errmsg);
+
+ if (!this.changed)
+ return;
let elapsedMillis = GLib.get_monotonic_time() / 1000 - this._startTimeMillis;
let targetPath = this.workdir.get_relative_path(this._taskCwd);
+
let meta = { taskMetaVersion: 0,
- taskVersion: this._version,
+ buildName: this._buildName,
success: success,
errmsg: errmsg,
elapsedMillis: elapsedMillis,
@@ -502,12 +450,5 @@ const TaskRunner = new Lang.Class({
}
JsonUtil.writeJsonFileAtomic(this._taskCwd.get_child('meta.json'), meta, cancellable);
-
- // Also remove any old interrupted versions
- this._topDir.cleanOldVersions(0, null);
-
- this._updateIndex(cancellable);
-
- BuildUtil.atomicSymlinkSwap(this.dir.get_child('current'), target, cancellable);
}
});
diff --git a/src/js/tasks/task-bdiff.js b/src/js/tasks/task-bdiff.js
index d1cffff..d37c326 100644
--- a/src/js/tasks/task-bdiff.js
+++ b/src/js/tasks/task-bdiff.js
@@ -83,32 +83,21 @@ const TaskBdiff = new Lang.Class({
},
execute: function(cancellable) {
- let builddb = this._getResultDb('build');
- let latestPath = builddb.getLatestPath();
- if (!latestPath)
- throw new Error("No builds!")
- let latestBuildVersion = builddb.parseVersionStr(latestPath.get_basename());
+ let latestSnapshotPath = this.builddir.get_child('snapshot.json');
+ let previousSnapshotPath = this.builddir.get_child('last-build/snapshot.json');
+ if (!previousSnapshotPath.query_exists(cancellable))
+ return;
- let previousPath = builddb.getPreviousPath(latestPath);
- if (!previousPath)
- throw new Error("No build previous to " + latestBuildVersion)
-
- let latestBuildData = builddb.loadFromPath(latestPath, cancellable);
- let latestBuildSnapshot = new Snapshot.Snapshot(latestBuildData['snapshot'], null);
- let previousBuildData = builddb.loadFromPath(previousPath, cancellable);
- let previousBuildSnapshot = new Snapshot.Snapshot(previousBuildData['snapshot'], null);
+ let latestBuildSnapshot = Snapshot.fromFile(latestSnapshotPath, cancellable);
+ let previousBuildSnapshot = Snapshot.fromFile(previousSnapshotData, cancellable);
let added = [];
let modified = [];
let removed = [];
- let result = {fromBuildVersion: builddb.parseVersionStr(previousPath.get_basename()),
- toBuildVersion: builddb.parseVersionStr(latestPath.get_basename()),
- fromSrcVersion: builddb.parseVersionStr(previousBuildData['snapshotName']),
- toSrcVersion: builddb.parseVersionStr(latestBuildData['snapshotName']),
- added: added,
- modified: modified,
- removed: removed};
+ let result = { added: added,
+ modified: modified,
+ removed: removed };
let modifiedNames = [];
@@ -146,7 +135,6 @@ const TaskBdiff = new Lang.Class({
diffstat: diffstat });
}
- let bdiffdb = this._getResultDb('bdiff');
- bdiffdb.store(result, cancellable);
+ JsonUtil.writeJsonFileAtomic(this.builddir.get_child('bdiff.json'), result, cancellable);
}
});
diff --git a/src/js/tasks/task-build.js b/src/js/tasks/task-build.js
index de96aba..198a17a 100644
--- a/src/js/tasks/task-build.js
+++ b/src/js/tasks/task-build.js
@@ -30,7 +30,6 @@ const AsyncUtil = imports.asyncutil;
const ProcUtil = imports.procutil;
const StreamUtil = imports.streamutil;
const JsonUtil = imports.jsonutil;
-const JsonDB = imports.jsondb;
const Snapshot = imports.snapshot;
const BuildUtil = imports.buildutil;
const Vcs = imports.vcs;
@@ -1128,14 +1127,11 @@ const TaskBuild = new Lang.Class({
this.forceBuildComponents[this.parameters.forceComponents[i]] = true;
this.cachedPatchdirRevision = null;
- let snapshotDir = this.workdir.get_child('snapshots');
- let srcdb = new JsonDB.JsonDB(snapshotDir);
- let snapshotPath = srcdb.getLatestPath();
- let workingSnapshotPath = Gio.File.new_for_path(snapshotPath.get_basename());
+ let snapshotPath = this.builddir.get_child('snapshot.json');
+ let workingSnapshotPath = Gio.File.new_for_path('snapshot.json');
GSystem.file_linkcopy(snapshotPath, workingSnapshotPath, Gio.FileCopyFlags.OVERWRITE,
cancellable);
- let data = srcdb.loadFromPath(workingSnapshotPath, cancellable);
- this._snapshot = new Snapshot.Snapshot(data, workingSnapshotPath);
+ this._snapshot = Snapshot.fromFile(workingSnapshotPath, cancellable);
let osname = this._snapshot.data['osname'];
this.osname = osname;
@@ -1145,10 +1141,6 @@ const TaskBuild = new Lang.Class({
let components = this._snapshot.data['components'];
- let builddb = this._getResultDb('build');
-
- let targetSourceVersion = builddb.parseVersionStr(this._snapshot.path.get_basename());
-
// Pick up overrides from $workdir/overrides/$name
for (let i = 0; i < components.length; i++) {
let component = components[i];
@@ -1170,19 +1162,6 @@ const TaskBuild = new Lang.Class({
haveLocalComponent = true;
}
- let latestBuildPath = builddb.getLatestPath();
- if (latestBuildPath != null) {
- let lastBuiltSourceData = builddb.loadFromPath(latestBuildPath, cancellable);
- let lastBuiltSourceVersion = builddb.parseVersionStr(lastBuiltSourceData['snapshotName']);
- if (!haveLocalComponent && lastBuiltSourceVersion == targetSourceVersion) {
- print("Already built source snapshot " + lastBuiltSourceVersion);
- return;
- } else {
- print("Last successful build was " + lastBuiltSourceVersion);
- }
- }
- print("building " + targetSourceVersion);
-
this._componentBuildCachePath = this.cachedir.get_child('component-builds.json');
if (this._componentBuildCachePath.query_exists(cancellable)) {
this._componentBuildCache = JsonUtil.loadJson(this._componentBuildCachePath, cancellable);
@@ -1438,7 +1417,6 @@ const TaskBuild = new Lang.Class({
this._writeStatus('built: ' + this._rebuiltComponents.join(' '), cancellable);
- let [path, modified] = builddb.store(buildData, cancellable);
- print("Build complete: " + path.get_path());
+ JsonUtil.writeJsonFileAtomic(this.builddir.get_child('build.json'), buildData, cancellable);
}
});
diff --git a/src/js/tasks/task-builddisks.js b/src/js/tasks/task-builddisks.js
index 10d638e..f1218ea 100644
--- a/src/js/tasks/task-builddisks.js
+++ b/src/js/tasks/task-builddisks.js
@@ -29,7 +29,6 @@ const Task = imports.task;
const ProcUtil = imports.procutil;
const BuildUtil = imports.buildutil;
const LibQA = imports.libqa;
-const VersionedDir = imports.versioneddir;
const JsonUtil = imports.jsonutil;
const JSUtil = imports.jsutil;
const GuestFish = imports.guestfish;
@@ -48,39 +47,18 @@ const TaskBuildDisks = new Lang.Class({
// Legacy
_VERSION_RE: /^(\d+)\.(\d+)$/,
- _imageSubdir: 'images',
_inheritPreviousDisk: true,
_onlyTreeSuffixes: ['-runtime'],
execute: function(cancellable) {
- let baseImageDir = this.workdir.resolve_relative_path(this._imageSubdir);
- let baseImageVersionedDir = new VersionedDir.VersionedDir(baseImageDir, this._VERSION_RE);
- GSystem.file_ensure_directory(baseImageDir, true, cancellable);
- let currentImageLink = baseImageDir.get_child('current');
- let previousImageLink = baseImageDir.get_child('previous');
-
- let builddb = this._getResultDb('build');
-
- let latestPath = builddb.getLatestPath();
- let buildVersion = builddb.parseVersionStr(latestPath.get_basename());
- this._buildVersion = buildVersion;
- let buildData = builddb.loadFromPath(latestPath, cancellable);
-
- let targetImageDir = baseImageDir.get_child(buildVersion);
-
- if (targetImageDir.query_exists(null)) {
- print("Already created " + targetImageDir.get_path());
- return;
- }
+ let buildData = JsonUtil.loadJson(this.builddir.get_child('build.json'), cancellable);
+ let prevImageDir = this.builddir.get_child('last-build/images');
+ let targetImageDir = this.builddir.get_child('images');
let workImageDir = Gio.File.new_for_path('images');
GSystem.file_ensure_directory(workImageDir, true, cancellable);
- let destPath = workImageDir.get_child('build-' + buildVersion + '.json');
- GSystem.file_linkcopy(latestPath, destPath, Gio.FileCopyFlags.ALL_METADATA, cancellable);
-
let targets = buildData['targets'];
-
let osname = buildData['snapshot']['osname'];
let originRepoUrl = buildData['snapshot']['repo'];
@@ -98,7 +76,7 @@ const TaskBuildDisks = new Lang.Class({
let squashedName = osname + '-' + targetName.substr(targetName.lastIndexOf('/') + 1);
let diskName = squashedName + '.qcow2';
let diskPath = workImageDir.get_child(diskName);
- let prevPath = currentImageLink.get_child(diskName);
+ let prevPath = prevImageDir.get_child(diskName);
GSystem.shutil_rm_rf(diskPath, cancellable);
let doCloneDisk = this._inheritPreviousDisk && prevPath.query_exists(null);
if (doCloneDisk) {
@@ -128,24 +106,6 @@ const TaskBuildDisks = new Lang.Class({
}
GSystem.file_rename(workImageDir, targetImageDir, cancellable);
-
- let currentInfo = null;
- try {
- currentInfo = currentImageLink.query_info('standard::symlink-target',
Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS, cancellable);
- } catch (e) {
- if (!e.matches(Gio.IOErrorEnum, Gio.IOErrorEnum.NOT_FOUND))
- throw e;
- }
- if (currentInfo != null) {
- let newPreviousTmppath = baseImageDir.get_child('previous-new.tmp');
- let currentLinkTarget = currentInfo.get_symlink_target();
- GSystem.shutil_rm_rf(newPreviousTmppath, cancellable);
- newPreviousTmppath.make_symbolic_link(currentLinkTarget, cancellable);
- GSystem.file_rename(newPreviousTmppath, previousImageLink, cancellable);
- }
- BuildUtil.atomicSymlinkSwap(baseImageDir.get_child('current'), targetImageDir, cancellable);
-
- baseImageVersionedDir.cleanOldVersions(IMAGE_RETAIN_COUNT, cancellable);
},
_postDiskCreation: function(squashedName, diskPath, cancellable) {
diff --git a/src/js/tasks/task-resolve.js b/src/js/tasks/task-resolve.js
index 94d334c..8b91e84 100644
--- a/src/js/tasks/task-resolve.js
+++ b/src/js/tasks/task-resolve.js
@@ -18,7 +18,6 @@
const Gio = imports.gi.Gio;
const Lang = imports.lang;
-const JsonDB = imports.jsondb;
const Task = imports.task;
const ProcUtil = imports.procutil;
const JsonUtil = imports.jsonutil;
@@ -38,12 +37,22 @@ const TaskResolve = new Lang.Class({
fetchComponents: [],
timeoutSec: 10},
- _getDb: function() {
- if (this._db == null) {
- let snapshotdir = this.workdir.get_child('snapshots');
- this._db = new JsonDB.JsonDB(snapshotdir);
- }
- return this._db;
+ _writeSnapshotToBuild: function(cancellable) {
+ let data = this._snapshot.data;
+ let buf = JsonUtil.serializeJson(data);
+
+ let oldSnapshot = this.builddir.get_child('last-build/snapshot.json');
+ if (oldSnapshot.query_exists(cancellable)) {
+ let oldBytes = GSystem.file_map_readonly(snapshotPath, cancellable);
+ let oldCsum = GLib.compute_checksum_for_bytes(GLib.ChecksumType.SHA256, oldBytes);
+ let newCsum = GLib.compute_checksum_for_string(GLib.ChecksumType.SHA256, buf, -1);
+ if (oldCsum == newCsum)
+ return false;
+ }
+
+ let snapshot = this.builddir.get_child('snapshot.json');
+ JsonUtil.writeJsonFileAtomic(snapshot, data, cancellable);
+ return true;
},
execute: function(cancellable) {
@@ -77,11 +86,11 @@ const TaskResolve = new Lang.Class({
component['revision'] = revision;
}
- let [path, modified] = this._getDb().store(this._snapshot.data, cancellable);
+ let modified = this._writeSnapshotToBuild(cancellable);
if (modified) {
- print("New source snapshot: " + path.get_path());
+ print("New source snapshot");
} else {
- print("Source snapshot unchanged: " + path.get_path());
+ print("Source snapshot unchanged");
}
let modifiedPath = Gio.File.new_for_path('modified.json');
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
