[gnome-ostree] build: Check out and commit final trees asynchronously
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-ostree] build: Check out and commit final trees asynchronously
- Date: Wed, 1 May 2013 00:26:51 +0000 (UTC)
commit 84d729ba227176666c0bb42e6723ddcb9cc4cada
Author: Colin Walters <walters verbum org>
Date: Mon Apr 29 21:31:20 2013 -0400
build: Check out and commit final trees asynchronously
This is a major speedup because we do stuff like the gtk icon cache
and such in parallel for 3 trees.
Makefile-ostbuild.am | 1 +
src/js/asyncutil.js | 71 +++++++++++++
src/js/tasks/task-build.js | 243 ++++++++++++++++++++++++++++++++++----------
3 files changed, 261 insertions(+), 54 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 278dfd4..f0a5ba3 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -40,6 +40,7 @@ utilsdir = $(pkglibdir)
jsostbuilddir=$(pkgdatadir)/js
jsostbuild_DATA= \
src/js/argparse.js \
+ src/js/asyncutil.js \
src/js/buildutil.js \
src/js/builtin.js \
src/js/fileutil.js \
diff --git a/src/js/asyncutil.js b/src/js/asyncutil.js
new file mode 100644
index 0000000..a5c3d73
--- /dev/null
+++ b/src/js/asyncutil.js
@@ -0,0 +1,71 @@
+// Copyright (C) 2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+
+const AsyncSet = new Lang.Class({
+ Name: 'AsyncSet',
+
+ _init: function(callback, cancellable) {
+ this._callback = callback;
+ this._cancellable = cancellable;
+ this._results = {};
+ this._err = null;
+ this._children = [];
+ },
+
+ addGAsyncResult: function(name, callback) {
+ this._children.push(callback);
+ let wrapped = Lang.bind(this, function(object, result) {
+ let success = false;
+ try {
+ this._results[name] = callback(object, result);
+ success = true;
+ } catch (e) {
+ if (this._cancellable)
+ this._cancellable.cancel();
+ if (!this._err) {
+ this._err = e.toString();
+ this._checkCallback();
+ return;
+ }
+ }
+
+ let i;
+ for (i = 0; i < this._children.length; i++) {
+ let child = this._children[i];
+ if (child === callback) {
+ break;
+ }
+ }
+ if (i == this._children.length)
+ throw new Error("Failed to find child task");
+ this._children.splice(i, 1);
+ this._checkCallback();
+ });
+ return wrapped;
+ },
+
+ _checkCallback: function() {
+ if (this._err)
+ this._callback(null, this._err);
+ else if (this._children.length == 0)
+ this._callback(this._results, null);
+ }
+});
diff --git a/src/js/tasks/task-build.js b/src/js/tasks/task-build.js
index 25bc577..91ec9b3 100644
--- a/src/js/tasks/task-build.js
+++ b/src/js/tasks/task-build.js
@@ -27,6 +27,7 @@ const Task = imports.task;
const Params = imports.params;
const JsonDB = imports.jsondb;
const FileUtil = imports.fileutil;
+const AsyncUtil = imports.asyncutil;
const ProcUtil = imports.procutil;
const StreamUtil = imports.streamutil;
const JsonUtil = imports.jsonutil;
@@ -567,7 +568,7 @@ const TaskBuild = new Lang.Class({
let rootdir;
if (params.installedTests)
rootdir = this._composeBuildrootCore(buildWorkdir, basename, architecture,
- [[this._installedTestsBuildrootRev, '/']], cancellable);
+ [[this._installedTestsBuildrootRev[architecture], '/']],
cancellable);
else
rootdir = this._composeBuildroot(buildWorkdir, basename, architecture, cancellable);
@@ -661,8 +662,9 @@ const TaskBuild = new Lang.Class({
return ostreeRevision;
},
- _checkoutOneTreeCore: function(name, composeContents, cancellable) {
+ _checkoutOneTreeCoreAsync: function(name, composeContents, cancellable, callback) {
let composeRootdir = this.subworkdir.get_child(name);
+ print("Checking out " + composeRootdir.get_path());
GSystem.shutil_rm_rf(composeRootdir, cancellable);
GSystem.file_ensure_directory(composeRootdir, true, cancellable);
@@ -677,21 +679,33 @@ const TaskBuild = new Lang.Class({
}
dataOut.close(cancellable);
- ProcUtil.runSync(['ostree', '--repo=' + this.repo.get_path(),
- 'checkout', '--user-mode', '--union',
- '--from-file=' + contentsTmpPath.get_path(), composeRootdir.get_path()],
- cancellable,
- {logInitiation: true});
- GSystem.file_unlink(contentsTmpPath, cancellable);
-
- let contentsPath = composeRootdir.resolve_relative_path('usr/share/contents.json');
- GSystem.file_ensure_directory(contentsPath.get_parent(), true, cancellable);
- JsonUtil.writeJsonFileAtomic(contentsPath, this._snapshot.data, cancellable);
+ let argv = ['ostree', '--repo=' + this.repo.get_path(),
+ 'checkout', '--user-mode', '--union',
+ '--from-file=' + contentsTmpPath.get_path(), composeRootdir.get_path()];
+ print("Running: " + argv.map(GLib.shell_quote).join(' '));
+ let proc = GSystem.Subprocess.new_simple_argv(argv,
+ GSystem.SubprocessStreamDisposition.INHERIT,
+ GSystem.SubprocessStreamDisposition.INHERIT,
+ cancellable);
+ proc.wait(cancellable, Lang.bind(this, function(proc, result) {
+ GSystem.file_unlink(contentsTmpPath, cancellable);
+ let [success, ecode] = proc.wait_finish(result);
+ try {
+ GLib.spawn_check_exit_status(ecode);
+ } catch (e) {
+ callback(null, ""+e);
+ return;
+ }
+
+ let contentsPath = composeRootdir.resolve_relative_path('usr/share/contents.json');
+ GSystem.file_ensure_directory(contentsPath.get_parent(), true, cancellable);
+ JsonUtil.writeJsonFileAtomic(contentsPath, this._snapshot.data, cancellable);
- return composeRootdir;
+ callback(composeRootdir, null);
+ }));
},
- _checkoutOneTree: function(target, componentBuildRevs, cancellable) {
+ _checkoutOneTreeAsync: function(target, componentBuildRevs, cancellable, callback) {
let base = target['base'];
let baseName = this.osname + '/bases/' + base['name'];
let runtimeName = this.osname +'/bases/' + base['runtime'];
@@ -737,17 +751,23 @@ const TaskBuild = new Lang.Class({
}
}
- let composeRootdir = this._checkoutOneTreeCore(target['name'], composeContents, cancellable);
-
- let shareOstree = composeRootdir.resolve_relative_path('usr/share/ostree');
- GSystem.file_ensure_directory(shareOstree, true, cancellable);
- let triggersRunPath = shareOstree.get_child('triggers-run');
- triggersRunPath.create(Gio.FileCreateFlags.REPLACE_DESTINATION, cancellable).close(cancellable);
-
- return [composeRootdir, relatedTmpPath];
+ this._checkoutOneTreeCoreAsync(target['name'], composeContents, cancellable,
+ Lang.bind(this, function(result, err) {
+ if (err) {
+ callback(null, err);
+ return;
+ } else {
+ let composeRootdir = result;
+ let shareOstree =
composeRootdir.resolve_relative_path('usr/share/ostree');
+ GSystem.file_ensure_directory(shareOstree, true, cancellable);
+ let triggersRunPath = shareOstree.get_child('triggers-run');
+
triggersRunPath.create(Gio.FileCreateFlags.REPLACE_DESTINATION, cancellable).close(cancellable);
+ callback([composeRootdir, relatedTmpPath], null);
+ }
+ }));
},
-
- _commitComposedTree: function(targetName, composeRootdir, relatedTmpPath, cancellable) {
+
+ _commitComposedTreeAsync: function(targetName, composeRootdir, relatedTmpPath, cancellable, callback) {
let treename = this.osname + '/' + targetName;
let args = ['ostree', '--repo=' + this.repo.get_path(),
'commit', '-b', treename, '-s', 'Compose',
@@ -755,13 +775,44 @@ const TaskBuild = new Lang.Class({
'--skip-if-unchanged'];
if (relatedTmpPath !== null)
args.push('--related-objects-file=' + relatedTmpPath.get_path());
- let ostreeRevision = ProcUtil.runSyncGetOutputUTF8Stripped(args, cancellable,
- {cwd: composeRootdir.get_path(),
- logInitiation: true});
- if (relatedTmpPath !== null)
- GSystem.file_unlink(relatedTmpPath, cancellable);
- GSystem.shutil_rm_rf(composeRootdir, cancellable);
- return [treename, ostreeRevision];
+
+ let membuf = Gio.MemoryOutputStream.new_resizable();
+
+ let asyncSet = new AsyncUtil.AsyncSet(Lang.bind(this, function(results, err) {
+ if (relatedTmpPath !== null)
+ GSystem.file_unlink(relatedTmpPath, cancellable);
+ GSystem.shutil_rm_rf(composeRootdir, cancellable);
+ if (err) {
+ callback(null, err);
+ return;
+ }
+ let revision = membuf.steal_as_bytes().toArray().toString();
+ revision = revision.replace(/[ \n]+$/, '');
+ print("Compose of " + targetName + " is " + revision);
+ callback([treename, revision], null);
+
+ }), cancellable);
+ print("Running: " + args.map(GLib.shell_quote).join(' '));
+ let context = new GSystem.SubprocessContext({ argv: args });
+ context.set_stdout_disposition(GSystem.SubprocessStreamDisposition.PIPE);
+ context.set_cwd(composeRootdir.get_path());
+ let proc = new GSystem.Subprocess({ context: context });
+ proc.init(cancellable);
+ let stdout = proc.get_stdout_pipe();
+ membuf.splice_async(stdout,
+ Gio.OutputStreamSpliceFlags.CLOSE_SOURCE |
Gio.OutputStreamSpliceFlags.CLOSE_TARGET,
+ GLib.PRIORITY_DEFAULT,
+ cancellable,
+ asyncSet.addGAsyncResult("splice",
+ Lang.bind(this, function(stream, result) {
+ stream.splice_finish(result);
+ })));
+ proc.wait(cancellable,
+ asyncSet.addGAsyncResult("wait",
+ Lang.bind(this, function(proc, result) {
+ let [success, ecode] = proc.wait_finish(result);
+ GLib.spawn_check_exit_status(ecode);
+ })));
},
_generateInitramfs: function(architecture, composeRootdir, initramfsDepends, cancellable) {
@@ -825,8 +876,8 @@ const TaskBuild = new Lang.Class({
'dracut', '--tmpdir=/tmp', '-f', '/tmp/initramfs-ostree.img',
kernelRelease];
+ print("Running: " + args.map(GLib.shell_quote).join(' '));
let context = new GSystem.SubprocessContext({ argv: args });
- print("Starting child process " + JSON.stringify(context.argv));
let proc = new GSystem.Subprocess({ context: context });
proc.init(cancellable);
proc.wait_sync_check(cancellable);
@@ -1138,6 +1189,7 @@ const TaskBuild = new Lang.Class({
}
}
+ this._installedTestsBuildrootRev = {};
let targetRevisions = {};
let finalInstalledTestRevisions = {};
let buildData = { snapshotName: this._snapshot.path.get_basename(),
@@ -1145,6 +1197,10 @@ const TaskBuild = new Lang.Class({
targets: targetRevisions };
buildData['installed-tests'] = finalInstalledTestRevisions;
+ let composeTreeTaskCount = 0;
+ let composeTreeTaskError = null;
+ let composeTreeTaskLoop = GLib.MainLoop.new(null, true);
+
// First loop over the -devel trees per architecture, and
// generate an initramfs.
let archInitramfsImages = {};
@@ -1164,20 +1220,45 @@ const TaskBuild = new Lang.Class({
initramfsDepends.push(component['name'] + ':' + buildRev);
}
- let [composeRootdir, relatedTmpPath] = this._checkoutOneTree(develTarget, componentBuildRevs,
cancellable);
-
- let [kernelRelease, initramfsPath] = this._generateInitramfs(architecture, composeRootdir,
initramfsDepends, cancellable);
- archInitramfsImages[architecture] = [kernelRelease, initramfsPath];
- let initramfsTargetName = 'initramfs-' + kernelRelease + '.img';
- let targetInitramfsPath =
composeRootdir.resolve_relative_path('boot').get_child(initramfsTargetName);
- GSystem.file_linkcopy(initramfsPath, targetInitramfsPath, Gio.FileCopyFlags.ALL_METADATA,
cancellable);
- let [treename, ostreeRev] = this._commitComposedTree(develTargetName, composeRootdir,
relatedTmpPath, cancellable);
- targetRevisions[treename] = ostreeRev;
- // Also note the revision of this, since it will be used
- // as the buildroot for installed tests
- this._installedTestsBuildrootRev = ostreeRev;
+ composeTreeTaskCount++;
+ this._checkoutOneTreeAsync(develTarget, componentBuildRevs, cancellable,
+ Lang.bind(this, function (result, err) {
+ if (err) {
+ if (composeTreeTaskError === null)
+ composeTreeTaskError = err;
+ composeTreeTaskLoop.quit();
+ return;
+ }
+ let [composeRootdir, relatedTmpPath] = result;
+ let [kernelRelease, initramfsPath] =
this._generateInitramfs(architecture, composeRootdir, initramfsDepends, cancellable);
+ archInitramfsImages[architecture] = [kernelRelease, initramfsPath];
+ let initramfsTargetName = 'initramfs-' + kernelRelease + '.img';
+ let targetInitramfsPath =
composeRootdir.resolve_relative_path('boot').get_child(initramfsTargetName);
+ GSystem.file_linkcopy(initramfsPath, targetInitramfsPath,
Gio.FileCopyFlags.ALL_METADATA, cancellable);
+ this._commitComposedTreeAsync(develTargetName, composeRootdir,
relatedTmpPath, cancellable,
+ Lang.bind(this, function(result,
err) {
+ if (err) {
+ if (composeTreeTaskError ===
null)
+ composeTreeTaskError =
err;
+ composeTreeTaskLoop.quit();
+ return;
+ }
+ composeTreeTaskCount--;
+ let [treename, ostreeRev] =
result;
+ targetRevisions[treename] =
ostreeRev;
+ // Also note the revision of
this, since it will be used
+ // as the buildroot for
installed tests
+
this._installedTestsBuildrootRev[architecture] = ostreeRev;
+ if (composeTreeTaskCount == 0)
+ composeTreeTaskLoop.quit();
+ }));
+ }));
}
+ composeTreeTaskLoop.run();
+ if (composeTreeTaskError)
+ throw new Error(composeTreeTaskError);
+
// Now loop over the other targets per architecture, reusing
// the initramfs cached from -devel generation.
let nonDevelTargets = ['runtime', 'runtime-debug', 'devel-debug'];
@@ -1188,16 +1269,44 @@ const TaskBuild = new Lang.Class({
let runtimeTargetName = 'buildmaster/' + architecture + '-' + target;
let runtimeTarget = this._findTargetInList(runtimeTargetName, targetsList);
- let [composeRootdir, relatedTmpPath] = this._checkoutOneTree(runtimeTarget,
componentBuildRevs, cancellable);
- let [kernelRelease, initramfsPath] = archInitramfsImages[architecture];
- let initramfsTargetName = 'initramfs-' + kernelRelease + '.img';
- let targetInitramfsPath =
composeRootdir.resolve_relative_path('boot').get_child(initramfsTargetName);
- GSystem.file_linkcopy(initramfsPath, targetInitramfsPath, Gio.FileCopyFlags.ALL_METADATA,
cancellable);
- let [treename, ostreeRev] = this._commitComposedTree(runtimeTargetName, composeRootdir,
relatedTmpPath, cancellable);
- targetRevisions[treename] = ostreeRev;
+ composeTreeTaskCount++;
+ this._checkoutOneTreeAsync(runtimeTarget, componentBuildRevs, cancellable,
+ Lang.bind(this, function(result, err) {
+ if (err) {
+ if (composeTreeTaskError === null)
+ composeTreeTaskError = err;
+ composeTreeTaskLoop.quit();
+ return;
+ }
+ composeTreeTaskCount--;
+ let [composeRootdir, relatedTmpPath] = result;
+ let [kernelRelease, initramfsPath] =
archInitramfsImages[architecture];
+ let initramfsTargetName = 'initramfs-' + kernelRelease +
'.img';
+ let targetInitramfsPath =
composeRootdir.resolve_relative_path('boot').get_child(initramfsTargetName);
+ GSystem.file_linkcopy(initramfsPath, targetInitramfsPath,
Gio.FileCopyFlags.ALL_METADATA, cancellable);
+ composeTreeTaskCount++;
+ this._commitComposedTreeAsync(runtimeTargetName,
composeRootdir, relatedTmpPath, cancellable,
+ Lang.bind(this, function(result,
err) {
+ if (err) {
+ if (composeTreeTaskError
=== null)
+ composeTreeTaskError
= err;
+
composeTreeTaskLoop.quit();
+ return;
+ }
+ composeTreeTaskCount--;
+ let [treename, ostreeRev] =
result;
+ targetRevisions[treename] =
ostreeRev;
+ if (composeTreeTaskCount ==
0)
+
composeTreeTaskLoop.quit();
+ }));
+ }));
}
}
+ composeTreeTaskLoop.run();
+ if (composeTreeTaskError)
+ throw new Error(composeTreeTaskError);
+
let installedTestComponentNames = this._snapshot.data['installed-tests-components'] || [];
print("Using installed test components: " + installedTestComponentNames.join(', '));
let installedTestRevs = {};
@@ -1233,11 +1342,37 @@ const TaskBuild = new Lang.Class({
for (let j = 0; j < revs.length; j++) {
composeContents.push([revs[j], '/runtime']);
}
- let composeRootdir = this._checkoutOneTreeCore(rootName, composeContents, cancellable);
- let [treename, rev] = this._commitComposedTree(rootName, composeRootdir, null, cancellable);
- finalInstalledTestRevisions[treename] = rev;
+ composeTreeTaskCount++;
+ this._checkoutOneTreeCoreAsync(rootName, composeContents, cancellable,
+ Lang.bind(this, function(result, err) {
+ if (err) {
+ if (composeTreeTaskError === null)
+ composeTreeTaskError = err;
+ composeTreeTaskLoop.quit();
+ return;
+ }
+ let composeRootdir = result;
+ this._commitComposedTreeAsync(rootName, composeRootdir, null,
cancellable,
+ Lang.bind(this, function(result,
err) {
+ if (err) {
+ if (composeTreeTaskError
=== null)
+ composeTreeTaskError
= err;
+
composeTreeTaskLoop.quit();
+ return;
+ }
+ let [treename, rev] = result;
+
finalInstalledTestRevisions[treename] = rev;
+ composeTreeTaskCount--;
+ if (composeTreeTaskCount ==
0)
+
composeTreeTaskLoop.quit();
+ }));
+ }));
}
+ composeTreeTaskLoop.run();
+ if (composeTreeTaskError)
+ throw new Error(composeTreeTaskError);
+
let [path, modified] = builddb.store(buildData, cancellable);
print("Build complete: " + path.get_path());
}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
