[gnome-ostree/wip/gjs-round2: 4/9] Port git-mirror to gjs
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-ostree/wip/gjs-round2: 4/9] Port git-mirror to gjs
- Date: Tue, 11 Dec 2012 04:14:26 +0000 (UTC)
commit 775af52f4eb27ea3413787a59dd55f21a4f955be
Author: Colin Walters <walters verbum org>
Date: Sun Dec 9 14:47:08 2012 -0500
Port git-mirror to gjs
Continuing the JS porting.
Makefile-ostbuild.am | 2 +-
src/ostbuild/js/argparse.js | 33 ++++++--
src/ostbuild/js/buildutil.js | 44 ++++++++++
src/ostbuild/js/git_mirror.js | 107 +++++++++++++++++++++++++
src/ostbuild/js/params.js | 1 -
src/ostbuild/js/procutil.js | 5 +-
src/ostbuild/js/vcs.js | 101 +++++++++++++++++++++--
src/ostbuild/pyostbuild/builtin_git_mirror.py | 105 ------------------------
src/ostbuild/pyostbuild/main.py | 6 +-
9 files changed, 275 insertions(+), 129 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index bae58e7..a67e35f 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -48,7 +48,6 @@ pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/builtin_deploy_qemu.py \
src/ostbuild/pyostbuild/builtin_deploy_root.py \
src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py \
- src/ostbuild/pyostbuild/builtin_git_mirror.py \
src/ostbuild/pyostbuild/builtin_resolve.py \
src/ostbuild/pyostbuild/builtin_source_diff.py \
src/ostbuild/pyostbuild/builtins.py \
@@ -77,6 +76,7 @@ jsostbuild_DATA= \
src/ostbuild/js/buildutil.js \
src/ostbuild/js/checkout.js \
src/ostbuild/js/config.js \
+ src/ostbuild/js/git_mirror.js \
src/ostbuild/js/jsondb.js \
src/ostbuild/js/jsonutil.js \
src/ostbuild/js/params.js \
diff --git a/src/ostbuild/js/argparse.js b/src/ostbuild/js/argparse.js
index c6e6ef1..b1867ca 100644
--- a/src/ostbuild/js/argparse.js
+++ b/src/ostbuild/js/argparse.js
@@ -45,6 +45,8 @@ const ArgumentParser = new Lang.Class({
if (opts.action == undefined)
opts.action = 'store';
+ if (opts.nargs == undefined)
+ opts.nargs = '1';
if (names.length == 0) {
throw new Error("Must specify at least one argument");
@@ -72,7 +74,7 @@ const ArgumentParser = new Lang.Class({
this._optNames[name] = opts;
if (opts._varName == null) {
if (name.indexOf('--') == 0)
- opts._varName = name.substr(2);
+ opts._varName = name.substr(2).replace(/-/g, '_');
else if (shortOpt == null && name[0] == '-' && name[1] != '-')
shortOpt = name.substr(1);
}
@@ -100,7 +102,11 @@ const ArgumentParser = new Lang.Class({
}
}
for (let name in this._argNames) {
- result[name] = null;
+ let opts = this._argNames[name];
+ if (opts.nargs == '*')
+ result[name] = [];
+ else
+ result[name] = null;
}
let rest = [];
@@ -118,9 +124,13 @@ const ArgumentParser = new Lang.Class({
if (!opts) this._failed();
if (opts.action == 'store') {
- if (i == argv.length - 1) this._failed();
- result[opts._varName] = argv[i+1];
- i++;
+ if (equalsIdx == -1) {
+ if (i == argv.length - 1) this._failed();
+ result[opts._varName] = argv[i+1];
+ i++;
+ } else {
+ result[opts._varName] = arg.substr(equalsIdx+1);
+ }
} else if (opts.action == 'storeTrue') {
result[opts._varName] = true;
i++;
@@ -131,10 +141,15 @@ const ArgumentParser = new Lang.Class({
}
for (let i = 0; i < this._namedArgs.length; i++) {
- let a = this._namedArgs[i];
- if (rest.length == 0) this._failed();
- let value = rest.shift();
- result[a._varName] = value;
+ let opts = this._namedArgs[i];
+ let varName = opts._varName;
+ if (opts.nargs == '*') {
+ result[varName].push.apply(result[varName], rest);
+ } else {
+ if (rest.length == 0) this._failed();
+ let value = rest.shift();
+ result[varName] = value;
+ }
}
return result;
diff --git a/src/ostbuild/js/buildutil.js b/src/ostbuild/js/buildutil.js
index 4a26373..8a615f6 100644
--- a/src/ostbuild/js/buildutil.js
+++ b/src/ostbuild/js/buildutil.js
@@ -37,3 +37,47 @@ function parseSrcKey(srckey) {
let uri = srckey.substr(idx+1);
return [keytype, uri];
}
+
+function resolveComponent(manifest, componentMeta) {
+ let result = {};
+ Lang.copyProperties(componentMeta, result);
+ let origSrc = componentMeta['src'];
+
+ let didExpand = false;
+ let vcsConfig = manifest['vcsconfig'];
+ for (let vcsprefix in vcsConfig) {
+ let expansion = vcsConfig[vcsprefix];
+ let prefix = vcsprefix + ':';
+ if (origSrc.indexOf(prefix) == 0) {
+ result['src'] = expansion + origSrc.substr(prefix.length);
+ didExpand = true;
+ break;
+ }
+ }
+
+ let name = componentMeta['name'];
+ let src, idx, name;
+ if (name == undefined) {
+ if (didExpand) {
+ src = origSrc;
+ idx = src.lastIndexOf(':');
+ name = src.substr(idx+1);
+ } else {
+ src = result['src'];
+ idx = src.lastIndexOf('/');
+ name = src.substr(idx+1);
+ }
+ if (name.lastIndexOf('.git') == name.length - 4) {
+ name = name.substr(0, name.length - 4);
+ }
+ name = name.replace(/\//g, '-');
+ result['name'] = name;
+ }
+
+ let branchOrTag = result['branch'] || result['tag'];
+ if (!branchOrTag) {
+ result['branch'] = 'master';
+ }
+
+ return result;
+}
diff --git a/src/ostbuild/js/git_mirror.js b/src/ostbuild/js/git_mirror.js
new file mode 100644
index 0000000..664ce6a
--- /dev/null
+++ b/src/ostbuild/js/git_mirror.js
@@ -0,0 +1,107 @@
+// Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const ProcUtil = imports.procutil;
+const Config = imports.config;
+const Snapshot = imports.snapshot;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const JsonUtil = imports.jsonutil;
+const JsonDB = imports.jsondb;
+const ArgParse = imports.argparse;
+
+var loop = GLib.MainLoop.new(null, true);
+
+const GitMirror = new Lang.Class({
+ Name: 'GitMirror',
+
+ _init: function() {
+
+ },
+
+ execute: function(argv) {
+ let cancellable = null;
+ let parser = new ArgParse.ArgumentParser("Update internal git mirror for one or more components");
+ parser.addArgument('--prefix');
+ parser.addArgument('--manifest');
+ parser.addArgument('--src-snapshot');
+ parser.addArgument('--fetch', {action:'storeTrue',
+ help:"Also do a git fetch for components"});
+ parser.addArgument(['-k', '--keep-going'], {action:'storeTrue',
+ help: "Don't exit on fetch failures"});
+ parser.addArgument('components', {nargs:'*'});
+
+ let args = parser.parse(argv);
+
+ this.config = Config.get();
+ this.workdir = Gio.File.new_for_path(this.config.getGlobal('workdir'));
+ this._mirrordir = Gio.File.new_for_path(this.config.getGlobal('mirrordir'));
+ this.prefix = args.prefix || this.config.getPrefix();
+ this._snapshotDir = this.workdir.get_child('snapshots');
+ this._srcDb = new JsonDB.JsonDB(this._snapshotDir, this.prefix + '-src-snapshot');
+
+ if (args.manifest != null) {
+ this._snapshot = JsonUtil.loadJson(Gio.File.new_for_path(args.manifest), cancellable);
+ let resolvedComponents = [];
+ let components = this._snapshot['components'];
+ for (let i = 0; i < components.length; i++) {
+ resolvedComponents.push(BuildUtil.resolveComponent(this._snapshot, components[i]));
+ }
+ this._snapshot['components'] = resolvedComponents;
+ this._snapshot['patches'] = BuildUtil.resolveComponent(this._snapshot, this._snapshot['patches']);
+ } else {
+ this._snapshot = Snapshot.load(this._srcDb, this.prefix, args.snapshot, cancellable);
+ }
+
+ let componentNames;
+ if (args.components.length == 0) {
+ componentNames = [];
+ this._snapshot['components'].forEach(function (component) {
+ componentNames.push(component['name']);
+ });
+ componentNames.push(this._snapshot['patches']['name']);
+ } else {
+ componentNames = args.components;
+ }
+
+ componentNames.forEach(Lang.bind(this, function (name) {
+ let component = Snapshot.getComponent(this._snapshot, name);
+ let src = component['src']
+ let [keytype, uri] = Vcs.parseSrcKey(src);
+ let branch = component['branch'];
+ let tag = component['tag'];
+ let branchOrTag = branch || tag;
+
+ if (!args.fetch) {
+ Vcs.ensureVcsMirror(this._mirrordir, keytype, uri, branchOrTag, cancellable);
+ } else {
+ log("Running git fetch for " + name);
+ Vcs.fetch(this._mirrordir, keytype, uri, branchOrTag, cancellable, {keepGoing:args.keep_going});
+ }
+ }));
+ }
+});
+
+var gitMirror = new GitMirror();
+GLib.idle_add(GLib.PRIORITY_DEFAULT,
+ function() { try { gitMirror.execute(ARGV); } finally { loop.quit(); }; return false; });
+loop.run();
diff --git a/src/ostbuild/js/params.js b/src/ostbuild/js/params.js
index 3b3798b..dbdf970 100644
--- a/src/ostbuild/js/params.js
+++ b/src/ostbuild/js/params.js
@@ -1,4 +1,3 @@
-// -*- mode: js; js-indent-level: 4; indent-tabs-mode: nil -*-
// Taken from gnome-shell/js/misc/params.js under the GNU General Public License
// parse:
// @params: caller-provided parameter object, or %null
diff --git a/src/ostbuild/js/procutil.js b/src/ostbuild/js/procutil.js
index fe387a8..a42b654 100644
--- a/src/ostbuild/js/procutil.js
+++ b/src/ostbuild/js/procutil.js
@@ -6,8 +6,10 @@ const Params = imports.params;
function _setContextFromParams(context, params) {
params = Params.parse(params, {cwd: null});
- if (params.cwd)
+ if (typeof(params.cwd) == 'string')
context.set_cwd(params.cwd);
+ else if (params.cwd)
+ context.set_cwd(params.cwd.get_path());
}
function _wait_sync_check_internal(proc, cancellable) {
@@ -33,6 +35,7 @@ function runSync(args, cancellable, params) {
function _runSyncGetOutputInternal(args, cancellable, params, splitLines) {
params = Params.parse(params, {cwd: null});
let context = new GSystem.SubprocessContext({argv: args});
+ _setContextFromParams(context, params);
context.set_stdout_disposition(GSystem.SubprocessStreamDisposition.PIPE);
context.set_stderr_disposition(GSystem.SubprocessStreamDisposition.INHERIT);
let proc = new GSystem.Subprocess({context: context});
diff --git a/src/ostbuild/js/vcs.js b/src/ostbuild/js/vcs.js
index 168b474..34c4d9a 100644
--- a/src/ostbuild/js/vcs.js
+++ b/src/ostbuild/js/vcs.js
@@ -20,6 +20,7 @@ const Gio = imports.gi.Gio;
const GSystem = imports.gi.GSystem;
const ProcUtil = imports.procutil;
+const BuildUtil = imports.buildutil;
function getMirrordir(mirrordir, keytype, uri, params) {
params = Params.parse(params, {prefix: ''});
@@ -37,7 +38,7 @@ function getMirrordir(mirrordir, keytype, uri, params) {
function _fixupSubmoduleReferences(mirrordir, cwd, cancellable) {
let lines = ProcUtil.runSyncGetOutputLines(['git', 'submodule', 'status'],
- cancellable, {cwd: cwd.get_path()});
+ cancellable, {cwd: cwd});
let haveSubmodules = false;
for (let i = 0; i < lines.length; i++) {
let line = lines[i];
@@ -47,10 +48,10 @@ function _fixupSubmoduleReferences(mirrordir, cwd, cancellable) {
let [subChecksum, subName, rest] = line.split(' ');
let configKey = Format.vprintf('submodule.%s.url', [subName]);
let subUrl = ProcUtil.runSyncGetOutputUTF8(['git', 'config', '-f', '.gitmodules', configKey],
- cancellable, {cwd: cwd.get_path()});
+ cancellable, {cwd: cwd});
let localMirror = getMirrordir(mirrordir, 'git', subUrl);
ProcUtil.runSync(['git', 'config', configKey, 'file://' + localMirror.get_path()],
- cancellable, {cwd:cwd.get_path()});
+ cancellable, {cwd:cwd});
}
return haveSubmodules;
}
@@ -77,15 +78,15 @@ function getVcsCheckout(mirrordir, keytype, uri, dest, branch, cancellable, para
if (ftype != Gio.FileType.DIRECTORY) {
ProcUtil.runSync(['git', 'clone', '-q', '--origin', 'localmirror',
'--no-checkout', moduleMirror.get_path(), tmpDest.get_path()], cancellable);
- ProcUtil.runSync(['git', 'remote', 'add', 'upstream', uri], cancellable, {cwd: tmpDest.get_path()});
+ ProcUtil.runSync(['git', 'remote', 'add', 'upstream', uri], cancellable, {cwd: tmpDest});
} else {
- ProcUtil.runSync(['git', 'fetch', 'localmirror'], cancellable, {cwd: tmpDest.get_path()});
+ ProcUtil.runSync(['git', 'fetch', 'localmirror'], cancellable, {cwd: tmpDest});
}
- ProcUtil.runSync(['git', 'checkout', '-q', branch], cancellable, {cwd: tmpDest.get_path()});
- ProcUtil.runSync(['git', 'submodule', 'init'], cancellable, {cwd: tmpDest.get_path()});
+ ProcUtil.runSync(['git', 'checkout', '-q', branch], cancellable, {cwd: tmpDest});
+ ProcUtil.runSync(['git', 'submodule', 'init'], cancellable, {cwd: tmpDest});
let haveSubmodules = _fixupSubmoduleReferences(mirrordir, tmpDest, cancellable);
if (haveSubmodules) {
- ProcUtil.runSync(['git', 'submodule', 'update'], cancellable, {cwd: tmpDest.get_path()});
+ ProcUtil.runSync(['git', 'submodule', 'update'], cancellable, {cwd: tmpDest});
}
if (!tmpDest.equal(dest)) {
GSystem.file_rename(tmpDest, dest, cancellable);
@@ -95,7 +96,7 @@ function getVcsCheckout(mirrordir, keytype, uri, dest, branch, cancellable, para
function clean(keytype, checkoutdir, cancellable) {
ProcUtil.runSync(['git', 'clean', '-d', '-f', '-x'], cancellable,
- {cwd: checkoutdir.get_path()});
+ {cwd: checkoutdir});
}
function parseSrcKey(srckey) {
@@ -132,4 +133,86 @@ function checkoutPatches(mirrordir, patchdir, component, cancellable, params) {
return patchdir;
}
+function getLastfetchPath(mirrordir, keytype, uri, branch) {
+ let mirror = getMirrordir(mirrordir, keytype, uri);
+ let branchSafename = branch.replace(/[\/.]/g, '_');
+ return mirror.get_parent().get_child(mirror.get_basename() + '.lastfetch-' + branchSafename);
+}
+
+function _listSubmodules(mirrordir, mirror, keytype, uri, branch, cancellable) {
+ let currentVcsVersion = ProcUtil.runSyncGetOutputUTF8(['git', 'rev-parse', branch], cancellable,
+ {cwd: mirror}).replace(/[ \n]/g, '');
+ let tmpCheckout = getMirrordir(mirrordir, keytype, uri, {prefix:'_tmp-checkouts'});
+ GSystem.shutil_rm_rf(tmpCheckout, cancellable);
+ GSystem.file_ensure_directory(tmpCheckout.get_parent(), true, cancellable);
+ ProcUtil.runSync(['git', 'clone', '-q', '--no-checkout', mirror.get_path(), tmpCheckout.get_path()], cancellable);
+ ProcUtil.runSync(['git', 'checkout', '-q', '-f', currentVcsVersion], cancellable,
+ {cwd: tmpCheckout});
+ let submodules = []
+ let lines = ProcUtil.runSyncGetOutputLines(['git', 'submodule', 'status'],
+ cancellable, {cwd: tmpCheckout});
+ for (let i = 0; i < lines.length; i++) {
+ let line = lines[i];
+ if (line == '') continue;
+ line = line.substr(1);
+ let [subChecksum, subName, rest] = line.split(' ');
+ let subUrl = ProcUtil.runSyncGetOutputUTF8(['git', 'config', '-f', '.gitmodules',
+ Format.vprintf('submodule.%s.url', [subName])], cancellable,
+ {cwd: tmpCheckout});
+ submodules.push([subChecksum, subName, subUrl]);
+ }
+ GSystem.shutil_rm_rf(tmpCheckout, cancellable);
+ return submodules;
+}
+
+function ensureVcsMirror(mirrordir, keytype, uri, branch, cancellable,
+ params) {
+ params = Params.parse(params, {fetch: false,
+ fetchKeepGoing: false});
+ let mirror = getMirrordir(mirrordir, keytype, uri);
+ let tmpMirror = mirror.get_parent().get_child(mirror.get_basename() + '.tmp');
+ let didUpdate = false;
+ let lastFetchPath = getLastfetchPath(mirrordir, keytype, uri, branch);
+ let lastFetchContents = null;
+ if (lastFetchPath.query_exists(cancellable)) {
+ lastFetchContents = GSystem.file_load_contents_utf8(lastFetchPath, cancellable).replace(/[ \n]/g, '');
+ }
+ GSystem.shutil_rm_rf(tmpMirror, cancellable);
+ if (!mirror.query_exists(cancellable)) {
+ ProcUtil.runSync(['git', 'clone', '--mirror', uri, tmpMirror.get_path()], cancellable);
+ ProcUtil.runSync(['git', 'config', 'gc.auto', '0'], cancellable, {cwd: tmpMirror});
+ GSystem.file_rename(tmpMirror, mirror);
+ } else if (params.fetch) {
+ try {
+ ProcUtil.runSync(['git', 'fetch'], cancellable, {cwd:mirror});
+ } catch (e) {
+ if (!params.fetchKeepGoing)
+ throw e;
+ }
+ }
+
+ let currentVcsVersion = ProcUtil.runSyncGetOutputUTF8(['git', 'rev-parse', branch], cancellable,
+ {cwd: mirror}).replace(/[ \n]/g, '');
+
+ let changed = currentVcsVersion != lastFetchContents;
+ if (changed) {
+ log(Format.vprintf("last fetch %s differs from branch %s", [lastFetchContents, currentVcsVersion]));
+ _listSubmodules(mirrordir, mirror, keytype, uri, branch, cancellable).forEach(function (elt) {
+ let [subChecksum, subName, subUrl] = elt;
+ log("Processing submodule " + subName + " at " + subChecksum + " from " + subUrl);
+ ensureVcsMirror(mirrordir, keytype, subUrl, subChecksum, cancellable, {fetch:fetch});
+ });
+ }
+ if (changed) {
+ lastFetchPath.replace_contents(currentVcsVersion, null, false, 0, cancellable);
+ }
+
+ return mirror;
+}
+
+function fetch(mirrordir, keytype, uri, branch, cancellable, params) {
+ params = Params.parse(params, {keepGoing: false});
+ ensureVcsMirror(mirrordir, keytype, uri, branch, cancellable,
+ {fetch:true, fetchKeepGoing: params.keepGoing});
+}
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 327cea1..11d091c 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -25,14 +25,14 @@ from . import builtins
from . import builtin_build
from . import builtin_deploy_root
from . import builtin_deploy_qemu
-from . import builtin_git_mirror
from . import builtin_privhelper_deploy_qemu
from . import builtin_resolve
from . import builtin_source_diff
JS_BUILTINS = {'autobuilder': "Run resolve and build",
'checkout': "Check out source tree",
- 'prefix': "Display or modify \"prefix\" (build target)"};
+ 'prefix': "Display or modify \"prefix\" (build target)",
+ 'git-mirror': "Update internal git mirror for one or more components"};
def usage(ecode):
print "Builtins:"
@@ -58,7 +58,7 @@ def main(args):
print "error: Unknown builtin '%s'" % (args[0], )
return usage(1)
else:
- child_args = ['ostbuild-js', name]
+ child_args = ['ostbuild-js', name.replace('-', '_')]
child_args.extend(args[1:])
os.execvp('ostbuild-js', child_args)
return builtin.execute(args[1:])
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
