[gnome-ostree/wip/gjs-round2: 4/9] Port git-mirror to gjs



commit 775af52f4eb27ea3413787a59dd55f21a4f955be
Author: Colin Walters <walters verbum org>
Date:   Sun Dec 9 14:47:08 2012 -0500

    Port git-mirror to gjs
    
    Continuing the JS porting.

 Makefile-ostbuild.am                          |    2 +-
 src/ostbuild/js/argparse.js                   |   33 ++++++--
 src/ostbuild/js/buildutil.js                  |   44 ++++++++++
 src/ostbuild/js/git_mirror.js                 |  107 +++++++++++++++++++++++++
 src/ostbuild/js/params.js                     |    1 -
 src/ostbuild/js/procutil.js                   |    5 +-
 src/ostbuild/js/vcs.js                        |  101 +++++++++++++++++++++--
 src/ostbuild/pyostbuild/builtin_git_mirror.py |  105 ------------------------
 src/ostbuild/pyostbuild/main.py               |    6 +-
 9 files changed, 275 insertions(+), 129 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index bae58e7..a67e35f 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -48,7 +48,6 @@ pyostbuild_PYTHON =					\
 	src/ostbuild/pyostbuild/builtin_deploy_qemu.py	\
 	src/ostbuild/pyostbuild/builtin_deploy_root.py	\
 	src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py	\
-	src/ostbuild/pyostbuild/builtin_git_mirror.py	\
 	src/ostbuild/pyostbuild/builtin_resolve.py	\
 	src/ostbuild/pyostbuild/builtin_source_diff.py	\
 	src/ostbuild/pyostbuild/builtins.py		\
@@ -77,6 +76,7 @@ jsostbuild_DATA= \
 	src/ostbuild/js/buildutil.js \
 	src/ostbuild/js/checkout.js \
 	src/ostbuild/js/config.js \
+	src/ostbuild/js/git_mirror.js \
 	src/ostbuild/js/jsondb.js \
 	src/ostbuild/js/jsonutil.js \
 	src/ostbuild/js/params.js \
diff --git a/src/ostbuild/js/argparse.js b/src/ostbuild/js/argparse.js
index c6e6ef1..b1867ca 100644
--- a/src/ostbuild/js/argparse.js
+++ b/src/ostbuild/js/argparse.js
@@ -45,6 +45,8 @@ const ArgumentParser = new Lang.Class({
 
 	if (opts.action == undefined)
 	    opts.action = 'store';
+	if (opts.nargs == undefined)
+	    opts.nargs = '1';
 
 	if (names.length == 0) {
 	    throw new Error("Must specify at least one argument");
@@ -72,7 +74,7 @@ const ArgumentParser = new Lang.Class({
 		this._optNames[name] = opts;
 		if (opts._varName == null) {
 		    if (name.indexOf('--') == 0)
-			opts._varName = name.substr(2);
+			opts._varName = name.substr(2).replace(/-/g, '_');
 		    else if (shortOpt == null && name[0] == '-' && name[1] != '-')
 			shortOpt = name.substr(1);
 		}
@@ -100,7 +102,11 @@ const ArgumentParser = new Lang.Class({
 	    }
 	}
 	for (let name in this._argNames) {
-	    result[name] = null;
+	    let opts = this._argNames[name];
+	    if (opts.nargs == '*')
+		result[name] = [];
+	    else
+		result[name] = null;
 	}
 
 	let rest = [];
@@ -118,9 +124,13 @@ const ArgumentParser = new Lang.Class({
 		if (!opts) this._failed();
 
 		if (opts.action == 'store') {
-		    if (i == argv.length - 1) this._failed();
-		    result[opts._varName] = argv[i+1];
-		    i++;
+		    if (equalsIdx == -1) {
+			if (i == argv.length - 1) this._failed();
+			result[opts._varName] = argv[i+1];
+			i++;
+		    } else {
+			result[opts._varName] = arg.substr(equalsIdx+1);
+		    }
 		} else if (opts.action == 'storeTrue') {
 		    result[opts._varName] = true;
 		    i++;
@@ -131,10 +141,15 @@ const ArgumentParser = new Lang.Class({
 	}
 	
 	for (let i = 0; i < this._namedArgs.length; i++) {
-	    let a = this._namedArgs[i];
-	    if (rest.length == 0) this._failed();
-	    let value = rest.shift();
-	    result[a._varName] = value;
+	    let opts = this._namedArgs[i];
+	    let varName = opts._varName;
+	    if (opts.nargs == '*') {
+		result[varName].push.apply(result[varName], rest);
+	    } else {
+		if (rest.length == 0) this._failed();
+		let value = rest.shift();
+		result[varName] = value;
+	    }
 	}
 
 	return result;
diff --git a/src/ostbuild/js/buildutil.js b/src/ostbuild/js/buildutil.js
index 4a26373..8a615f6 100644
--- a/src/ostbuild/js/buildutil.js
+++ b/src/ostbuild/js/buildutil.js
@@ -37,3 +37,47 @@ function parseSrcKey(srckey) {
     let uri = srckey.substr(idx+1);
     return [keytype, uri];
 }
+
+function resolveComponent(manifest, componentMeta) {
+    let result = {};
+    Lang.copyProperties(componentMeta, result);
+    let origSrc = componentMeta['src'];
+
+    let didExpand = false;
+    let vcsConfig = manifest['vcsconfig'];
+    for (let vcsprefix in vcsConfig) {
+	let expansion = vcsConfig[vcsprefix];
+        let prefix = vcsprefix + ':';
+        if (origSrc.indexOf(prefix) == 0) {
+            result['src'] = expansion + origSrc.substr(prefix.length);
+            didExpand = true;
+            break;
+	}
+    }
+
+    let name = componentMeta['name'];
+    let src, idx, name;
+    if (name == undefined) {
+        if (didExpand) {
+            src = origSrc;
+            idx = src.lastIndexOf(':');
+            name = src.substr(idx+1);
+        } else {
+            src = result['src'];
+            idx = src.lastIndexOf('/');
+            name = src.substr(idx+1);
+	}
+        if (name.lastIndexOf('.git') == name.length - 4) {
+            name = name.substr(0, name.length - 4);
+	}
+        name = name.replace(/\//g, '-');
+        result['name'] = name;
+    }
+
+    let branchOrTag = result['branch'] || result['tag'];
+    if (!branchOrTag) {
+        result['branch'] = 'master';
+    }
+    
+    return result;
+}
diff --git a/src/ostbuild/js/git_mirror.js b/src/ostbuild/js/git_mirror.js
new file mode 100644
index 0000000..664ce6a
--- /dev/null
+++ b/src/ostbuild/js/git_mirror.js
@@ -0,0 +1,107 @@
+// Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const ProcUtil = imports.procutil;
+const Config = imports.config;
+const Snapshot = imports.snapshot;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const JsonUtil = imports.jsonutil;
+const JsonDB = imports.jsondb;
+const ArgParse = imports.argparse;
+
+var loop = GLib.MainLoop.new(null, true);
+
+const GitMirror = new Lang.Class({
+    Name: 'GitMirror',
+    
+    _init: function() {
+
+    },
+
+    execute: function(argv) {
+	let cancellable = null;
+        let parser = new ArgParse.ArgumentParser("Update internal git mirror for one or more components");
+        parser.addArgument('--prefix');
+        parser.addArgument('--manifest');
+        parser.addArgument('--src-snapshot');
+        parser.addArgument('--fetch', {action:'storeTrue',
+				       help:"Also do a git fetch for components"});
+        parser.addArgument(['-k', '--keep-going'], {action:'storeTrue',
+						    help: "Don't exit on fetch failures"});
+        parser.addArgument('components', {nargs:'*'});
+
+        let args = parser.parse(argv);
+
+	this.config = Config.get();
+	this.workdir = Gio.File.new_for_path(this.config.getGlobal('workdir'));
+	this._mirrordir = Gio.File.new_for_path(this.config.getGlobal('mirrordir'));
+	this.prefix = args.prefix || this.config.getPrefix();
+	this._snapshotDir = this.workdir.get_child('snapshots');
+	this._srcDb = new JsonDB.JsonDB(this._snapshotDir, this.prefix + '-src-snapshot');
+
+        if (args.manifest != null) {
+            this._snapshot = JsonUtil.loadJson(Gio.File.new_for_path(args.manifest), cancellable);
+	    let resolvedComponents = [];
+	    let components = this._snapshot['components'];
+	    for (let i = 0; i < components.length; i++) {
+		resolvedComponents.push(BuildUtil.resolveComponent(this._snapshot, components[i]));
+	    }
+            this._snapshot['components'] = resolvedComponents;
+            this._snapshot['patches'] = BuildUtil.resolveComponent(this._snapshot, this._snapshot['patches']);
+        } else {
+	    this._snapshot = Snapshot.load(this._srcDb, this.prefix, args.snapshot, cancellable);
+	}
+
+	let componentNames;
+        if (args.components.length == 0) {
+            componentNames = [];
+	    this._snapshot['components'].forEach(function (component) {
+		componentNames.push(component['name']);
+	    });
+            componentNames.push(this._snapshot['patches']['name']);
+        } else {
+            componentNames = args.components;
+	}
+
+	componentNames.forEach(Lang.bind(this, function (name) {
+            let component = Snapshot.getComponent(this._snapshot, name);
+            let src = component['src']
+            let [keytype, uri] = Vcs.parseSrcKey(src);
+            let branch = component['branch'];
+            let tag = component['tag'];
+            let branchOrTag = branch || tag;
+
+            if (!args.fetch) {
+                Vcs.ensureVcsMirror(this._mirrordir, keytype, uri, branchOrTag, cancellable);
+	    } else {
+		log("Running git fetch for " + name);
+		Vcs.fetch(this._mirrordir, keytype, uri, branchOrTag, cancellable, {keepGoing:args.keep_going});
+	    }
+	}));
+    }
+});
+
+var gitMirror = new GitMirror();
+GLib.idle_add(GLib.PRIORITY_DEFAULT,
+	      function() { try { gitMirror.execute(ARGV); } finally { loop.quit(); }; return false; });
+loop.run();
diff --git a/src/ostbuild/js/params.js b/src/ostbuild/js/params.js
index 3b3798b..dbdf970 100644
--- a/src/ostbuild/js/params.js
+++ b/src/ostbuild/js/params.js
@@ -1,4 +1,3 @@
-// -*- mode: js; js-indent-level: 4; indent-tabs-mode: nil -*-
 // Taken from gnome-shell/js/misc/params.js under the GNU General Public License
 // parse:
 // @params: caller-provided parameter object, or %null
diff --git a/src/ostbuild/js/procutil.js b/src/ostbuild/js/procutil.js
index fe387a8..a42b654 100644
--- a/src/ostbuild/js/procutil.js
+++ b/src/ostbuild/js/procutil.js
@@ -6,8 +6,10 @@ const Params = imports.params;
 
 function _setContextFromParams(context, params) {
     params = Params.parse(params, {cwd: null});
-    if (params.cwd)
+    if (typeof(params.cwd) == 'string')
 	context.set_cwd(params.cwd);
+    else if (params.cwd)
+	context.set_cwd(params.cwd.get_path());
 }
 
 function _wait_sync_check_internal(proc, cancellable) {
@@ -33,6 +35,7 @@ function runSync(args, cancellable, params) {
 function _runSyncGetOutputInternal(args, cancellable, params, splitLines) {
     params = Params.parse(params, {cwd: null});
     let context = new GSystem.SubprocessContext({argv: args});
+    _setContextFromParams(context, params);
     context.set_stdout_disposition(GSystem.SubprocessStreamDisposition.PIPE);
     context.set_stderr_disposition(GSystem.SubprocessStreamDisposition.INHERIT);
     let proc = new GSystem.Subprocess({context: context});
diff --git a/src/ostbuild/js/vcs.js b/src/ostbuild/js/vcs.js
index 168b474..34c4d9a 100644
--- a/src/ostbuild/js/vcs.js
+++ b/src/ostbuild/js/vcs.js
@@ -20,6 +20,7 @@ const Gio = imports.gi.Gio;
 const GSystem = imports.gi.GSystem;
 
 const ProcUtil = imports.procutil;
+const BuildUtil = imports.buildutil;
 
 function getMirrordir(mirrordir, keytype, uri, params) {
     params = Params.parse(params, {prefix: ''});
@@ -37,7 +38,7 @@ function getMirrordir(mirrordir, keytype, uri, params) {
 
 function _fixupSubmoduleReferences(mirrordir, cwd, cancellable) {
     let lines = ProcUtil.runSyncGetOutputLines(['git', 'submodule', 'status'],
-					       cancellable, {cwd: cwd.get_path()}); 
+					       cancellable, {cwd: cwd}); 
     let haveSubmodules = false;
     for (let i = 0; i < lines.length; i++) {
 	let line = lines[i];
@@ -47,10 +48,10 @@ function _fixupSubmoduleReferences(mirrordir, cwd, cancellable) {
         let [subChecksum, subName, rest] = line.split(' ');
 	let configKey = Format.vprintf('submodule.%s.url', [subName]);
         let subUrl = ProcUtil.runSyncGetOutputUTF8(['git', 'config', '-f', '.gitmodules', configKey],
-						   cancellable, {cwd: cwd.get_path()});
+						   cancellable, {cwd: cwd});
         let localMirror = getMirrordir(mirrordir, 'git', subUrl);
 	ProcUtil.runSync(['git', 'config', configKey, 'file://' + localMirror.get_path()],
-			 cancellable, {cwd:cwd.get_path()});
+			 cancellable, {cwd:cwd});
     }
     return haveSubmodules;
 }
@@ -77,15 +78,15 @@ function getVcsCheckout(mirrordir, keytype, uri, dest, branch, cancellable, para
     if (ftype != Gio.FileType.DIRECTORY) {
         ProcUtil.runSync(['git', 'clone', '-q', '--origin', 'localmirror',
 			  '--no-checkout', moduleMirror.get_path(), tmpDest.get_path()], cancellable);
-        ProcUtil.runSync(['git', 'remote', 'add', 'upstream', uri], cancellable, {cwd: tmpDest.get_path()});
+        ProcUtil.runSync(['git', 'remote', 'add', 'upstream', uri], cancellable, {cwd: tmpDest});
     } else {
-        ProcUtil.runSync(['git', 'fetch', 'localmirror'], cancellable, {cwd: tmpDest.get_path()});
+        ProcUtil.runSync(['git', 'fetch', 'localmirror'], cancellable, {cwd: tmpDest});
     }
-    ProcUtil.runSync(['git', 'checkout', '-q', branch], cancellable, {cwd: tmpDest.get_path()});
-    ProcUtil.runSync(['git', 'submodule', 'init'], cancellable, {cwd: tmpDest.get_path()});
+    ProcUtil.runSync(['git', 'checkout', '-q', branch], cancellable, {cwd: tmpDest});
+    ProcUtil.runSync(['git', 'submodule', 'init'], cancellable, {cwd: tmpDest});
     let haveSubmodules = _fixupSubmoduleReferences(mirrordir, tmpDest, cancellable);
     if (haveSubmodules) {
-        ProcUtil.runSync(['git', 'submodule', 'update'], cancellable, {cwd: tmpDest.get_path()});
+        ProcUtil.runSync(['git', 'submodule', 'update'], cancellable, {cwd: tmpDest});
     }
     if (!tmpDest.equal(dest)) {
         GSystem.file_rename(tmpDest, dest, cancellable);
@@ -95,7 +96,7 @@ function getVcsCheckout(mirrordir, keytype, uri, dest, branch, cancellable, para
 
 function clean(keytype, checkoutdir, cancellable) {
     ProcUtil.runSync(['git', 'clean', '-d', '-f', '-x'], cancellable,
-		     {cwd: checkoutdir.get_path()});
+		     {cwd: checkoutdir});
 }
 
 function parseSrcKey(srckey) {
@@ -132,4 +133,86 @@ function checkoutPatches(mirrordir, patchdir, component, cancellable, params) {
     return patchdir;
 }
 
+function getLastfetchPath(mirrordir, keytype, uri, branch) {
+    let mirror = getMirrordir(mirrordir, keytype, uri);
+    let branchSafename = branch.replace(/[\/.]/g, '_');
+    return mirror.get_parent().get_child(mirror.get_basename() + '.lastfetch-' + branchSafename);
+}
+
+function _listSubmodules(mirrordir, mirror, keytype, uri, branch, cancellable) {
+    let currentVcsVersion = ProcUtil.runSyncGetOutputUTF8(['git', 'rev-parse', branch], cancellable,
+							  {cwd: mirror}).replace(/[ \n]/g, '');
+    let tmpCheckout = getMirrordir(mirrordir, keytype, uri, {prefix:'_tmp-checkouts'});
+    GSystem.shutil_rm_rf(tmpCheckout, cancellable);
+    GSystem.file_ensure_directory(tmpCheckout.get_parent(), true, cancellable);
+    ProcUtil.runSync(['git', 'clone', '-q', '--no-checkout', mirror.get_path(), tmpCheckout.get_path()], cancellable);
+    ProcUtil.runSync(['git', 'checkout', '-q', '-f', currentVcsVersion], cancellable,
+		     {cwd: tmpCheckout});
+    let submodules = []
+    let lines = ProcUtil.runSyncGetOutputLines(['git', 'submodule', 'status'],
+					       cancellable, {cwd: tmpCheckout}); 
+    for (let i = 0; i < lines.length; i++) {
+	let line = lines[i];
+        if (line == '') continue;
+        line = line.substr(1);
+        let [subChecksum, subName, rest] = line.split(' ');
+        let subUrl = ProcUtil.runSyncGetOutputUTF8(['git', 'config', '-f', '.gitmodules',
+						    Format.vprintf('submodule.%s.url', [subName])], cancellable,
+						   {cwd: tmpCheckout});
+        submodules.push([subChecksum, subName, subUrl]);
+    }
+    GSystem.shutil_rm_rf(tmpCheckout, cancellable);
+    return submodules;
+}
+
+function ensureVcsMirror(mirrordir, keytype, uri, branch, cancellable,
+			 params) {
+    params = Params.parse(params, {fetch: false,
+				   fetchKeepGoing: false});
+    let mirror = getMirrordir(mirrordir, keytype, uri);
+    let tmpMirror = mirror.get_parent().get_child(mirror.get_basename() + '.tmp');
+    let didUpdate = false;
+    let lastFetchPath = getLastfetchPath(mirrordir, keytype, uri, branch);
+    let lastFetchContents = null;
+    if (lastFetchPath.query_exists(cancellable)) {
+	lastFetchContents = GSystem.file_load_contents_utf8(lastFetchPath, cancellable).replace(/[ \n]/g, '');
+    }
+    GSystem.shutil_rm_rf(tmpMirror, cancellable);
+    if (!mirror.query_exists(cancellable)) {
+        ProcUtil.runSync(['git', 'clone', '--mirror', uri, tmpMirror.get_path()], cancellable);
+        ProcUtil.runSync(['git', 'config', 'gc.auto', '0'], cancellable, {cwd: tmpMirror});
+        GSystem.file_rename(tmpMirror, mirror);
+    } else if (params.fetch) {
+	try {
+            ProcUtil.runSync(['git', 'fetch'], cancellable, {cwd:mirror});
+	} catch (e) {
+	    if (!params.fetchKeepGoing)
+		throw e;
+	}
+    }
+
+    let currentVcsVersion = ProcUtil.runSyncGetOutputUTF8(['git', 'rev-parse', branch], cancellable,
+							  {cwd: mirror}).replace(/[ \n]/g, '');
+
+    let changed = currentVcsVersion != lastFetchContents; 
+    if (changed) {
+        log(Format.vprintf("last fetch %s differs from branch %s", [lastFetchContents, currentVcsVersion]));
+	_listSubmodules(mirrordir, mirror, keytype, uri, branch, cancellable).forEach(function (elt) {
+	    let [subChecksum, subName, subUrl] = elt;
+	    log("Processing submodule " + subName + " at " + subChecksum + " from " + subUrl);
+            ensureVcsMirror(mirrordir, keytype, subUrl, subChecksum, cancellable, {fetch:fetch});
+	});
+    }
     
+    if (changed) {
+	lastFetchPath.replace_contents(currentVcsVersion, null, false, 0, cancellable); 
+    }
+
+    return mirror;
+}
+
+function fetch(mirrordir, keytype, uri, branch, cancellable, params) {
+    params = Params.parse(params, {keepGoing: false});
+    ensureVcsMirror(mirrordir, keytype, uri, branch, cancellable,
+		      {fetch:true, fetchKeepGoing: params.keepGoing});
+}
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 327cea1..11d091c 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -25,14 +25,14 @@ from . import builtins
 from . import builtin_build
 from . import builtin_deploy_root
 from . import builtin_deploy_qemu
-from . import builtin_git_mirror
 from . import builtin_privhelper_deploy_qemu
 from . import builtin_resolve
 from . import builtin_source_diff
 
 JS_BUILTINS = {'autobuilder': "Run resolve and build",
                'checkout': "Check out source tree",
-               'prefix': "Display or modify \"prefix\" (build target)"};
+               'prefix': "Display or modify \"prefix\" (build target)",
+               'git-mirror': "Update internal git mirror for one or more components"};
 
 def usage(ecode):
     print "Builtins:"
@@ -58,7 +58,7 @@ def main(args):
                 print "error: Unknown builtin '%s'" % (args[0], )
                 return usage(1)
             else:
-                child_args = ['ostbuild-js', name]
+                child_args = ['ostbuild-js', name.replace('-', '_')]
                 child_args.extend(args[1:])
                 os.execvp('ostbuild-js', child_args)
         return builtin.execute(args[1:])



[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]