[gnome-continuous] Add support for "child components"

[Owen Taylor <otaylor src gnome org>]

commit 95095be820e0d16dfaa5d675ed204815f79173fc
Author: Owen W. Taylor <otaylor fishsoup net>
Date:   Mon Jul 7 19:39:49 2014 -0400

    Add support for "child components"
    
    A child component is a bit like a submodule - it's a directory inside a parent
    module that is checked out from version control. But instead of being
    configured in git, it is configured in manifest.json - the reason for
    this is to allow us to rebuild the outer module whenever the inner
    module changes, rather than having a fixed version of the inner module.
    
    Child components must have unique names globally within manifest.json - this
    is because they are treated like main components in certain contexts:
    in particular names of child components can be passed in the
    'fetchComponents' argument of 'resolve' and child components appear
    independently in the bdiff output.
    
    This will be used for the kernel build where the outer module has our
    configuration and build scripts, and the inner module is the actual upstream
    kernel tree.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=732395

 src/js/builtins/checkout.js   |   10 +++++
 src/js/builtins/git_mirror.js |   30 +++++++++----
 src/js/snapshot.js            |   33 ++++++++++++++-
 src/js/tasks/task-build.js    |   90 ++++++++++++++++++++++++++++++----------
 src/js/tasks/task-resolve.js  |   64 ++++++++++++++++++-----------
 5 files changed, 168 insertions(+), 59 deletions(-)
---
diff --git a/src/js/builtins/checkout.js b/src/js/builtins/checkout.js
index 27a0afd..767a536 100644
--- a/src/js/builtins/checkout.js
+++ b/src/js/builtins/checkout.js
@@ -89,6 +89,16 @@ function _checkoutOneComponent(workdir, mirrordir, patchdir, component, cancella
     let metadataPath = checkoutdir.get_child('_ostbuild-meta.json');
     JsonUtil.writeJsonFileAtomic(metadataPath, component, cancellable);
 
+    if (component['child-components']) {
+        let childComponents = component['child-components'];
+        for (let i = 0; i < childComponents.length; i++) {
+           let childParams = {};
+           Lang.copyProperties(params, childParams);
+            childParams.checkoutdir = checkoutdir.get_child(childComponents[i]['name']).get_path();
+            _checkoutOneComponent(workdir, mirrordir, patchdir, childComponents[i], cancellable, 
childParams);
+        }
+    }
+
     print("Checked out " + component['name'] + " at " + component['revision'] + " in " + 
checkoutdir.get_path());
 }
 
diff --git a/src/js/builtins/git_mirror.js b/src/js/builtins/git_mirror.js
index 9e4d024..88e2835 100644
--- a/src/js/builtins/git_mirror.js
+++ b/src/js/builtins/git_mirror.js
@@ -42,6 +42,23 @@ const GitMirror = new Lang.Class({
         this.parser.addArgument('components', {nargs:'*'});
     },
 
+    _mirrorComponent: function(component, args, recurse, cancellable) {
+        if (!args.fetch) {
+            Vcs.ensureVcsMirror(this.mirrordir, component, cancellable);
+       } else {
+           print("Running git fetch for " + component['name']);
+           Vcs.fetch(this.mirrordir, component, cancellable,
+                     { keepGoing:args.keep_going,
+                       timeoutSec: args.timeout_sec });
+       }
+
+        if (recurse && component['child-components']) {
+            let childComponents = component['child-components'];
+            for (let i = 0; i < childComponents.length; i++)
+                this._mirrorComponent(childComponents[i], args, recurse, cancellable);
+        }
+    },
+
     execute: function(args, loop, cancellable) {
        this._initWorkdir(args.workdir, cancellable);
 
@@ -56,23 +73,18 @@ const GitMirror = new Lang.Class({
        }
 
        let componentNames;
+        let recurse;
         if (args.components.length == 0) {
            componentNames = this._snapshot.getAllComponentNames();
+            recurse = true;
         } else {
             componentNames = args.components;
+            recurse = false;
        }
 
        componentNames.forEach(Lang.bind(this, function (name) {
            let component = this._snapshot.getComponent(name);
-
-            if (!args.fetch) {
-                Vcs.ensureVcsMirror(this.mirrordir, component, cancellable);
-           } else {
-               print("Running git fetch for " + name);
-               Vcs.fetch(this.mirrordir, component, cancellable,
-                         { keepGoing:args.keep_going,
-                           timeoutSec: args.timeout_sec });
-           }
+            this._mirrorComponent(component, args, recurse, cancellable);
        }));
 
        return true;
diff --git a/src/js/snapshot.js b/src/js/snapshot.js
index c8b1a72..c442f50 100644
--- a/src/js/snapshot.js
+++ b/src/js/snapshot.js
@@ -30,6 +30,16 @@ function _componentDict(snapshot) {
        if (r[name])
             throw new Error("Duplicate component name " + name);
         r[name] = component;
+
+        if (component['child-components']) {
+            let childComponents = component['child-components'];
+            for (let j = 0; j < childComponents.length; j++) {
+                let childName = childComponents[j]['name'];
+               if (r[childName])
+                    throw new Error("Duplicate component name " + chidlName);
+                r[childName] = childComponents[j];
+            }
+        }
     }
 
     let patches = snapshot['patches'];
@@ -84,12 +94,21 @@ const Snapshot = new Lang.Class({
            for (let i = 0; i < data['components'].length; i++) {
                let component = this._resolveComponent(data, data['components'][i]);
                data['components'][i] = component;
+                if (component['child-components']) {
+                    let childComponents = component['child-components'];
+                    for (let j = 0; j < childComponents.length; j++) {
+                        childComponents[j] = this._resolveComponent(data, childComponents[j]);
+                        childComponents[j]['is-child'] = true;
+                    }
+                }
            }
        }
        this._componentDict = _componentDict(data);
        this._componentNames = [];
-       for (let k in this._componentDict)
-           this._componentNames.push(k);
+       for (let k in this._componentDict) {
+            if (!this._componentDict[k]['is-child'])
+               this._componentNames.push(k);
+        }
     },
 
     _resolveComponent: function(manifest, componentMeta) {
@@ -161,20 +180,30 @@ const Snapshot = new Lang.Class({
                r['patches'] = patches;
            }
        }
+        let childComponents = component['child-components'];
+        if (childComponents) {
+            r['child-components'] = [];
+            for (let i = 0; i < childComponents.length; i++)
+                r['child-components'].push(this._expandComponent(childComponents[i]));
+        }
+
        let configOpts = (this.data['config-opts'] || []).concat();
        configOpts.push.apply(configOpts, component['config-opts'] || []);
        r['config-opts'] = configOpts;
        return r;
     },
 
+    /* child components are not included */
     getAllComponentNames: function() {
        return this._componentNames;
     },
 
+    /* the component map includes child components! */
     getComponentMap: function() {
        return this._componentDict;
     },
 
+    /* the name of a child component can be passed */
     getComponent: function(name, allowNone) {
        let r = this._componentDict[name] || null;
        if (!r && !allowNone)
diff --git a/src/js/tasks/task-build.js b/src/js/tasks/task-build.js
index 952a050..4d0df93 100644
--- a/src/js/tasks/task-build.js
+++ b/src/js/tasks/task-build.js
@@ -239,7 +239,7 @@ const TaskBuild = new Lang.Class({
                }
            }
        }
-            
+
         if (previousMetadata['patches']) {
             if (!newMetadata['patches']) {
                 return 'patches differ';
@@ -260,6 +260,26 @@ const TaskBuild = new Lang.Class({
        } else if (newMetadata['patches']) {
            return 'patches differ';
        }
+
+        if (previousMetadata['child-components']) {
+            if (!newMetadata['child-components'])
+                return 'child components differ';
+
+            let oldChildComponents = previousMetadata['child-components'];
+            let newChildComponents = newMetadata['child-components'];
+
+            if (oldChildComponents.length != newChildComponents.length)
+                return 'child components differ';
+
+            for (let i = 0; i <oldChildComponents.length; i++) {
+                if (this._needsRebuild(oldChildComponents[i], newChildComponents[i]))
+                    return 'child component ' + newChildComponents[i].name + ' differs';
+            }
+
+        } else if (newMetadata['child-components']) {
+           return 'child components differ';
+        }
+
         return null;
     },
 
@@ -576,28 +596,7 @@ const TaskBuild = new Lang.Class({
                            cancellable);
     },
 
-    _buildOneComponent: function(component, architecture, cancellable, params) {
-       params = Params.parse(params, { installedTests: false });
-        let basename = component['name'];
-
-       if (params.installedTests)
-           basename = basename + '-installed-tests';
-        let archBuildname = Format.vprintf('%s/%s', [basename, architecture]);
-        let unixBuildname = archBuildname.replace(/\//g, '_');
-        let buildRef = this._componentBuildRefFromName(basename, architecture);
-
-        let currentVcsVersion = component['revision'];
-        let expandedComponent = this._snapshot.getExpanded(component['name']);
-        let previousMetadata = this._componentBuildCache[buildRef];
-       let previousBuildVersion = null;
-       let previousVcsVersion = null;
-        if (previousMetadata != null) {
-            previousBuildVersion = previousMetadata['ostree'];
-            previousVcsVersion = previousMetadata['revision'];
-        } else {
-            print("No previous build for " + archBuildname);
-       }
-
+    _expandPatchDetails: function(expandedComponent, previousMetadata, cancellable) {
        let patchdir;
         if (expandedComponent['patches']) {
             let patchesRevision = expandedComponent['patches']['revision'];
@@ -627,6 +626,49 @@ const TaskBuild = new Lang.Class({
             patchdir = null;
        }
 
+        return patchdir;
+    },
+
+    _buildOneComponent: function(component, architecture, cancellable, params) {
+       params = Params.parse(params, { installedTests: false });
+        let basename = component['name'];
+
+       if (params.installedTests)
+           basename = basename + '-installed-tests';
+        let archBuildname = Format.vprintf('%s/%s', [basename, architecture]);
+        let unixBuildname = archBuildname.replace(/\//g, '_');
+        let buildRef = this._componentBuildRefFromName(basename, architecture);
+
+        let currentVcsVersion = component['revision'];
+        let expandedComponent = this._snapshot.getExpanded(component['name']);
+        let previousMetadata = this._componentBuildCache[buildRef];
+       let previousBuildVersion = null;
+       let previousVcsVersion = null;
+        if (previousMetadata != null) {
+            previousBuildVersion = previousMetadata['ostree'];
+            previousVcsVersion = previousMetadata['revision'];
+        } else {
+            print("No previous build for " + archBuildname);
+       }
+
+        let patchdir = this._expandPatchDetails(expandedComponent, previousMetadata, cancellable);
+        let childComponentPatchdirs = []
+
+        if (expandedComponent['child-components']) {
+            let childComponents = expandedComponent['child-components'];
+            for (let i = 0; i < childComponents.length; i++) {
+                let expandedChildComponent = childComponents[i];
+                let previousChildComponentMetadata;
+                if (previousMetadata != null && previousMetadata['child-components'] &&
+                    previousMetadata['child-components'].length == childComponents.length &&
+                    previousMetadata['child-components'][i]['name'] == expandedChildComponent['name'])
+                    previousChildComponentMetadata = previousMetadata['child-components'][i];
+                else
+                    previousChildComponentMetadata = null;
+                childComponentPatchdirs.push(this._expandPatchDetails(expandedChildComponent, 
previousChildComponentMetadata, cancellable));
+            }
+        }
+
         let forceRebuild = (this.forceBuildComponents[basename] ||
                             expandedComponent['src'].indexOf('local:') == 0);
 
@@ -1250,6 +1292,8 @@ const TaskBuild = new Lang.Class({
             // We don't want to attempt to apply patches over top
             // of what the override has.
             delete component['patches'];
+            // We also assume that the override also has any child components
+            delete component['child-components'];
         }
     },
 
diff --git a/src/js/tasks/task-resolve.js b/src/js/tasks/task-resolve.js
index a729393..e470cf1 100644
--- a/src/js/tasks/task-resolve.js
+++ b/src/js/tasks/task-resolve.js
@@ -69,6 +69,44 @@ const TaskResolve = new Lang.Class({
        return commit;
     },
 
+    _storeComponentRevision: function(component, resolveCache, cancellable) {
+        let tagOrBranch = component['tag'] || component['branch'] || 'master';
+        let mirrordir;
+        let modifiedCache = false;
+
+        try {
+            mirrordir = Vcs.ensureVcsMirror(this.mirrordir, component, cancellable);
+        } catch (e) {
+            print("Failed to create mirror for component " + component['name']);
+            throw e;
+        }
+        let currentCommit = Vcs.revParse(mirrordir, tagOrBranch, cancellable);
+        let revision = null;
+        let cachedEntry = resolveCache[component['name']];
+        if (cachedEntry) {
+            let previousCommit = cachedEntry['revision'];
+            if (currentCommit == previousCommit)
+                revision = cachedEntry['describe'];
+        }
+        if (revision == null) {
+            print("Describe cache miss for " + component['name']);
+            revision = Vcs.describeVersion(mirrordir, tagOrBranch);
+            modifiedCache = true;
+            resolveCache[component['name']] = {'revision': currentCommit,
+                                               'describe': revision};
+        }
+        component['revision'] = revision;
+
+        if (component['child-components']) {
+            let childComponents = component['child-components'];
+            for (let i = 0; i < childComponents.length; i++) {
+                modifiedCache = this._storeComponentRevision(childComponents[i], resolveCache, cancellable);
+            }
+        }
+
+        return modifiedCache;
+    },
+
     execute: function(cancellable) {
         let manifestPath = this.workdir.get_child('manifest.json');
         this._snapshot = Snapshot.fromFile(manifestPath, cancellable, { prepareResolve: true });
@@ -101,31 +139,7 @@ const TaskResolve = new Lang.Class({
        let componentNames = this._snapshot.getAllComponentNames();
        for (let i = 0; i < componentNames.length; i++) {
            let component = this._snapshot.getComponent(componentNames[i]);
-           let tagOrBranch = component['tag'] || component['branch'] || 'master';
-            let mirrordir;
-
-           try {
-               mirrordir = Vcs.ensureVcsMirror(this.mirrordir, component, cancellable);
-           } catch (e) {
-               print("Failed to create mirror for component " + component['name']);
-               throw e;
-           }
-           let currentCommit = Vcs.revParse(mirrordir, tagOrBranch, cancellable);
-           let revision = null;
-           let cachedEntry = resolveCache[component['name']];
-           if (cachedEntry) {
-               let previousCommit = cachedEntry['revision'];
-               if (currentCommit == previousCommit)
-                   revision = cachedEntry['describe'];
-           }
-           if (revision == null) {
-               print("Describe cache miss for " + component['name']);
-               revision = Vcs.describeVersion(mirrordir, tagOrBranch);
-               modifiedCache = true;
-               resolveCache[component['name']] = {'revision': currentCommit,
-                                                  'describe': revision};
-           }
-            component['revision'] = revision;
+            modifiedCache = this._storeComponentRevision(component, resolveCache, cancellable);
        }
 
        if (modifiedCache)