[gnome-continuous/wip/platform1.7: 3/3] Use tags to pick components for different targets



commit 932cda02155d2c678c446e21c9b04dfd621ba023
Author: Alexander Larsson <alexl redhat com>
Date:   Thu Sep 25 11:57:11 2014 +0200

    Use tags to pick components for different targets
    
    Right now we're using a single-value "component" property to pick
    what targets to put each component in. Then there is some hardcoded
    logic to pick which of these to put in each target. In order to handle
    the hwtest and the runtime sharing packages a "minial" component was
    introduced to allow some components to be in multiple targets.
    
    However, such a setup requires a strict hierarchical definition of all
    components, which scales badly with adding more targets with different
    subsets. Instead we switch to a tag based system, where all the
    components which previously had "component = minimal" now have "tags =
    runtime, hwtest" and the other components just keeps the old component
    as the single tag.
    
    We also define a "targets" toplevel item that lists the targets
    to compose and which tags each should contain, and which subtrees
    to use when composing.
    
    This doesn't change what is built or how, but it makes things more
    flexible when adding more targets in the future.

 manifest.json              |  158 +++++++++++++++++++++++++------------------
 src/js/tasks/task-build.js |   94 +++++++++++---------------
 2 files changed, 131 insertions(+), 121 deletions(-)
---
diff --git a/manifest.json b/manifest.json
index 6c04eb2..c9d0a08 100644
--- a/manifest.json
+++ b/manifest.json
@@ -51,7 +51,7 @@
 
   "components": [
                 {"src": "gnome:gnome-continuous-kernel",
-                 "component": "minimal",
+                 "tags": ["runtime", "hwtest"],
                  "child-components": [
                      {"src": "kernel:stable/linux-stable",
                       "name": "linux",
@@ -59,17 +59,17 @@
                  ]},
 
                {"src": "gnome:gtk-doc-stub",
-                "component": "devel",
+                 "tags": ["devel"],
                 "noarch": true},
 
                {"src": "gnome:glib",
-                "component": "minimal",
+                "tags": ["runtime", "hwtest"],
                  "requires": {"pkgconfig": ["libffi >= 3.0.10"]},
                  "config-opts": ["--disable-modular-tests",
                                 "--enable-installed-tests"]},
 
                {"src": "gnome:gnome-common",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "savannah:gperf.git",
                 "tag": "v3.0.4",
@@ -95,11 +95,11 @@
                 "patches": ["usbutils-autogen.patch"]},
 
                 {"src": "git:git://git.kernel.org/pub/scm/utils/kernel/kmod/kmod.git",
-                "component": "minimal",
+                "tags": ["runtime", "hwtest"],
                 "config-opts": ["--disable-manpages"]},
 
                 {"src": "gnome:gobject-introspection",
-                 "component": "minimal",
+                 "tags": ["runtime", "hwtest"],
                 "config-opts": ["--disable-tests"]},
 
                {"src": "gnome:vala-bootstrap",
@@ -111,7 +111,7 @@
                 "config-opts": ["--enable-vapigen"]},
 
                {"src": "git:git://anongit.freedesktop.org/systemd/systemd",
-                "component": "minimal",
+                "tags": ["runtime", "hwtest"],
                 "initramfs-depends": true,
                 "cflags": {
                     "x86_64": "-O0 -g -m64 -mtune=generic",
@@ -133,12 +133,12 @@
                 "setuid": ["/runtime/usr/bin/linux-user-chroot"]},
 
                {"src": "gnome:libxml2",
-                "component": "minimal"},
+                "tags": ["runtime", "hwtest"]},
 
                {"src": "gnome:libxslt"},
 
                {"src": "gnome:libgsystem",
-                "component": "minimal"},
+                "tags": ["runtime", "hwtest"]},
 
                {"src": "fd:pixman"},
 
@@ -150,107 +150,107 @@
                 "config-opts": ["--with-cache-dir=/usr/lib/fontconfig/cache"]},
 
                {"src": "fd:xorg/util/macros",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/bigreqsproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/compositeproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/damageproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/dmxproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/dri2proto",
                 "tag": "ead89ad84877551cc15d26b95cb19a3e205df71f",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/dri3proto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/evieproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/fontsproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/glproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/inputproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/kbproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/presentproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/randrproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/recordproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/renderproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/resourceproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/scrnsaverproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/videoproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/xcmiscproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/xextproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/xf86bigfontproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/xf86dgaproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/xf86driproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/xf86miscproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/xf86vidmodeproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/xineramaproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/fixesproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/proto/xproto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/lib/libxtrans"},
 
                {"src": "fd:xorg/lib/libXau"},
 
                {"src": "fd:xcb/proto",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/font/util",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/lib/libfontenc"},
 
                {"src": "fd:xcb/pthread-stubs",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xcb/libxcb"},
 
@@ -307,7 +307,7 @@
                {"src": "fd:xorg/lib/libxshmfence"},
 
                {"src": "fd-dbus:dbus",
-                "component": "minimal",
+                "tags": ["runtime", "hwtest"],
                 "config-opts": ["--with-dbus-user=dbus",
                                 "--enable-inotify",
                                 "--enable-verbose-mode",
@@ -332,7 +332,7 @@
                {"src": "fd:wayland/libinput"},
 
                {"src": "fd:xorg/util/makedepend",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:xorg/lib/libpciaccess"},
 
@@ -463,7 +463,7 @@
                  "checksum": "913fc974433238ffd4e0549ce11ba2a3360d1d159cf5c3b988d72a77acb74d04"},
 
                 {"src": "git:https://github.com/libarchive/libarchive.git";,
-                "component": "minimal",
+                "tags": ["runtime", "hwtest"],
                 "config-opts": ["--without-lzo2"],
                 "tag": "v3.1.2",
                 "patches": ["libarchive-autogen.patch"]},
@@ -599,7 +599,7 @@
                {"src": "fd-fprint:fprintd"},
 
                {"src": "fd-p11:p11-kit",
-                 "component": "minimal"},
+                 "tags": ["runtime", "hwtest"]},
 
                {"src": "gnome:gcr",
                 "config-opts": ["--disable-valgrind"]},
@@ -690,7 +690,7 @@
                {"src": "gnome:yelp-tools"},
 
                {"src": "gnome:gsettings-desktop-schemas",
-                 "component": "minimal"},
+                 "tags": ["runtime", "hwtest"]},
 
                {"src": "gnome:gnome-desktop",
                 "config-opts": [
@@ -705,7 +705,7 @@
 
                {"src": "git:git://github.com/stephenc/tango-icon-naming.git",
                 "patches": ["tango-icon-naming-python.patch"],
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "fd:sound-theme-freedesktop"},
 
@@ -716,11 +716,11 @@
                {"src": "gnome:gnome-menus"},
 
                 {"src": "gnome:glib-networking",
-                "component": "minimal",
+                "tags": ["runtime", "hwtest"],
                 "config-opts": ["--enable-installed-tests"]},
 
                {"src": "gnome:libsoup",
-                "component": "minimal",
+                "tags": ["runtime", "hwtest"],
                 "config-args": [
                                "--disable-tls-check",
                                "--enable-installed-tests"]},
@@ -731,7 +731,7 @@
                                 "--enable-installed-tests"]},
 
                {"src": "gnome:gnome-ostree-integration",
-                 "component": "minimal"},
+                 "tags": ["runtime", "hwtest"]},
 
                {"src": "fd-gstreamer:gst-plugins-good",
                 "config-opts": ["--with-gtk=3.0",
@@ -824,7 +824,7 @@
 
                 {"src": "tarball:https://ftp.mozilla.org/pub/mozilla.org/js/mozjs-24.2.0.tar.bz2";,
                 "name": "js24",
-                 "component": "minimal",
+                 "tags": ["runtime", "hwtest"],
                 "checksum": "e62f3f331ddd90df1e238c09d61a505c516fe9fd8c5c95336611d191d18437d8",
                 "patches": ["js24-buildapi.patch", "js24-virtualenv.patch"],
                 "config-opts": ["--enable-threadsafe",
@@ -837,25 +837,25 @@
 
                {"src": "gnome:gjs",
                  "name": "gjs-minimal",
-                 "component": "hwtest",
+                 "tags": ["hwtest"],
                 "config-opts": ["--without-cairo", "--without-gtk"]},
 
                {"src": "gnome:gjs",
                 "config-opts": ["--enable-installed-tests"]},
 
                {"src": "gnome:ostree",
-                 "component": "minimal",
+                 "tags": ["runtime", "hwtest"],
                 "initramfs-depends": true,
                 "config-opts": ["--with-soup-gnome",
                                 "--with-dracut",
                                 "--enable-installed-tests"]},
 
                {"src": "gnome:gnome-hwtest",
-                 "component": "minimal",
+                 "tags": ["runtime", "hwtest"],
                  "initramfs-depends": true},
 
                {"src": "mikechristie:open-iscsi",
-                 "component": "minimal",
+                 "tags": ["runtime", "hwtest"],
                 "initramfs-depends": true,
                 "patches": ["iscsi-no-iscsiuio.patch",
                              "iscsi-no-static-build.patch",
@@ -864,21 +864,21 @@
                              "iscsi-buildsys.patch"]},
 
                 {"src": "socat:socat.git",
-                 "component": "minimal",
+                 "tags": ["runtime", "hwtest"],
                 "patches": ["socat-disable-man-pages.patch",
                              "socat-buildsys.patch"]},
 
                 {"src": "dnsmasq:dnsmasq.git",
-                 "component": "hwtest",
+                 "tags": ["hwtest"],
                 "patches": ["dnsmasq-buildsys.patch"]},
 
                 {"src": "ipxe:ipxe.git",
-                 "component": "hwtest",
+                 "tags": ["hwtest"],
                 "patches": ["ipxe-buildsys.patch",
                              "ipxe-disable-isos.patch"]},
 
                {"src": "savannah:grub.git",
-                 "component": "hwtest",
+                 "tags": ["hwtest"],
                  "cflags": { "x86_64": "" },
                  "config-opts": ["--with-platform=pc",
                                  "--disable-grub-mkfont",
@@ -887,7 +887,7 @@
                                  "--enable-libzfs=no"]},
 
                {"src": "fujita:tgt",
-                 "component": "hwtest",
+                 "tags": ["hwtest"],
                 "patches": ["tgt-cflags.patch",
                              "tgt-buildsys.patch",
                              "tgt-disable-man.patch"],
@@ -897,7 +897,7 @@
                  "name": "lighttpd",
                  "rm-configure": true,
                  "checksum": "113e9b72ccbd1da5deb0774bf93cf0ca15dc82aad2da0f04e5ab27d37d3f30a3",
-                 "component": "hwtest",
+                 "tags": ["hwtest"],
                 "config-opts": ["--without-pcre"]},
 
                 {"src": "ibus:ibus.git",
@@ -1088,14 +1088,14 @@
 
                {"src": "gnome:glade",
                 "config-opts": ["--disable-man-pages"],
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "gnome:jhbuild",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "git:git://git.fishsoup.net/git-bz",
                 "config-opts": ["--disable-documentation"],
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "gnome:nautilus"},
 
@@ -1144,7 +1144,7 @@
                {"src": "gnome:gdl"},
 
                {"src": "gnome:anjuta",
-                "component": "devel"},
+                "tags": ["devel"]},
 
                {"src": "gnome:gnome-font-viewer"},
 
@@ -1204,7 +1204,7 @@
                {"src": "gnome:file-roller"},
 
                {"src": "gnome:gnome-desktop-testing",
-                "component": "testing"},
+                "tags": ["testing"]},
 
                {"src": "gnome:gnome-tweak-tool"},
 
@@ -1227,6 +1227,32 @@
 
        ],
 
+    "targets": [
+                {"name": "runtime",
+                 "tags": ["runtime"],
+                 "trees": ["/runtime"],
+                 "base": "runtime"
+                },
+
+                {"name": "hwtest",
+                 "tags": ["hwtest"],
+                 "trees": ["/runtime"],
+                 "base": "runtime"
+                },
+
+                {"name": "devel-debug",
+                 "tags": ["runtime", "devel"],
+                 "trees": ["/runtime", "/devel", "/tests", "/doc", "/debug"],
+                 "base": "devel"
+                },
+
+                {"name": "testing",
+                 "tags": ["testing"],
+                 "trees": ["/runtime"],
+                 "base": "devel"
+                }
+    ],
+
     "installed-tests-components": [],
 
     "installed-tests-xfail": [],
diff --git a/src/js/tasks/task-build.js b/src/js/tasks/task-build.js
index 4d0df93..32539d4 100644
--- a/src/js/tasks/task-build.js
+++ b/src/js/tasks/task-build.js
@@ -1321,6 +1321,7 @@ const TaskBuild = new Lang.Class({
 
        this.patchdir = this.workdir.get_child('patches');
 
+        let targets = this._snapshot.data['targets'];
         let components = this._snapshot.data['components'];
 
        // Pick up overrides from $workdir/overrides/$name
@@ -1347,33 +1348,31 @@ const TaskBuild = new Lang.Class({
 
         let componentToArches = {};
 
-        let runtimeComponents = [];
-        let hwtestComponents = [];
-        let develComponents = [];
-        let testingComponents = [];
+        let componentLists = {};
+
+        for (let j = 0; j < targets.length; j++) {
+            let target = targets[j];
+            let targetName = target['name'];
+            componentLists[targetName] = [];
+        }
 
         for (let i = 0; i < components.length; i++) {
            let component = components[i];
             let name = component['name']
 
-            let resolved = component['component'] || 'runtime';
-            let isMinimal = resolved == 'minimal';
-            let isHWTest = resolved == 'minimal' || resolved == 'hwtest';
-            let isHWTestOnly = resolved == 'hwtest';
-            let isRuntime = resolved == 'minimal' || resolved == 'runtime';
-            let isTesting = resolved == 'testing';
-
-            if (isRuntime) {
-                runtimeComponents.push(component);
-           } else if (isTesting) {
-               testingComponents.push(component);
-           }
-            if (isHWTest) {
-                hwtestComponents.push(component);
-            }
+            let tags = component['tags'] || ['runtime'];
+
+            for (let j = 0; j < targets.length; j++) {
+                let target = targets[j];
+                let targetName = target['name'];
+                let targetTags = target['tags'];
 
-            if (!isHWTestOnly) {
-               develComponents.push(component);
+                for (let k = 0; k < targetTags.length; k++) {
+                    if (tags.indexOf(targetTags[k]) != -1) {
+                        componentLists[targetName].push(component);
+                        break;
+                    }
+                }
             }
 
            let isNoarch = component['noarch'] || false;
@@ -1435,36 +1434,24 @@ const TaskBuild = new Lang.Class({
        }
 
         let targetsList = [];
-       let componentTypes = ['runtime', 'hwtest', 'devel-debug'];
-        for (let i = 0; i < componentTypes.length; i++) {
-           let targetComponentType = componentTypes[i];
+        for (let i = 0; i < targets.length; i++) {
+            let targetInfo = targets[i];
+            let targetName = targetInfo['name'];
             for (let i = 0; i < architectures.length; i++) {
                let architecture = architectures[i];
                 let target = {};
                 targetsList.push(target);
-                target['name'] = 'buildmaster/' + architecture + '-' + targetComponentType;
+                target['name'] = 'buildmaster/' + architecture + '-' + targetName;
 
                 let baseRuntimeRef = baseName + '/' + architecture + '-runtime';
                 let buildrootRef = baseName + '/' + architecture + '-devel';
-               let baseRef;
-                if (targetComponentType == 'runtime' || targetComponentType == 'hwtest') {
-                    baseRef = baseRuntimeRef;
-                } else {
-                    baseRef = buildrootRef;
-               }
+                let baseRef = baseName + '/' + architecture + '-' + targetInfo['base'];
+
                 target['base'] = {'name': baseRef,
                                   'runtime': baseRuntimeRef,
                                   'devel': buildrootRef};
 
-               let targetComponents;
-                if (targetComponentType == 'runtime') {
-                    targetComponents = runtimeComponents;
-                } else if (targetComponentType == 'hwtest') {
-                    targetComponents = hwtestComponents;
-                } else {
-                    targetComponents = runtimeComponents;
-                    targetComponents = develComponents;
-               }
+                let targetComponents = componentLists[targetName];
 
                 let contents = [];
                 for (let i = 0; i < targetComponents.length; i++) {
@@ -1478,11 +1465,7 @@ const TaskBuild = new Lang.Class({
                    }
                     let binaryName = component['name'] + '/' + architecture;
                     let componentRef = {'name': binaryName};
-                    if (targetComponentType == 'runtime' || targetComponentType == 'hwtest') {
-                        componentRef['trees'] = ['/runtime'];
-                   } else if (targetComponentType == 'devel-debug') {
-                        componentRef['trees'] = ['/runtime', '/devel', '/tests', '/doc', '/debug'];
-                   }
+                    componentRef['trees'] = targetInfo['trees'];
                     contents.push(componentRef);
                }
                 target['contents'] = contents;
@@ -1535,13 +1518,14 @@ const TaskBuild = new Lang.Class({
 
        // Now loop over the other targets per architecture, reusing
        // the initramfs cached from -devel generation.
-       for (let i = 0; i < componentTypes.length; i++) {
-           let target = componentTypes[i];
-           if (target == 'devel-debug')
+        for (let i = 0; i < targets.length; i++) {
+            let targetInfo = targets[i];
+            let targetName = targetInfo['name'];
+           if (targetName == 'devel-debug' || targetName == 'testing')
                continue;
             for (let j = 0; j < architectures.length; j++) {
                let architecture = architectures[j];
-               let runtimeTargetName = 'buildmaster/' + architecture + '-' + target;
+               let runtimeTargetName = 'buildmaster/' + architecture + '-' + targetName;
                let runtimeTarget = this._findTargetInList(runtimeTargetName, targetsList);
 
                let composeRootdir;
@@ -1551,7 +1535,7 @@ const TaskBuild = new Lang.Class({
                let kernelInitramfsData = archInitramfsImages[architecture];
                this._installKernelAndInitramfs(kernelInitramfsData, composeRootdir, cancellable);
 
-                if (target == 'hwtest')
+                if (targetName == 'hwtest')
                     this._setHWTestTarget(composeRootdir, cancellable);
 
                this._cleanupGarbage(composeRootdir, cancellable);
@@ -1569,8 +1553,8 @@ const TaskBuild = new Lang.Class({
         for (let i = 0; i < architectures.length; i++) {
            installedTestContents[architectures[i]] = [];
        }
-       for (let i = 0; i < testingComponents.length; i++) {
-           let component = testingComponents[i];
+       for (let i = 0; i < componentLists['testing'].length; i++) {
+           let component = componentLists['testing'][i];
            let name = component['name'];
             for (let j = 0; j < architectures.length; j++) {
                let architecture = architectures[j];
@@ -1581,13 +1565,13 @@ const TaskBuild = new Lang.Class({
                installedTestContents[architecture].push([rev, '/runtime']);
            }
        }
-       for (let i = 0; i < runtimeComponents.length; i++) {
-           let component = runtimeComponents[i];
+       for (let i = 0; i < componentLists['runtime'].length; i++) {
+           let component = componentLists['runtime'][i];
            for (let j = 0; j < architectures.length; j++) {
                let architecture = architectures[j];
                let archname = component['name'] + '/' + architecture;
                let rev = componentBuildRevs[archname];
-               installedTestContents[architecture].push([rev, '/tests'])
+               installedTestContents[architecture].push([rev, '/tests']);
            }
        }
         for (let i = 0; i < installedTestComponentNames.length; i++) {


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]