[gnome-ostree/wip/integrated-build] Automatically build Yocto base

[Colin Walters <walters src gnome org>]

commit 002a97d37809ade276e427be3c400e4a2796bdf5
Author: Colin Walters <walters verbum org>
Date:   Wed Sep 19 17:53:57 2012 -0400

    Automatically build Yocto base
    
    Rather than requiring users to set up things manually, do an automated
    Yocto build.
    
    Thus the gnome-ostree module is now responsible for building
    everything.
    
    Also, when we use git submodules for everything, with this we'll be
    able to speak about one gnome-ostree revision which describes an
    entire source tree.

 Makefile-ostbuild.am                       |    7 ++-
 gnomeos-3.6.json                           |    3 +-
 src/ostbuild/ostree-build-yocto            |   78 ++++++++++++++++++++++++++++
 src/ostbuild/pyostbuild/builtin_build.py   |   35 ++++++++++++
 src/ostbuild/pyostbuild/builtin_resolve.py |   15 +++++
 src/ostbuild/pyostbuild/builtins.py        |    8 +--
 6 files changed, 137 insertions(+), 9 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index d8e60fd..b8530b3 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -26,8 +26,11 @@ EXTRA_DIST += ostbuild/ostbuild.in
 if BUILDSYSTEM
 bin_SCRIPTS += ostbuild 
 
-compileone_DATA = src/ostbuild/ostree-build-compile-one
-compileonedir = $(libdir)/ostbuild
+utils_SCRIPTS = \
+	src/ostbuild/ostree-build-compile-one \
+	src/ostbuild/ostree-build-yocto \
+	$(NULL)
+utilsdir = $(libdir)/ostbuild
 
 pyostbuilddir=$(libdir)/ostbuild/pyostbuild
 pyostbuild_PYTHON =					\
diff --git a/gnomeos-3.6.json b/gnomeos-3.6.json
index 4b213e3..52bc280 100644
--- a/gnomeos-3.6.json
+++ b/gnomeos-3.6.json
@@ -5,7 +5,8 @@
   "architectures": ["i686"],
   "base": {
             "name": "yocto",
-	    "src": "cgwalters:poky"
+	    "src": "cgwalters:poky",
+            "branch": "gnomeos-3.6"
           },
 
   "config-opts": ["--disable-static", "--disable-silent-rules"],
diff --git a/src/ostbuild/ostree-build-yocto b/src/ostbuild/ostree-build-yocto
new file mode 100644
index 0000000..73ff288
--- /dev/null
+++ b/src/ostbuild/ostree-build-yocto
@@ -0,0 +1,78 @@
+#!/bin/bash
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+# Copyright (C) 2012 Adrian Perez <aperez igalia com>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+umask 022
+
+die () {
+    echo "$*" 1>&2
+    exit 1
+}
+
+usage () {
+    echo "$0: SRCDIR BUILDDIR"
+}
+
+srcdir=$1
+test -n "${srcdir}" || (usage; exit 1)
+builddir=$2
+test -n "${builddir}" || (usage; exit 1)
+
+src_gitrev=$(cd ${srcdir} && git rev-parse HEAD)
+built_gitrev_path=${builddir}/built-revision
+if test -f ${built_gitrev_path}; then
+    built_gitrev=$(cat ${built_gitrev_path})
+else
+    built_gitrev=
+fi
+
+if test x${src_gitrev} = x${built_gitrev}; then
+    echo "Already built ${src_gitrev}"
+    exit 0
+else
+    if test -n "${built_gitrev}"; then
+	echo "Yocto build ${src_gitrev} differs from previous ${built_gitrev}"
+    fi
+fi
+
+. "${srcdir}/oe-init-build-env" "${builddir}"
+
+if ! grep -q meta-gnomeos ${builddir}/conf/bblayers.conf; then
+    echo "BBLAYERS += \"${srcdir}/meta-gnomeos\"" >> ${builddir}/conf/bblayers.conf
+fi
+
+if ! grep -q "DISTRO.*gnomeosdistro" ${builddir}/conf/local.conf; then
+    numcpus=$(( $(getconf _NPROCESSORS_ONLN) / 2 + 1 ))
+    cat >> ${builddir}/conf/local.conf <<EOF
+PARALLEL_MAKE = "-j ${numcpus}"
+BB_NUMBER_THREADS = "${numcpus}"
+DISTRO = "gnomeosdistro"
+EOF
+fi
+
+if ! test -L ${builddir}/repo; then
+    ln -s tmp-eglibc/deploy/images/repo ${builddir}/repo
+fi
+
+echo "Building ${src_gitrev}"
+bitbake gnomeos-contents-{runtime,devel}
+
+echo ${src_gitrev} > ${built_gitrev_path}.tmp
+mv ${built_gitrev_path}.tmp ${built_gitrev_path}
+exit 0
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index 8fced39..9225682 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -348,6 +348,38 @@ class OstbuildBuild(builtins.Builtin):
         f.close()
         os.rename(temppath, self.args.status_json_path)
 
+    def _initialize_repo(self):
+        """Set up an OSTree repository in $workdir/repo.
+This is used to store the build results of both the Yocto component
+and the manifest input."""
+        fileutil.ensure_dir(self.repo)
+        if not os.path.isdir(os.path.join(self.repo, 'objects')):
+            run_sync(['ostree', '--repo=' + self.repo, 'init', '--archive'])
+
+    def _build_base(self):
+        """Build the Yocto base system."""
+        basemeta = self.snapshot['base']
+        checkoutdir = os.path.join(self.workdir, 'checkouts', basemeta['name'])
+        fileutil.ensure_parent_dir(checkoutdir)
+
+        (keytype, uri) = buildutil.parse_src_key(basemeta['src'])
+        vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
+                             basemeta['revision'],
+                             overwrite=False)
+
+        builddir = os.path.join(self.workdir, 'build-' + basemeta['name'])
+        image_deploy_dir = os.path.join(builddir, 'tmp-eglibc', 'deploy', 'images')
+        repo_link = os.path.join(image_deploy_dir, 'repo')
+        if not os.path.islink(repo_link):
+            os.symlink(self.repo, repo_link)
+
+        cmd = ['linux-user-chroot', '--unshare-pid', '/',
+               os.path.join(LIBDIR, 'ostbuild', 'ostree-build-yocto'),
+               checkoutdir, builddir]
+        # We specifically want to kill off any environment variables jhbuild
+        # may have set.
+        run_sync(cmd, env=buildutil.BUILD_ENV)
+        
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--prefix')
@@ -384,6 +416,9 @@ class OstbuildBuild(builtins.Builtin):
 
         self.cached_patchdir_revision = None
 
+        self._initialize_repo()
+        self._build_base()
+
         components = self.snapshot['components']
 
         prefix = self.snapshot['prefix']
diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py
index e415404..27eddff 100755
--- a/src/ostbuild/pyostbuild/builtin_resolve.py
+++ b/src/ostbuild/pyostbuild/builtin_resolve.py
@@ -46,6 +46,8 @@ class OstbuildResolve(builtins.Builtin):
                             help="Path to manifest file")
         parser.add_argument('--fetch-patches', action='store_true',
                             help="Git fetch the patches")
+        parser.add_argument('--fetch-base', action='store_true',
+                            help="Git fetch the base system")
         parser.add_argument('--fetch', action='store_true',
                             help="Also perform a git fetch")
         parser.add_argument('--fetch-keep-going', action='store_true',
@@ -77,6 +79,19 @@ class OstbuildResolve(builtins.Builtin):
                 fatal("Duplicate component name '%s'" % (name, ))
             unique_component_names.add(name)
 
+        if args.fetch and len(args.components) == 0:
+            args.fetch_patches = args.fetch_base = True
+
+        base_meta = buildutil.resolve_component_meta(self.snapshot, self.snapshot['base'])
+        self.snapshot['base'] = base_meta
+        (keytype, uri) = vcs.parse_src_key(base_meta['src'])
+        mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, base_meta['branch'])
+        if args.fetch_base:
+            run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False)
+
+        base_revision = buildutil.get_git_version_describe(mirrordir, base_meta['branch'])
+        base_meta['revision'] = base_revision
+
         global_patches_meta = buildutil.resolve_component_meta(self.snapshot, self.snapshot['patches'])
         self.snapshot['patches'] = global_patches_meta
         (keytype, uri) = vcs.parse_src_key(global_patches_meta['src'])
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index 33bb062..a4d5cd4 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -168,15 +168,11 @@ class Builtin(object):
     def init_repo(self):
         if self.repo is not None:
             return self.repo
-        repo = ostbuildrc.get_key('repo', default=None)
+        repo = ostbuildrc.get_key('override_repo', default=None)
         if repo is not None:
             self.repo = os.path.expanduser(repo)
         else:
-            shadow_path = os.path.join(self.workdir, 'shadow-repo')
-            if os.path.isdir(shadow_path):
-                self.repo = shadow_path
-            else:
-                fatal("No repository configured, and shadow-repo not found.  Use \"ostbuild init\" to make one")
+            self.repo = os.path.join(self.workdir, 'repo')
 
     def parse_prefix(self, prefix):
         if prefix is not None: