[gnome-ostree] ostbuild: Don't require gnome-ostree in bootstrap

[Colin Walters <walters src gnome org>]

commit a02bef8dbb6be7e11b131f54c0e22932fb122a2e
Author: Colin Walters <walters verbum org>
Date:   Tue Sep 11 19:16:29 2012 -0400

    ostbuild: Don't require gnome-ostree in bootstrap
    
    Rather than calling ostbuild compile-one inside the chroot, copy a
    build script from outside to the root, then run it chrooted.
    
    This required separating out ostree-build-compile-one as a standalone
    script.

 Makefile-ostbuild.am                               |   10 +-
 src/ostbuild/ostbuild.in                           |    1 +
 src/ostbuild/ostree-build-compile-one              |  336 ++++++++++++++++++++
 .../pyostbuild/builtin_chroot_compile_one.py       |    8 +-
 src/ostbuild/pyostbuild/builtin_compile_one.py     |  298 -----------------
 src/ostbuild/pyostbuild/main.py                    |    1 -
 6 files changed, 351 insertions(+), 303 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 0199e5a..d8e60fd 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -16,19 +16,25 @@
 # Boston, MA 02111-1307, USA.
 
 ostbuild: src/ostbuild/ostbuild.in Makefile
-	sed -e s,@libdir\@,$(libdir), -e s,@datarootdir\@,$(datarootdir), -e s,@PYTHON\@,$(PYTHON), $< > $  tmp && mv $  tmp $@
+	sed -e s,@libdir\@,$(libdir), \
+	    -e s,@datarootdir\@,$(datarootdir), \
+	    -e s,@pkgdatadir\@,$(pkgdatadir), \
+	    -e s,@PYTHON\@,$(PYTHON), \
+	    $< > $  tmp && mv $  tmp $@
 EXTRA_DIST += ostbuild/ostbuild.in
 
 if BUILDSYSTEM
 bin_SCRIPTS += ostbuild 
 
+compileone_DATA = src/ostbuild/ostree-build-compile-one
+compileonedir = $(libdir)/ostbuild
+
 pyostbuilddir=$(libdir)/ostbuild/pyostbuild
 pyostbuild_PYTHON =					\
 	src/ostbuild/pyostbuild/buildutil.py		\
 	src/ostbuild/pyostbuild/builtin_build.py	\
 	src/ostbuild/pyostbuild/builtin_checkout.py	\
 	src/ostbuild/pyostbuild/builtin_chroot_compile_one.py	\
-	src/ostbuild/pyostbuild/builtin_compile_one.py	\
 	src/ostbuild/pyostbuild/builtin_deploy_qemu.py	\
 	src/ostbuild/pyostbuild/builtin_deploy_root.py	\
 	src/ostbuild/pyostbuild/builtin_run_qemu.py	\
diff --git a/src/ostbuild/ostbuild.in b/src/ostbuild/ostbuild.in
index a200235..81c71a5 100755
--- a/src/ostbuild/ostbuild.in
+++ b/src/ostbuild/ostbuild.in
@@ -22,6 +22,7 @@ import sys
 import __builtin__
 
 __builtin__.__dict__['DATADIR'] = '@datarootdir@'
+__builtin__.__dict__['LIBDIR'] = '@libdir@'
 # This is a private directory, we don't want to pollute the global
 # namespace.
 path = os.path.join('@libdir@', 'ostbuild')
diff --git a/src/ostbuild/ostree-build-compile-one b/src/ostbuild/ostree-build-compile-one
new file mode 100755
index 0000000..271444c
--- /dev/null
+++ b/src/ostbuild/ostree-build-compile-one
@@ -0,0 +1,336 @@
+#!/usr/bin/env python
+# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+from StringIO import StringIO
+import json
+from multiprocessing import cpu_count
+import select,time
+
+def log(x):
+    sys.stdout.write('ob: ' + x)
+    sys.stdout.write('\n')
+    sys.stdout.flush()
+
+def fatal(x):
+    log(x)
+    sys.exit(1)
+
+def _get_env_for_cwd(cwd=None, env=None):
+    # This dance is necessary because we want to keep the PWD
+    # environment variable up to date.  Not doing so is a recipie
+    # for triggering edge conditions in pwd lookup.
+    if (cwd is not None) and (env is None or ('PWD' in env)):
+        if env is None:
+            env_copy = os.environ.copy()
+        else:
+            env_copy = env.copy()
+        if ('PWD' in env_copy) and (not cwd.startswith('/')):
+            env_copy['PWD'] = os.path.join(env_copy['PWD'], cwd)
+        else:
+            env_copy['PWD'] = cwd
+    else:
+        env_copy = env
+    return env_copy
+
+def run_sync(args, cwd=None, env=None):
+    log("running: %s" % (subprocess.list2cmdline(args),))
+
+    env_copy = _get_env_for_cwd(cwd, env)
+
+    stdin_target = open('/dev/null', 'r')
+    stdout_target = sys.stdout
+    stderr_target = sys.stderr
+
+    proc = subprocess.Popen(args, stdin=stdin_target, stdout=stdout_target, stderr=stderr_target,
+                            close_fds=True, cwd=cwd, env=env_copy)
+    stdin_target.close()
+    returncode = proc.wait()
+    if returncode != 0:
+        logfn = fatal
+    else:
+        logfn = None
+    if logfn is not None:
+        logfn("pid %d exited with code %d" % (proc.pid, returncode))
+    return returncode
+
+PREFIX = '/usr'
+
+# Applied to filenames only
+_IGNORE_FILENAME_REGEXPS = map(re.compile,
+                               [r'.*\.py[co]$'])
+
+_DOC_DIRS = ['usr/share/doc',
+             'usr/share/gtk-doc',
+             'usr/share/man',
+             'usr/share/info']
+
+_DEVEL_DIRS = ['usr/include',
+               'usr/share/aclocal',
+               'usr/share/pkgconfig',
+               'usr/lib/pkgconfig']
+
+tempfiles = []
+
+def _has_buildapi_configure_variable(name):
+    var = '#buildapi-variable-%s' % (name, )
+    for line in open('configure'):
+        if line.find(var) >= 0:
+            return True
+    return False
+
+def main(args):
+    default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )]
+
+    starttime = time.time()
+    
+    uname=os.uname()
+    kernel=uname[0].lower()
+    machine=uname[4]
+    build_target='%s-%s' % (machine, kernel)
+
+    configargs = ['--build=' + build_target,
+                  '--prefix=' + PREFIX,
+                  '--libdir=' + os.path.join(PREFIX, 'lib'),
+                  '--sysconfdir=/etc',
+                  '--localstatedir=/var',
+                  '--bindir=' + os.path.join(PREFIX, 'bin'),
+                  '--sbindir=' + os.path.join(PREFIX, 'sbin'),
+                  '--datadir=' + os.path.join(PREFIX, 'share'),
+                  '--includedir=' + os.path.join(PREFIX, 'include'),
+                  '--libexecdir=' + os.path.join(PREFIX, 'libexec'),
+                  '--mandir=' + os.path.join(PREFIX, 'share', 'man'),
+                  '--infodir=' + os.path.join(PREFIX, 'share', 'info')]
+    makeargs = ['make']
+
+    ostbuild_resultdir='_ostbuild-results'
+    ostbuild_meta_path='_ostbuild-meta.json'
+
+    chdir = None
+    opt_install = False
+
+    for arg in args:
+        if arg.startswith('--ostbuild-resultdir='):
+            ostbuild_resultdir=arg[len('--ostbuild-resultdir='):]
+        elif arg.startswith('--ostbuild-meta='):
+            ostbuild_meta_path=arg[len('--ostbuild-meta='):]
+        elif arg.startswith('--chdir='):
+            os.chdir(arg[len('--chdir='):])
+        else:
+            makeargs.append(arg)
+        
+    f = open(ostbuild_meta_path)
+    metadata = json.load(f)
+    f.close()
+
+    configargs.extend(metadata.get('config-opts', []))
+
+    if metadata.get('rm-configure', False):
+        configure_path = 'configure'
+        if os.path.exists(configure_path):
+            os.unlink(configure_path)
+
+    autogen_script = None
+    if not os.path.exists('configure'):
+        log("No 'configure' script found, looking for autogen/bootstrap")
+        for name in ['autogen', 'autogen.sh', 'bootstrap']:
+            if os.path.exists(name):
+                log("Using bootstrap script '%s'" % (name, ))
+                autogen_script = name
+        if autogen_script is None:
+            fatal("No configure or autogen script detected; unknown buildsystem")
+
+    if autogen_script is not None:
+        env = dict(os.environ)
+        env['NOCONFIGURE'] = '1'
+        run_sync(['./' + autogen_script], env=env)
+    else:
+        log("Using existing 'configure' script")
+            
+    builddir = '_build'
+
+    use_builddir = True
+    doesnot_support_builddir = _has_buildapi_configure_variable('no-builddir')
+    if doesnot_support_builddir:
+        log("Found no-builddir Build API variable; copying source tree to _build")
+        if os.path.isdir('_build'):
+            shutil.rmtree('_build')
+        shutil.copytree('.', '_build', symlinks=True,
+                        ignore=shutil.ignore_patterns('_build'))
+        use_builddir = False
+    
+    if use_builddir:
+        log("Using build directory %r" % (builddir, ))
+        if not os.path.isdir(builddir):
+            os.mkdir(builddir)
+    
+    if use_builddir:
+        args = ['../configure']
+    else:
+        args = ['./configure']
+    args.extend(configargs)
+    run_sync(args, cwd=builddir)
+
+    makefile_path = None
+    for name in ['Makefile', 'makefile', 'GNUmakefile']:
+        makefile_path = os.path.join(builddir, name)
+        if os.path.exists(makefile_path):
+            break
+    if makefile_path is None:
+        fatal("No Makefile found")
+
+    args = list(makeargs)
+    user_specified_jobs = False
+    for arg in args:
+        if arg == '-j':
+            user_specified_jobs = True
+    
+    if not user_specified_jobs:
+        has_notparallel = False
+        for line in open(makefile_path):
+            if line.startswith('.NOTPARALLEL'):
+                has_notparallel = True
+                log("Found .NOTPARALLEL")
+
+        if not has_notparallel:
+            log("Didn't find NOTPARALLEL, using parallel make by default")
+            args.extend(default_buildapi_jobs)
+
+    run_sync(args, cwd=builddir)
+
+    tempdir = tempfile.mkdtemp(prefix='ostbuild-destdir-%s' % (metadata['name'].replace('/', '_'), ))
+    tempfiles.append(tempdir)
+    args = ['make', 'install', 'DESTDIR=' + tempdir]
+    run_sync(args, cwd=builddir)
+    
+    runtime_path = os.path.join(ostbuild_resultdir, 'runtime')
+    devel_path = os.path.join(ostbuild_resultdir, 'devel')
+    doc_path = os.path.join(ostbuild_resultdir, 'doc')
+    for artifact_type in ['runtime', 'devel', 'doc']:
+        resultdir = os.path.join(ostbuild_resultdir, artifact_type)
+        if os.path.isdir(resultdir):
+            shutil.rmtree(resultdir)
+        os.makedirs(resultdir)
+
+    # Remove /var from the install - components are required to
+    # auto-create these directories on demand.
+    varpath = os.path.join(tempdir, 'var')
+    if os.path.isdir(varpath):
+        shutil.rmtree(varpath)
+
+    # Move symbolic links for shared libraries as well
+    # as static libraries.  And delete all .la files.
+    for libdirname in ['lib', 'usr/lib']:
+        path = os.path.join(tempdir, libdirname)
+        if not os.path.isdir(path):
+            continue
+        for filename in os.listdir(path):
+            subpath = os.path.join(path, filename)
+            if filename.endswith('.la'):
+                os.unlink(subpath)
+                continue
+            if not ((filename.endswith('.so')
+                     and os.path.islink(filename))
+                    or filename.endswith('.a')):
+                    continue
+            dest = os.path.join(devel_path, libdirname, filename)
+            _install_and_unlink(subpath, dest)
+
+    for dirname in _DEVEL_DIRS:
+        dirpath = os.path.join(tempdir, dirname)
+        if os.path.isdir(dirpath):
+            dest = os.path.join(devel_path, dirname)
+            _install_and_unlink(dirpath, dest)
+
+    for dirname in _DOC_DIRS:
+        dirpath = os.path.join(tempdir, dirname)
+        if os.path.isdir(dirpath):
+            dest = os.path.join(doc_path, dirname)
+            _install_and_unlink(dirpath, dest)
+    
+    for filename in os.listdir(tempdir):
+        src_path = os.path.join(tempdir, filename)
+        dest_path = os.path.join(runtime_path, filename)
+        _install_and_unlink(src_path, dest_path)
+
+    for tmpname in tempfiles:
+        assert os.path.isabs(tmpname)
+        if os.path.isdir(tmpname):
+            shutil.rmtree(tmpname)
+        else:
+            try:
+                os.unlink(tmpname)
+            except OSError, e:
+                pass
+
+    endtime = time.time()
+
+    log("Compilation succeeded; %d seconds elapsed" % (int(endtime - starttime),))
+    log("Results placed in %s" % (ostbuild_resultdir, ))
+
+def _install_and_unlink(src, dest):
+    statsrc = os.lstat(src)
+    dirname = os.path.dirname(dest)
+    if not os.path.isdir(dirname):
+        os.makedirs(dirname)
+
+    # Ensure that all installed files are at least rw-rw-r--;
+    # we don't support private/hidden files.
+    # Directories also need u+x, i.e. they're rwxrw-r--
+    if not stat.S_ISLNK(statsrc.st_mode):
+        minimal_mode = (stat.S_IRUSR | stat.S_IWUSR |
+                        stat.S_IRGRP | stat.S_IWGRP |
+                        stat.S_IROTH)
+        if stat.S_ISDIR(statsrc.st_mode):
+            minimal_mode |= stat.S_IXUSR
+        os.chmod(src, statsrc.st_mode | minimal_mode)
+
+    if stat.S_ISDIR(statsrc.st_mode):
+        if not os.path.isdir(dest):
+            os.mkdir(dest)
+        for filename in os.listdir(src):
+            src_child = os.path.join(src, filename)
+            dest_child = os.path.join(dest, filename)
+
+            _install_and_unlink(src_child, dest_child)
+        os.rmdir(src)
+    else:
+        basename = os.path.basename(src)
+        ignored = False
+        for r in _IGNORE_FILENAME_REGEXPS:
+            if r.match(basename):
+                ignored = True
+                break
+        if ignored:
+            log("Not installing %s" % (src, ))
+            os.unlink(src)
+            return
+        try:
+            os.rename(src, dest)
+        except OSError, e:
+            if stat.S_ISLNK(statsrc.st_mode):
+                linkto = os.readlink(src)
+                os.symlink(linkto, dest)
+            else:
+                shutil.copy2(src, dest)
+            os.unlink(src)
+
+main(sys.argv)
diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
index 8e48655..390936b 100755
--- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
@@ -191,6 +191,11 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         rootdir = self._compose_buildroot(component_name, args.arch)
 
         log("Checked out buildroot: %s" % (rootdir, ))
+
+        src_compile_one_path = os.path.join(LIBDIR, 'ostbuild', 'ostree-build-compile-one')
+        dest_compile_one_path = os.path.join(rootdir, 'ostree-build-compile-one')
+        shutil.copy(src_compile_one_path, dest_compile_one_path)
+        os.chmod(dest_compile_one_path, 0755)
         
         sourcedir=os.path.join(rootdir, 'ostbuild', 'source', component_name)
         fileutil.ensure_dir(sourcedir)
@@ -213,8 +218,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         if args.debug_shell:
             child_args.extend([rootdir, '/bin/sh'])
         else:
-            child_args.extend([rootdir, '/usr/bin/ostbuild',
-                               'compile-one',
+            child_args.extend([rootdir, '/ostree-build-compile-one',
                                '--ostbuild-resultdir=/ostbuild/results',
                                '--ostbuild-meta=_ostbuild-meta.json'])
         env_copy = dict(buildutil.BUILD_ENV)
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index c71ddef..3ded483 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -25,7 +25,6 @@ from . import builtins
 from . import builtin_build
 from . import builtin_checkout
 from . import builtin_chroot_compile_one
-from . import builtin_compile_one
 from . import builtin_deploy_root
 from . import builtin_deploy_qemu
 from . import builtin_git_mirror