[ostree] osbuild: Add some prototype python scripts

[Colin Walters <walters src gnome org>]

commit 6e0564787537f0f4fdc2684b9d4ed9c74e422546
Author: Colin Walters <walters verbum org>
Date:   Thu Nov 10 22:12:26 2011 -0500

    osbuild: Add some prototype python scripts

 Makefile-osbuild.am                                |   22 +--
 osbuild/ostree-buildone                            |  280 ++++++++++++++++++++
 osbuild/ostree-buildone-make                       |  205 ++++++++++++++
 .../ostree-buildone-makeinstall-split-artifacts    |  104 ++++++++
 4 files changed, 592 insertions(+), 19 deletions(-)
---
diff --git a/Makefile-osbuild.am b/Makefile-osbuild.am
index 1ef7753..3f2e56b 100644
--- a/Makefile-osbuild.am
+++ b/Makefile-osbuild.am
@@ -1,5 +1,3 @@
-# Makefile for C source code
-#
 # Copyright (C) 2011 Colin Walters <walters verbum org>
 #
 # This library is free software; you can redistribute it and/or
@@ -17,21 +15,7 @@
 # Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 # Boston, MA 02111-1307, USA.
 
-bin_PROGRAMS += ostree-build
-
-osbuild_common_cflags = -I$(srcdir)/libotutil -I$(srcdir)/osbuild -DLOCALEDIR=\"$(datadir)/locale\" -DG_LOG_DOMAIN=\"osbuild\" $(GIO_UNIX_CFLAGS)
-osbuild_common_ldadd = libotutil.la $(GIO_UNIX_LIBS)
-
-ostree_build_SOURCES = osbuild/main.c \
-	osbuild/ob-builtins.h \
-	osbuild/ob-builtin-buildone.c \
-	$(NULL)
-ostree_build_CFLAGS = $(osbuild_common_cflags)
-ostree_build_LDADD = $(osbuild_common_ldadd)
-
-bin_PROGRAMS += osbuild-raw-makeinstall
-
-osbuild_raw_makeinstall_SOURCES = osbuild/osbuild-raw-makeinstall.c \
+bin_SCRIPTS += osbuild/ostree-buildone \
+	osbuild/ostree-buildone-make \
+	osbuild/ostree-buildone-makeinstall-split-artifacts \
 	$(NULL)
-osbuild_raw_makeinstall_CFLAGS = $(osbuild_common_cflags)
-osbuild_raw_makeinstall_LDADD = $(osbuild_common_ldadd)
diff --git a/osbuild/ostree-buildone b/osbuild/ostree-buildone
new file mode 100644
index 0000000..077be41
--- /dev/null
+++ b/osbuild/ostree-buildone
@@ -0,0 +1,280 @@
+#!/usr/bin/python
+#
+# ostree-buildone:
+# Copyright 2010, 2011 Colin Walters <walters verbum org>
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+
+# The build output is automatically logged to $TMPDIR/build-$(PWD).log.
+# For example, invoking metabuild in a directory named "foo" will log
+# to /tmp/build-foo.log
+#
+# You can pass arguments to metabuild; if they start with '--', they're
+# given to configure.  Otherwise, they're passed to make.
+#
+# $ metabuild --enable-libfoo  # passed to configure
+# $ metabuild -j 1             # passed to make
+
+import os,sys,subprocess,tempfile,re
+import select,time,stat,fcntl
+
+subprocess_nice_args = []
+
+# In the future we should test for this better; possibly implement a
+# custom fork handler
+try:
+    chrt_args = ['chrt', '--idle', '0']
+    proc = subprocess.Popen(chrt_args + ['true'])
+    if proc.wait() == 0:
+        subprocess_nice_args.extend(chrt_args)
+except OSError, e:
+    pass
+
+try:
+    ionice_args = ['ionice', '-c', '3', '-t']
+    proc = subprocess.Popen(ionice_args + ['true'])
+    if proc.wait() == 0:
+        subprocess_nice_args.extend(ionice_args)
+except OSError, e:
+    pass
+
+warning_re = re.compile(r'(: ((warning)|(error)|(fatal error)): )|(make(\[[0-9]+\])?: \*\*\*)')
+output_whitelist_re = re.compile(r'^(make(\[[0-9]+\])?: Entering directory)|(ostree-build )')
+
+_bold_sequence = None
+_normal_sequence = None
+if os.isatty(1):
+    _bold_sequence = subprocess.Popen(['tput', 'bold'], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w')).communicate()[0]
+    _normal_sequence = subprocess.Popen(['tput', 'sgr0'], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w')).communicate()[0]
+def _bold(text):
+    if _bold_sequence is not None:
+        return '%s%s%s' % (_bold_sequence, text, _normal_sequence)
+    else:
+        return text
+
+class Mainloop(object):
+    DEFAULT = None
+    def __init__(self):
+        self._running = True
+        self.poll = select.poll()
+        self._timeouts = []
+        self._pid_watches = {}
+        self._fd_callbacks = {}
+
+    @classmethod
+    def get(cls, context):
+        if context is None:
+            if cls.DEFAULT is None:
+                cls.DEFAULT = cls()
+            return cls.DEFAULT
+        raise NotImplementedError("Unknown context %r" % (context, ))
+
+    def watch_fd(self, fd, callback):
+        self.poll.register(fd)
+        self._fd_callbacks[fd] = callback
+
+    def unwatch_fd(self, fd):
+        self.poll.unregister(fd)
+        del self._fd_callbacks[fd]
+
+    def watch_pid(self, pid, callback):
+        self._pid_watches[pid] = callback
+
+    def timeout_add(self, ms, callback):
+        self._timeouts.append((ms, callback))
+
+    def quit(self):
+        self._running = False
+
+    def run_once(self):
+        min_timeout = None
+        if len(self._pid_watches) > 0:
+            min_timeout = 500
+        for (ms, callback) in self._timeouts:
+            if (min_timeout is None) or (ms < min_timeout):
+                min_timeout = ms
+        origtime = time.time() * 1000
+        fds = self.poll.poll(min_timeout)
+        for fd in fds:
+            self._fd_callbacks[fd]()
+        for pid in self._pid_watches:
+            (opid, status) = os.waitpid(pid, os.WNOHANG)
+            if opid != 0:
+                self._pid_watches[pid](opid, status)
+                del self._pid_watches[pid]
+        newtime = time.time() * 1000
+        diff = int(newtime - origtime)
+        if diff < 0: diff = 0
+        for i,(ms, callback) in enumerate(self._timeouts):
+            remaining_ms = ms - diff
+            if remaining_ms <= 0:
+                callback()
+            else:
+                self._timeouts[i] = (remaining_ms, callback)
+
+    def run(self):
+        while self._running:
+            self.run_once()
+
+class FileMonitor(object):
+    def __init__(self):
+        self._paths = {}
+        self._path_modtimes = {}
+        self._timeout = 1000
+        self._timeout_installed = False
+        self._loop = Mainloop.get(None)
+
+    def _stat(self, path):
+        try:
+            st = os.stat(path)
+            return st[stat.ST_MTIME]
+        except OSError, e:
+            return None
+
+    def add(self, path, callback):
+        if path not in self._paths:
+            self._paths[path] = []
+            self._path_modtimes[path] = self._stat(path)
+        self._paths[path].append(callback)
+        if not self._timeout_installed:
+            self._timeout_installed = True
+            self._loop.timeout_add(self._timeout, self._check_files)
+
+    def _check_files(self):
+        for (path,callbacks) in self._paths.iteritems():
+            mtime = self._stat(path)
+            orig_mtime = self._path_modtimes[path]
+            if (mtime is not None) and (orig_mtime is None or (mtime > orig_mtime)):
+                self._path_modtimes[path] = mtime
+                for cb in callbacks:
+                    cb()
+
+_filemon = FileMonitor()
+
+class OutputFilter(object):
+    def __init__(self, filename, output):
+        self.filename = filename
+        self.output = output
+
+        # inherit globals
+        self._warning_re = warning_re
+        self._nonfilter_re = output_whitelist_re
+
+        self._buf = ''
+        self._warning_count = 0
+        self._filtered_line_count = 0
+        _filemon.add(filename, self._on_changed)
+        self._fd = os.open(filename, os.O_RDONLY)
+        fcntl.fcntl(self._fd, fcntl.F_SETFL, os.O_NONBLOCK)
+
+    def _do_read(self):
+        while True:
+            buf = os.read(self._fd, 4096)
+            if buf == '':
+                break
+            self._buf += buf
+        self._flush()
+
+    def _write_last_log_lines(self):
+        _last_line_limit = 100
+        f = open(logfile_path)
+        lines = []
+        for line in f:
+            if line.startswith('ostree-build '):
+                continue
+            lines.append(line)
+            if len(lines) > _last_line_limit:
+                lines.pop(0)
+        f.close()
+        for line in lines:
+            self.output.write('| ')
+            self.output.write(line)
+
+    def _flush(self):
+        while True:
+            p = self._buf.find('\n')
+            if p < 0:
+                break
+            line = self._buf[0:p]
+            self._buf = self._buf[p+1:]
+            match = self._warning_re.search(line)
+            if match:
+                self._warning_count += 1
+                self.output.write(line + '\n')
+            else:    
+                match = self._nonfilter_re.search(line)
+                if match:
+                    self.output.write(line + '\n')
+                else:
+                    self._filtered_line_count += 1
+
+    def _on_changed(self):
+        self._do_read()
+
+    def start(self):
+        self._do_read()
+
+    def finish(self, successful):
+        self._do_read()
+        if not successful:
+            self._write_last_log_lines()
+            pass
+        self.output.write("ostree-build %s: %d warnings\n" % ('success' if successful else _bold('failed'),
+                                                            self._warning_count, ))
+        self.output.write("ostree-build: full log path: %s\n" % (logfile_path, ))
+        
+        if successful:
+            for f in os.listdir('_build'):
+                path = os.path.join('_build', f)
+                if f.startswith('artifact-'):
+                    self.output.write("ostree-build: created artifact: %s\n" % (f, ))
+        sys.exit(0 if successful else 1)
+
+def _on_makeinstall_exit(pid, estatus):
+    _output_filter.finish(estatus == 0)
+
+def _on_make_exit(pid, estatus):
+    if estatus == 0:
+        args = list(subprocess_nice_args)
+        args.append('ostree-buildone-makeinstall-split-artifacts')
+        _logfile_f.write("Running: %r\n" % (args, ))
+        _logfile_f.flush()
+        proc = subprocess.Popen(args, stdin=devnull, stdout=logfile_write_fd, stderr=logfile_write_fd)
+        _loop.watch_pid(proc.pid, _on_makeinstall_exit)
+    else:
+        _output_filter.finish(False)
+
+if __name__ == '__main__':
+    user_tmpdir = os.environ.get('XDG_RUNTIME_DIR')
+    if user_tmpdir is None:
+        user_tmpdir = os.path.join(os.environ.get('TMPDIR', '/tmp'), 'metabuild-%s' % (os.getuid(), ))
+    else:
+        user_tmpdir = os.path.join(user_tmpdir, 'ostree-build') 
+    if not os.path.isdir(user_tmpdir):
+        os.makedirs(user_tmpdir)
+    logfile_path = os.path.join(user_tmpdir, '%s.log' % (os.path.basename(os.getcwd()), ))
+    try:
+        os.unlink(logfile_path)
+    except OSError, e:
+        pass
+    logfile_write_fd = os.open(logfile_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL)
+    global _logfile_f
+    _logfile_f = os.fdopen(logfile_write_fd, "w")
+    sys.stdout.write('ostree-build: logging to %r\n' % (logfile_path, ))
+    sys.stdout.flush()
+
+    global _output_filter
+    _output_filter = OutputFilter(logfile_path, sys.stdout)
+    _output_filter.start()
+
+    args = list(subprocess_nice_args)
+    args.append('ostree-buildone-make')
+    args.extend(sys.argv[1:])
+    devnull=open('/dev/null')
+    _logfile_f.write("Running: %r\n" % (args, ))
+    _logfile_f.flush()
+    proc = subprocess.Popen(args, stdin=devnull, stdout=logfile_write_fd, stderr=logfile_write_fd)
+
+    global _loop
+    _loop = Mainloop.get(None)
+    _loop.watch_pid(proc.pid, _on_make_exit)
+    _loop.run()
diff --git a/osbuild/ostree-buildone-make b/osbuild/ostree-buildone-make
new file mode 100644
index 0000000..b6e89f5
--- /dev/null
+++ b/osbuild/ostree-buildone-make
@@ -0,0 +1,205 @@
+#!/usr/bin/python
+
+# ostree-buildone-raw: Generic build system wrapper
+# Copyright 2010, 2011 Colin Walters <walters verbum org>
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+
+# ostree-buildone-raw wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,subprocess,tempfile,re
+from multiprocessing import cpu_count
+import select,time
+
+root = None
+
+prefix = '/usr'
+
+# libdir detection
+if os.path.isdir('/lib64'):
+    libdir=os.path.join(prefix, 'lib64')
+else:
+    libdir=os.path.join(prefix, 'lib')
+
+default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )]
+configargs = ['--prefix=' + prefix,
+              '--libdir=' + libdir,
+              '--sysconfdir=/etc',
+              '--localstatedir=/var',
+              '--bindir=' + os.path.join(prefix, 'bin'),
+              '--sbindir=' + os.path.join(prefix, 'sbin'),
+              '--datadir=' + os.path.join(prefix, 'share'),
+              '--includedir=' + os.path.join(prefix, 'include'),
+              '--libexecdir=' + os.path.join(prefix, 'libexec'),
+              '--mandir=' + os.path.join(prefix, 'share', 'man'),
+              '--infodir=' + os.path.join(prefix, 'share', 'info')]
+makeargs = ['make']
+
+top_srcdir=os.getcwd()
+
+for arg in sys.argv[1:]:
+    if arg.startswith('--'):
+        configargs.append(arg)
+    else:
+        makeargs.append(arg)
+
+def log(msg):
+    fullmsg = 'ostree-buildone: ' + msg + '\n'
+    sys.stdout.write(fullmsg)
+    sys.stdout.flush()
+
+def fatal(msg):
+    log(msg)
+    sys.exit(1)
+
+def run_sync(args, env=None):
+    log("Running: %r" % (args, ))
+    f = open('/dev/null', 'r')
+    proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr,
+                            close_fds=True, env=env)
+    f.close()
+    returncode = proc.wait()
+    log("pid %d exited with code %d" % (proc.pid, returncode))
+    if returncode != 0:
+        sys.exit(1)
+
+class BuildSystemScanner(object):
+    @classmethod
+    def _find_file(cls, names):
+        for name in names:
+            if os.path.exists(name):
+                return name
+        return None
+
+    @classmethod
+    def get_configure_source_script(cls):
+        return cls._find_file(('./configure.ac', './configure.in'))
+
+    @classmethod
+    def get_configure_script(cls):
+        return cls._find_file(('./configure', ))
+
+    @classmethod
+    def get_bootstrap_script(cls):
+        return cls._find_file(('./autogen.sh', ))
+
+    @classmethod
+    def get_silent_rules(cls):
+        src = cls.get_configure_source_script()
+        if not src:
+            return False
+        f = open(src)
+        for line in f:
+            if line.find('AM_SILENT_RULES') >= 0:
+                f.close()
+                return True
+        f.close()
+        return False
+
+def _search_file(filename, pattern):
+    f = open(filename)
+    for line in f:
+        if line.startswith(pattern):
+            f.close()
+            return line
+    f.close()
+    return None
+
+def _find_buildapi_makevariable(name):
+    var = '.%s:' % (name, )
+    line = None
+    if os.path.exists('Makefile.in'):
+        line = _search_file('Makefile.in', var)
+    if not line and os.path.exists('Makefile'):
+        line = _search_file('Makefile', var)
+    return line is not None
+
+def phase_bootstrap():        
+    have_configure = BuildSystemScanner.get_configure_script() 
+    have_configure_source = BuildSystemScanner.get_configure_source_script()
+    if not (have_configure or have_configure_source):
+        fatal("No configure or bootstrap script detected; unknown buildsystem")
+        return
+
+    need_v1 = BuildSystemScanner.get_silent_rules()
+    if need_v1:
+        log("Detected AM_SILENT_RULES, adding --disable-silent-rules to configure")
+        configargs.append('--disable-silent-rules')
+
+    if have_configure:
+        phase_configure()
+    else:
+        bootstrap = BuildSystemScanner.get_bootstrap_script()
+        if bootstrap:
+            log("Detected bootstrap script: %s, using it" % (bootstrap, ))
+            args = [bootstrap]
+            args.extend(configargs)
+            # Add NOCONFIGURE; GNOME style scripts use this
+            env = dict(os.environ)
+            env['NOCONFIGURE'] = '1'
+            run_sync(args, env=env)
+        else:
+            log("No bootstrap script found; using generic autoreconf")
+            run_sync(['autoreconf', '-f', '-i'])
+        phase_configure()
+
+def phase_configure():
+    use_builddir = True
+    doesnot_support_builddir = _find_buildapi_makevariable('buildapi-no-builddir')
+    if doesnot_support_builddir:
+        log("Found .buildapi-no-builddir; copying source tree to _build")
+        shutil.rmtree('_build')
+        os.mkdir('_build')
+        shutil.copytree('.', '_build', symlinks=True,
+                        ignore=shutil.ignore_patterns('_build'))
+        use_builddir = False
+        builddir = '.'
+    else:
+        builddir = '_build'
+
+    if not use_builddir:
+        configdir = './'
+    else:
+        configdir = os.getcwd()
+        builddir = builddir
+        log("Using build directory %r" % (builddir, ))
+        if not os.path.isdir(builddir):
+            os.mkdir(builddir)
+        os.chdir(builddir)
+
+    configstatus = 'config.status'
+    if not os.path.exists(configstatus):
+        args = [os.path.join(configdir, 'configure')]
+        args.extend(configargs)
+        run_sync(args)
+    else:
+        log("Found %s, skipping configure" % (configstatus, ))
+    phase_build()
+
+build_status = False
+
+def phase_build():
+    if not os.path.exists('Makefile'):
+        log("No Makefile found")
+        sys.exit(1)
+    args = makeargs
+    user_specified_jobs = False
+    for arg in args:
+        if arg == '-j':
+            user_specified_jobs = True
+
+    if not user_specified_jobs:
+        notparallel = _find_buildapi_makevariable('NOTPARALLEL')
+        if not notparallel:
+            log("Didn't find NOTPARALLEL, using parallel make by default")
+            args.extend(default_buildapi_jobs)
+
+    run_sync(args)
+
+def phase_complete():
+    sys.exit(0)
+
+log("invocation arguments: %r" % (sys.argv, ))
+
+# Start off the process
+phase_bootstrap()
diff --git a/osbuild/ostree-buildone-makeinstall-split-artifacts b/osbuild/ostree-buildone-makeinstall-split-artifacts
new file mode 100644
index 0000000..accb527
--- /dev/null
+++ b/osbuild/ostree-buildone-makeinstall-split-artifacts
@@ -0,0 +1,104 @@
+#!/usr/bin/python
+
+# ostree-buildone-raw: Generic build system wrapper
+# Copyright 2010, 2011 Colin Walters <walters verbum org>
+# Licensed under the new-BSD license (http://www.opensource.org/licenses/bsd-license.php)
+
+import os,sys,re,subprocess
+import tempfile,shutil
+
+_devel_regexps = map(re.compile,
+                     [r'/usr/include/',
+                      r'/usr/share/pkgconfig/',
+                      r'/.*lib(?:|(?:32)|(?:64))/pkgconfig/.*\.pc',
+                      r'/.*lib(?:|(?:32)|(?:64))/.*\.so$'])
+
+def log(msg):
+    fullmsg = 'ostree-buildone: ' + msg + '\n'
+    sys.stdout.write(fullmsg)
+    sys.stdout.flush()
+
+tempfiles = []
+
+def do_exit(code):
+    for tmpname in tempfiles:
+        if os.path.isdir(tmpname):
+            shutil.rmtree(tmpname)
+        else:
+            try:
+                os.unlink(tmpname)
+                pass
+            except OSError, e:
+                pass
+    sys.exit(code)
+
+def fatal(msg):
+    log(msg)
+    do_exit(1)
+
+def run_sync(args, env=None):
+    log("Running: %r" % (args, ))
+    f = open('/dev/null', 'r')
+    proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr,
+                            close_fds=True, env=env)
+    f.close()
+    returncode = proc.wait()
+    if returncode == 0:
+        func = log
+    else:
+        func = fatal
+    func("pid %d exited with code %d" % (proc.pid, returncode))
+
+basename=os.path.basename(os.getcwd())
+artifact_prefix='artifact-%s' % (basename, )
+origdir=os.getcwd()
+os.chdir('_build')
+
+if not os.path.exists('Makefile'):
+    log("No Makefile found")
+    do_exit(1)
+
+(fd,fakeroot_temp)=tempfile.mkstemp(prefix='ostree-fakeroot-%s-' % (basename,))
+os.close(fd)
+tempfiles.append(fakeroot_temp)
+tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (basename,))
+tempfiles.append(tempdir)
+args = ['fakeroot', '-s', fakeroot_temp, 'make', 'install', 'DESTDIR=' + tempdir]
+run_sync(args)
+
+devel_files = set()
+runtime_files = set()
+
+oldpwd=os.getcwd()
+os.chdir(tempdir)
+for root, dirs, files in os.walk('.'):
+    for filename in files:
+        path = os.path.join(root, filename)
+        matched = False
+        for r in _devel_regexps:
+            if r.match(path[1:]):
+                devel_files.add(path)
+                matched = True
+                break
+        if not matched:    
+            runtime_files.add(path)
+os.chdir(oldpwd)
+
+def make_artifact(name, from_files):
+    artifact_target = '%s-%s.tar.gz' % (artifact_prefix, name)
+    (fd,filelist_temp)=tempfile.mkstemp(prefix='ostree-filelist-%s-%s' % (basename, name))
+    os.close(fd)
+    tempfiles.append(filelist_temp)
+    f = open(filelist_temp, 'w')
+    for filename in from_files:
+        f.write(filename)
+        f.write('\n')
+    f.close()
+    args = ['fakeroot', '-i', fakeroot_temp, 'tar', '-c', '-z', '-C', tempdir, '-f', artifact_target, '-T', filelist_temp]
+    run_sync(args)
+
+if devel_files:
+    make_artifact('devel', devel_files)
+make_artifact('runtime', runtime_files)
+
+do_exit(0)