[gnome-ostree] Move ostbuild into this module
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-ostree] Move ostbuild into this module
- Date: Fri, 8 Jun 2012 23:01:35 +0000 (UTC)
commit 04ba1926e33b442af1e357460e4415d4c0311e87
Author: Colin Walters <walters verbum org>
Date: Fri Jun 8 18:24:15 2012 -0400
Move ostbuild into this module
This buildsystem is really quite specific to GNOME-OSTree; it doesn't
make a lot of sense to have it live in the "ostree" module.
Makefile-ostbuild.am | 57 +++
Makefile.am | 39 ++
autogen.sh | 20 +
configure.ac | 40 ++
gnome-ostree.doap | 6 +-
ostbuild/ostbuild.in | 32 ++
ostbuild/pyostbuild/buildutil.py | 187 ++++++++++
ostbuild/pyostbuild/builtin_build.py | 382 ++++++++++++++++++++
ostbuild/pyostbuild/builtin_checkout.py | 127 +++++++
ostbuild/pyostbuild/builtin_chroot_compile_one.py | 223 ++++++++++++
ostbuild/pyostbuild/builtin_compile_one.py | 273 ++++++++++++++
ostbuild/pyostbuild/builtin_deploy_qemu.py | 65 ++++
ostbuild/pyostbuild/builtin_deploy_root.py | 67 ++++
ostbuild/pyostbuild/builtin_git_mirror.py | 109 ++++++
ostbuild/pyostbuild/builtin_import_tree.py | 93 +++++
ostbuild/pyostbuild/builtin_init.py | 58 +++
ostbuild/pyostbuild/builtin_prefix.py | 73 ++++
.../pyostbuild/builtin_privhelper_deploy_qemu.py | 115 ++++++
ostbuild/pyostbuild/builtin_privhelper_run_qemu.py | 63 ++++
ostbuild/pyostbuild/builtin_resolve.py | 106 ++++++
ostbuild/pyostbuild/builtin_run_qemu.py | 49 +++
ostbuild/pyostbuild/builtin_source_diff.py | 162 +++++++++
ostbuild/pyostbuild/builtins.py | 237 ++++++++++++
ostbuild/pyostbuild/filemonitor.py | 64 ++++
ostbuild/pyostbuild/fileutil.py | 26 ++
ostbuild/pyostbuild/jsondb.py | 112 ++++++
ostbuild/pyostbuild/kvfile.py | 23 ++
ostbuild/pyostbuild/main.py | 61 +++
ostbuild/pyostbuild/mainloop.py | 96 +++++
ostbuild/pyostbuild/odict.py | 45 +++
ostbuild/pyostbuild/ostbuildlog.py | 35 ++
ostbuild/pyostbuild/ostbuildrc.py | 52 +++
ostbuild/pyostbuild/privileged_subproc.py | 39 ++
ostbuild/pyostbuild/subprocess_helpers.py | 145 ++++++++
ostbuild/pyostbuild/vcs.py | 155 ++++++++
ostbuild/pyostbuild/warningfilter.py | 113 ++++++
36 files changed, 3546 insertions(+), 3 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
new file mode 100644
index 0000000..7f28fd1
--- /dev/null
+++ b/Makefile-ostbuild.am
@@ -0,0 +1,57 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+ostbuild: ostbuild/ostbuild.in Makefile
+ sed -e s,@libdir\@,$(libdir), -e s,@datarootdir\@,$(datarootdir), -e s,@PYTHON\@,$(PYTHON), $< > $ tmp && mv $ tmp $@
+bin_SCRIPTS += ostbuild
+EXTRA_DIST += ostbuild/ostbuild.in
+
+pyostbuilddir=$(libdir)/ostbuild/pyostbuild
+pyostbuild_PYTHON = \
+ ostbuild/pyostbuild/buildutil.py \
+ ostbuild/pyostbuild/builtin_build.py \
+ ostbuild/pyostbuild/builtin_checkout.py \
+ ostbuild/pyostbuild/builtin_chroot_compile_one.py \
+ ostbuild/pyostbuild/builtin_compile_one.py \
+ ostbuild/pyostbuild/builtin_deploy_qemu.py \
+ ostbuild/pyostbuild/builtin_deploy_root.py \
+ ostbuild/pyostbuild/builtin_run_qemu.py \
+ ostbuild/pyostbuild/builtin_import_tree.py \
+ ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py \
+ ostbuild/pyostbuild/builtin_privhelper_run_qemu.py \
+ ostbuild/pyostbuild/builtin_git_mirror.py \
+ ostbuild/pyostbuild/builtin_prefix.py \
+ ostbuild/pyostbuild/builtin_resolve.py \
+ ostbuild/pyostbuild/builtin_init.py \
+ ostbuild/pyostbuild/builtin_source_diff.py \
+ ostbuild/pyostbuild/builtins.py \
+ ostbuild/pyostbuild/filemonitor.py \
+ ostbuild/pyostbuild/fileutil.py \
+ ostbuild/pyostbuild/__init__.py \
+ ostbuild/pyostbuild/jsondb.py \
+ ostbuild/pyostbuild/kvfile.py \
+ ostbuild/pyostbuild/main.py \
+ ostbuild/pyostbuild/mainloop.py \
+ ostbuild/pyostbuild/odict.py \
+ ostbuild/pyostbuild/ostbuildlog.py \
+ ostbuild/pyostbuild/ostbuildrc.py \
+ ostbuild/pyostbuild/privileged_subproc.py \
+ ostbuild/pyostbuild/warningfilter.py \
+ ostbuild/pyostbuild/subprocess_helpers.py \
+ ostbuild/pyostbuild/vcs.py \
+ $(NULL)
+
diff --git a/Makefile.am b/Makefile.am
new file mode 100644
index 0000000..e2d996e
--- /dev/null
+++ b/Makefile.am
@@ -0,0 +1,39 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+ACLOCAL_AMFLAGS = -I m4 ${ACLOCAL_FLAGS}
+AM_CPPFLAGS = -DDATADIR='"$(datadir)"' -DLIBEXECDIR='"$(libexecdir)"'
+AM_CFLAGS = $(WARN_CFLAGS)
+
+NULL =
+BUILT_SOURCES =
+MANPAGES =
+CLEANFILES =
+EXTRA_DIST =
+bin_PROGRAMS =
+sbin_PROGRAMS =
+bin_SCRIPTS =
+libexec_PROGRAMS =
+noinst_LTLIBRARIES =
+noinst_PROGRAMS =
+privlibdir = $(pkglibdir)
+privlib_LTLIBRARIES =
+
+include Makefile-ostbuild.am
+
+release-tag:
+ git tag -m "Release $(VERSION)" v$(VERSION)
diff --git a/autogen.sh b/autogen.sh
new file mode 100755
index 0000000..6035bc0
--- /dev/null
+++ b/autogen.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+test -n "$srcdir" || srcdir=`dirname "$0"`
+test -n "$srcdir" || srcdir=.
+
+olddir=`pwd`
+cd $srcdir
+
+AUTORECONF=`which autoreconf`
+if test -z $AUTORECONF; then
+ echo "*** No autoreconf found, please intall it ***"
+ exit 1
+fi
+
+mkdir -p m4
+
+autoreconf --force --install --verbose
+
+cd $olddir
+test -n "$NOCONFIGURE" || "$srcdir/configure" "$@"
diff --git a/configure.ac b/configure.ac
new file mode 100644
index 0000000..04b9847
--- /dev/null
+++ b/configure.ac
@@ -0,0 +1,40 @@
+AC_PREREQ([2.63])
+AC_INIT([gnome-ostree], [2012.1], [walters verbum org])
+AC_CONFIG_HEADER([config.h])
+AC_CONFIG_MACRO_DIR([m4])
+AC_CONFIG_AUX_DIR([build-aux])
+
+AM_INIT_AUTOMAKE([1.11 -Wno-portability foreign no-define tar-ustar no-dist-gzip dist-xz])
+AM_MAINTAINER_MODE([enable])
+AM_SILENT_RULES([yes])
+
+AC_PROG_CC
+AM_PROG_CC_C_O
+
+changequote(,)dnl
+if test "x$GCC" = "xyes"; then
+ WARN_CFLAGS="-Wall -Werror=missing-prototypes"
+fi
+changequote([,])dnl
+AC_SUBST(WARN_CFLAGS)
+
+# Initialize libtool
+LT_PREREQ([2.2.4])
+LT_INIT([disable-static])
+
+PKG_PROG_PKG_CONFIG
+
+GIO_DEPENDENCY="gio-unix-2.0 >= 2.28"
+
+PKG_CHECK_MODULES(OT_DEP_GIO_UNIX, $GIO_DEPENDENCY)
+
+AM_PATH_PYTHON([2.7])
+
+AC_CONFIG_FILES([
+Makefile
+])
+AC_OUTPUT
+
+echo "
+ GNOME-OSTree $VERSION
+"
diff --git a/gnome-ostree.doap b/gnome-ostree.doap
index 2cfa482..50fc50c 100644
--- a/gnome-ostree.doap
+++ b/gnome-ostree.doap
@@ -8,11 +8,11 @@
<name>gnome-ostree</name>
<shortname>gnome-ostree</shortname>
- <shortdesc xml:lang="en">GNOME OSTree manifest and patches</shortdesc>
+ <shortdesc xml:lang="en">GNOME OSTree build tool, manifest, and patches</shortdesc>
- <description xml:lang="en">GNOME OSTree manifest and patches</description>
+ <description xml:lang="en">GNOME OSTree build tool, manifest and patches</description>
- <homepage rdf:resource="http://live.gnome.org/OSTree"; />
+ <homepage rdf:resource="https://live.gnome.org/OSTree/GnomeOSTree"; />
<license rdf:resource="http://usefulinc.com/doap/licenses/lgpl"; />
<mailing-list rdf:resource="mailto:ostree-list gnome org" />
diff --git a/ostbuild/ostbuild.in b/ostbuild/ostbuild.in
new file mode 100755
index 0000000..a200235
--- /dev/null
+++ b/ostbuild/ostbuild.in
@@ -0,0 +1,32 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import sys
+import __builtin__
+
+__builtin__.__dict__['DATADIR'] = '@datarootdir@'
+# This is a private directory, we don't want to pollute the global
+# namespace.
+path = os.path.join('@libdir@', 'ostbuild')
+sys.path.insert(0, path)
+
+from pyostbuild.main import main
+
+sys.exit(main(sys.argv[1:]))
diff --git a/ostbuild/pyostbuild/__init__.py b/ostbuild/pyostbuild/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/ostbuild/pyostbuild/buildutil.py b/ostbuild/pyostbuild/buildutil.py
new file mode 100755
index 0000000..94ad63b
--- /dev/null
+++ b/ostbuild/pyostbuild/buildutil.py
@@ -0,0 +1,187 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import re
+import urlparse
+import tempfile
+import StringIO
+
+from . import ostbuildrc
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync_get_output
+
+BUILD_ENV = {
+ 'HOME' : '/',
+ 'HOSTNAME' : 'ostbuild',
+ 'LANG': 'C',
+ 'PATH' : '/usr/bin:/bin:/usr/sbin:/sbin',
+ 'SHELL' : '/bin/bash',
+ 'TERM' : 'vt100',
+ 'TMPDIR' : '/tmp',
+ 'TZ': 'EST5EDT'
+ }
+
+def parse_src_key(srckey):
+ idx = srckey.find(':')
+ if idx < 0:
+ raise ValueError("Invalid SRC uri=%s" % (srckey, ))
+ keytype = srckey[:idx]
+ if keytype not in ['git', 'local']:
+ raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
+ uri = srckey[idx+1:]
+ return (keytype, uri)
+
+
+def get_mirrordir(mirrordir, keytype, uri, prefix=''):
+ if keytype != 'git':
+ fatal("Unhandled keytype '%s' for uri '%s'" % (keytype, uri))
+ parsed = urlparse.urlsplit(uri)
+ return os.path.join(mirrordir, prefix, keytype, parsed.scheme, parsed.netloc, parsed.path[1:])
+
+def find_user_chroot_path():
+ # We need to search PATH here manually so we correctly pick up an
+ # ostree install in e.g. ~/bin even though we're going to set PATH
+ # below for our children inside the chroot.
+ ostbuild_user_chroot_path = None
+ for dirname in os.environ['PATH'].split(':'):
+ path = os.path.join(dirname, 'linux-user-chroot')
+ if os.access(path, os.X_OK):
+ ostbuild_user_chroot_path = path
+ break
+ if ostbuild_user_chroot_path is None:
+ ostbuild_user_chroot_path = 'linux-user-chroot'
+ return ostbuild_user_chroot_path
+
+def branch_name_for_artifact(a):
+ return 'artifacts/%s/%s/%s' % (a['buildroot'],
+ a['name'],
+ a['branch'])
+
+def get_git_version_describe(dirpath, commit=None):
+ args = ['git', 'describe', '--long', '--abbrev=42', '--always']
+ if commit is not None:
+ args.append(commit)
+ version = run_sync_get_output(args, cwd=dirpath)
+ return version.strip()
+
+def ref_to_unix_name(ref):
+ return ref.replace('/', '.')
+
+def tsort_components(components, key):
+ (fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-tsort-')
+ f = os.fdopen(fd, 'w')
+ for name,component in components.iteritems():
+ build_prev = component.get(key)
+ if (build_prev is not None and len(build_prev) > 0):
+ for dep_name in build_prev:
+ f.write('%s %s\n' % (name, dep_name))
+ f.close()
+
+ output = run_sync_get_output(['tsort', path])
+ os.unlink(path)
+ output_stream = StringIO.StringIO(output)
+ result = []
+ for line in output_stream:
+ result.append(line.strip())
+ return result
+
+def _recurse_depends(depkey, component_name, components, dep_names):
+ component = components[component_name]
+ depends = component.get(depkey)
+ if (depends is None or len(depends) == 0):
+ return
+ for depname in depends:
+ dep_names.add(depname)
+ _recurse_depends(depkey, depname, components, dep_names)
+
+def _sorted_depends(deptype, component_name, components):
+ dep_names = set()
+ _recurse_depends(deptype, component_name, components, dep_names)
+ dep_components = {}
+ for component_name in dep_names:
+ dep_components[component_name] = components[component_name]
+ result = tsort_components(dep_components, deptype)
+ result.reverse()
+ return result
+
+def build_depends(component_name, components):
+ return _sorted_depends('build-depends', component_name, components)
+
+def runtime_depends(component_name, components):
+ return _sorted_depends('runtime-depends', component_name, components)
+
+def find_component_in_manifest(manifest, component_name):
+ for component in manifest['components']:
+ if component['name'] == component_name:
+ return component
+ return None
+
+def compose(repo, target, artifacts):
+ child_args = ['ostree', '--repo=' + repo, 'compose',
+ '-b', target, '-s', 'Compose']
+ (fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+ f = os.fdopen(fd, 'w')
+ for artifact in artifacts:
+ f.write(artifact)
+ f.write('\n')
+ f.close()
+ child_args.extend(['-F', path])
+ revision = run_sync_get_output(child_args, log_initiation=True).strip()
+ os.unlink(path)
+ return revision
+
+def get_base_user_chroot_args():
+ path = find_user_chroot_path()
+ args = [path, '--unshare-pid', '--unshare-ipc']
+ if not ostbuildrc.get_key('preserve_net', default=False):
+ args.append('--unshare-net')
+ return args
+
+
+def resolve_component_meta(snapshot, component_meta):
+ result = dict(component_meta)
+ orig_src = component_meta['src']
+
+ did_expand = False
+ for (vcsprefix, expansion) in snapshot['vcsconfig'].iteritems():
+ prefix = vcsprefix + ':'
+ if orig_src.startswith(prefix):
+ result['src'] = expansion + orig_src[len(prefix):]
+ did_expand = True
+ break
+
+ name = component_meta.get('name')
+ if name is None:
+ if did_expand:
+ src = orig_src
+ idx = src.rindex(':')
+ name = src[idx+1:]
+ else:
+ src = result['src']
+ idx = src.rindex('/')
+ name = src[idx+1:]
+ if name.endswith('.git'):
+ name = name[:-4]
+ name = name.replace('/', '-')
+ result['name'] = name
+
+ branch_or_tag = result.get('branch') or result.get('tag')
+ if branch_or_tag is None:
+ result['branch'] = 'master'
+
+ return result
diff --git a/ostbuild/pyostbuild/builtin_build.py b/ostbuild/pyostbuild/builtin_build.py
new file mode 100755
index 0000000..225584b
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_build.py
@@ -0,0 +1,382 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import hashlib
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from .subprocess_helpers import run_sync_monitor_log_file
+from . import ostbuildrc
+from . import buildutil
+from . import fileutil
+from . import kvfile
+from . import odict
+from . import vcs
+
+class BuildOptions(object):
+ pass
+
+class OstbuildBuild(builtins.Builtin):
+ name = "build"
+ short_description = "Build multiple components and generate trees"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def _get_ostbuild_chroot_args(self, architecture, component, component_resultdir):
+ basename = component['name']
+ current_machine = os.uname()[4]
+ if current_machine != architecture:
+ args = ['setarch', architecture]
+ else:
+ args = []
+ args.extend(['ostbuild', 'chroot-compile-one',
+ '--snapshot=' + self.snapshot_path,
+ '--name=' + basename, '--arch=' + architecture,
+ '--resultdir=' + component_resultdir])
+ return args
+
+ def _launch_debug_shell(self, architecture, component, component_resultdir, cwd=None):
+ args = self._get_ostbuild_chroot_args(architecture, component, component_resultdir)
+ args.append('--debug-shell')
+ run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
+ fatal("Exiting after debug shell")
+
+ def _build_one_component(self, component, architecture):
+ basename = component['name']
+
+ buildname = '%s/%s/%s' % (self.snapshot['prefix'], basename, architecture)
+ build_ref = 'components/%s' % (buildname, )
+
+ current_vcs_version = component.get('revision')
+
+ expanded_component = self.expand_component(component)
+
+ # TODO - deduplicate this with chroot_compile_one
+ current_meta_io = StringIO()
+ json.dump(expanded_component, current_meta_io, indent=4, sort_keys=True)
+ current_metadata_text = current_meta_io.getvalue()
+ sha = hashlib.sha256()
+ sha.update(current_metadata_text)
+ current_meta_digest = sha.hexdigest()
+
+ if (self.buildopts.force_rebuild or
+ basename in self.force_build_components):
+ previous_build_version = None
+ else:
+ previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', build_ref],
+ stderr=open('/dev/null', 'w'),
+ none_on_error=True)
+ if (current_vcs_version is not None
+ and previous_build_version is not None):
+ log("Previous build of '%s' is %s" % (buildname, previous_build_version))
+
+ previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'cat', previous_build_version,
+ '/_ostbuild-meta.json'],
+ log_initiation=True)
+ sha = hashlib.sha256()
+ sha.update(previous_metadata_text)
+ previous_meta_digest = sha.hexdigest()
+
+ if current_meta_digest == previous_meta_digest:
+ log("Metadata is unchanged from previous")
+ return previous_build_version
+ else:
+ previous_metadata = json.loads(previous_metadata_text)
+ previous_vcs_version = previous_metadata.get('revision')
+ if current_vcs_version == previous_vcs_version:
+ log("Metadata differs; VCS version unchanged")
+ if self.buildopts.skip_vcs_matches:
+ return previous_build_version
+ for k,v in expanded_component.iteritems():
+ previous_v = previous_metadata.get(k)
+ if v != previous_v:
+ log("Key %r differs: old: %r new: %r" % (k, previous_v, v))
+ else:
+ log("Metadata differs; note vcs version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
+ else:
+ log("No previous build for '%s' found" % (buildname, ))
+
+ checkoutdir = os.path.join(self.workdir, 'checkouts')
+ component_src = os.path.join(checkoutdir, buildname)
+ fileutil.ensure_parent_dir(component_src)
+ child_args = ['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
+ '--checkoutdir=' + component_src,
+ '--clean', '--overwrite', basename]
+ if self.args.patches_path:
+ child_args.append('--patches-path=' + self.args.patches_path)
+ run_sync(child_args)
+
+ artifact_meta = dict(component)
+
+ logdir = os.path.join(self.workdir, 'logs', buildname)
+ fileutil.ensure_dir(logdir)
+ log_path = os.path.join(logdir, 'compile.log')
+ if os.path.isfile(log_path):
+ curtime = int(time.time())
+ saved_name = os.path.join(logdir, 'compile-prev.log')
+ os.rename(log_path, saved_name)
+
+ component_resultdir = os.path.join(self.workdir, 'results', buildname)
+ if os.path.isdir(component_resultdir):
+ shutil.rmtree(component_resultdir)
+ fileutil.ensure_dir(component_resultdir)
+
+ log("Logging to %s" % (log_path, ))
+ f = open(log_path, 'w')
+ chroot_args = self._get_ostbuild_chroot_args(architecture, component, component_resultdir)
+ if self.buildopts.shell_on_failure:
+ ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
+ if ecode != 0:
+ self._launch_debug_shell(architecture, component, component_resultdir, cwd=component_src)
+ else:
+ run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src)
+
+ args = ['ostree', '--repo=' + self.repo,
+ 'commit', '-b', build_ref, '-s', 'Build',
+ '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
+ '--skip-if-unchanged']
+
+ setuid_files = artifact_meta.get('setuid', [])
+ statoverride_path = None
+ if len(setuid_files) > 0:
+ (fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-')
+ f = os.fdopen(fd, 'w')
+ for path in setuid_files:
+ f.write('+2048 ' + path)
+ f.write('\n')
+ f.close()
+ args.append('--statoverride=' + statoverride_path)
+
+ run_sync(args, cwd=component_resultdir)
+ if statoverride_path is not None:
+ os.unlink(statoverride_path)
+
+ if os.path.islink(component_src):
+ os.unlink(component_src)
+ else:
+ shutil.rmtree(component_src)
+ shutil.rmtree(component_resultdir)
+
+ return run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', build_ref])
+
+ def _compose_one_target(self, target, component_build_revs):
+ base = target['base']
+ base_name = 'bases/%s' % (base['name'], )
+ runtime_name = 'bases/%s' % (base['runtime'], )
+ devel_name = 'bases/%s' % (base['devel'], )
+
+ compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
+ fileutil.ensure_parent_dir(compose_rootdir)
+ if os.path.isdir(compose_rootdir):
+ shutil.rmtree(compose_rootdir)
+ os.mkdir(compose_rootdir)
+
+ related_refs = {}
+
+ base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', base_name])
+
+ runtime_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', runtime_name])
+ related_refs[runtime_name] = runtime_revision
+ devel_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', devel_name])
+ related_refs[devel_name] = devel_revision
+
+ for name,rev in component_build_revs.iteritems():
+ build_ref = 'components/%s/%s' % (self.snapshot['prefix'], name)
+ related_refs[build_ref] = rev
+
+ (related_fd, related_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+ related_f = os.fdopen(related_fd, 'w')
+ for (name, rev) in related_refs.iteritems():
+ related_f.write(name)
+ related_f.write(' ')
+ related_f.write(rev)
+ related_f.write('\n')
+ related_f.close()
+
+ compose_contents = [(base_revision, '/')]
+ for tree_content in target['contents']:
+ name = tree_content['name']
+ rev = component_build_revs[name]
+ subtrees = tree_content['trees']
+ for subpath in subtrees:
+ compose_contents.append((rev, subpath))
+
+ (contents_fd, contents_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+ contents_f = os.fdopen(contents_fd, 'w')
+ for (branch, subpath) in compose_contents:
+ contents_f.write(branch)
+ contents_f.write('\0')
+ contents_f.write(subpath)
+ contents_f.write('\0')
+ contents_f.close()
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'checkout', '--user-mode', '--no-triggers', '--union',
+ '--from-file=' + contents_tmppath, compose_rootdir])
+ os.unlink(contents_tmppath)
+
+ contents_path = os.path.join(compose_rootdir, 'contents.json')
+ f = open(contents_path, 'w')
+ json.dump(self.snapshot, f, indent=4, sort_keys=True)
+ f.close()
+
+ treename = 'trees/%s' % (target['name'], )
+
+ child_args = ['ostree', '--repo=' + self.repo,
+ 'commit', '-b', treename, '-s', 'Compose',
+ '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
+ '--related-objects-file=' + related_tmppath,
+ ]
+ if not self.buildopts.no_skip_if_unchanged:
+ child_args.append('--skip-if-unchanged')
+ run_sync(child_args, cwd=compose_rootdir)
+ os.unlink(related_tmppath)
+ shutil.rmtree(compose_rootdir)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--src-snapshot')
+ parser.add_argument('--patches-path')
+ parser.add_argument('--force-rebuild', action='store_true')
+ parser.add_argument('--skip-vcs-matches', action='store_true')
+ parser.add_argument('--no-compose', action='store_true')
+ parser.add_argument('--no-skip-if-unchanged', action='store_true')
+ parser.add_argument('--compose-only', action='store_true')
+ parser.add_argument('--shell-on-failure', action='store_true')
+ parser.add_argument('--debug-shell', action='store_true')
+ parser.add_argument('components', nargs='*')
+
+ args = parser.parse_args(argv)
+ self.args = args
+
+ self.parse_config()
+ self.parse_snapshot(args.prefix, args.src_snapshot)
+
+ log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
+
+ self.buildopts = BuildOptions()
+ self.buildopts.shell_on_failure = args.shell_on_failure
+ self.buildopts.force_rebuild = args.force_rebuild
+ self.buildopts.skip_vcs_matches = args.skip_vcs_matches
+ self.buildopts.no_skip_if_unchanged = args.no_skip_if_unchanged
+
+ self.force_build_components = set()
+
+ components = self.snapshot['components']
+
+ prefix = self.snapshot['prefix']
+ base_prefix = '%s/%s' % (self.snapshot['base']['name'], prefix)
+
+ architectures = self.snapshot['architectures']
+
+ component_to_arches = {}
+
+ runtime_components = []
+ devel_components = []
+
+ for component in components:
+ name = component['name']
+
+ is_runtime = component.get('component', 'runtime') == 'runtime'
+
+ if is_runtime:
+ runtime_components.append(component)
+ devel_components.append(component)
+
+ is_noarch = component.get('noarch', False)
+ if is_noarch:
+ # Just use the first specified architecture
+ component_arches = [architectures[0]]
+ else:
+ component_arches = component.get('architectures', architectures)
+ component_to_arches[name] = component_arches
+
+ for name in args.components:
+ component = self.get_component(name)
+ self.force_build_components.add(component['name'])
+
+ components_to_build = []
+ component_skipped_count = 0
+
+ component_build_revs = {}
+
+ if not args.compose_only:
+ for component in components:
+ for architecture in architectures:
+ components_to_build.append((component, architecture))
+
+ log("%d components to build" % (len(components_to_build), ))
+ for (component, architecture) in components_to_build:
+ archname = '%s/%s' % (component['name'], architecture)
+ build_rev = self._build_one_component(component, architecture)
+ component_build_revs[archname] = build_rev
+
+ targets_list = []
+ for target_component_type in ['runtime', 'devel']:
+ for architecture in architectures:
+ target = {}
+ targets_list.append(target)
+ target['name'] = '%s-%s-%s' % (prefix, architecture, target_component_type)
+
+ runtime_ref = '%s-%s-runtime' % (base_prefix, architecture)
+ buildroot_ref = '%s-%s-devel' % (base_prefix, architecture)
+ if target_component_type == 'runtime':
+ base_ref = runtime_ref
+ else:
+ base_ref = buildroot_ref
+ target['base'] = {'name': base_ref,
+ 'runtime': runtime_ref,
+ 'devel': buildroot_ref}
+
+ if target_component_type == 'runtime':
+ target_components = runtime_components
+ else:
+ target_components = devel_components
+
+ contents = []
+ for component in target_components:
+ builds_for_component = component_to_arches[component['name']]
+ if architecture not in builds_for_component:
+ continue
+ binary_name = '%s/%s' % (component['name'], architecture)
+ component_ref = {'name': binary_name}
+ if target_component_type == 'runtime':
+ component_ref['trees'] = ['/runtime']
+ else:
+ component_ref['trees'] = ['/runtime', '/devel', '/doc']
+ contents.append(component_ref)
+ target['contents'] = contents
+
+ for target in targets_list:
+ self._compose_one_target(target, component_build_revs)
+
+builtins.register(OstbuildBuild)
diff --git a/ostbuild/pyostbuild/builtin_checkout.py b/ostbuild/pyostbuild/builtin_checkout.py
new file mode 100755
index 0000000..153ea8a
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_checkout.py
@@ -0,0 +1,127 @@
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import json
+import urlparse
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import ostbuildrc
+from . import buildutil
+from . import fileutil
+from . import odict
+from . import vcs
+
+class OstbuildCheckout(builtins.Builtin):
+ name = "checkout"
+ short_description = "Check out specified modules"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--overwrite', action='store_true')
+ parser.add_argument('--prefix')
+ parser.add_argument('--patches-path')
+ parser.add_argument('--snapshot')
+ parser.add_argument('--checkoutdir')
+ parser.add_argument('-a', '--active-tree', action='store_true')
+ parser.add_argument('--clean', action='store_true')
+ parser.add_argument('component')
+
+ args = parser.parse_args(argv)
+ self.args = args
+
+ self.parse_config()
+
+ if args.active_tree:
+ self.parse_active_branch()
+ else:
+ self.parse_snapshot(args.prefix, args.snapshot)
+
+ component_name = args.component
+
+ found = False
+ component = self.get_expanded_component(component_name)
+ (keytype, uri) = buildutil.parse_src_key(component['src'])
+
+ is_local = (keytype == 'local')
+
+ if is_local:
+ if args.checkoutdir:
+ checkoutdir = args.checkoutdir
+ # Kind of a hack, but...
+ if os.path.islink(checkoutdir):
+ os.unlink(checkoutdir)
+ if args.overwrite and os.path.isdir(checkoutdir):
+ shutil.rmtree(checkoutdir)
+ os.symlink(uri, checkoutdir)
+ else:
+ checkoutdir = uri
+ else:
+ if args.checkoutdir:
+ checkoutdir = args.checkoutdir
+ else:
+ checkoutdir = os.path.join(os.getcwd(), component_name)
+ fileutil.ensure_parent_dir(checkoutdir)
+ vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
+ component['revision'],
+ overwrite=args.overwrite)
+
+ if args.clean:
+ if is_local:
+ log("note: ignoring --clean argument due to \"local:\" specification")
+ else:
+ vcs.clean(keytype, checkoutdir)
+
+ patches = component.get('patches')
+ if patches is not None:
+ if self.args.patches_path:
+ (patches_keytype, patches_uri) = ('local', self.args.patches_path)
+ else:
+ (patches_keytype, patches_uri) = buildutil.parse_src_key(patches['src'])
+ if patches_keytype == 'git':
+ patches_mirror = buildutil.get_mirrordir(self.mirrordir, patches_keytype, patches_uri)
+ vcs.get_vcs_checkout(self.mirrordir, patches_keytype, patches_uri,
+ self.patchdir, patches['branch'],
+ overwrite=True)
+ patchdir = self.patchdir
+ else:
+ patchdir = patches_uri
+
+ patch_subdir = patches.get('subdir', None)
+ if patch_subdir is not None:
+ patchdir = os.path.join(patchdir, patch_subdir)
+ else:
+ patchdir = self.patchdir
+ for patch in patches['files']:
+ patch_path = os.path.join(patchdir, patch)
+ run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=checkoutdir)
+
+ metadata_path = os.path.join(checkoutdir, '_ostbuild-meta.json')
+ f = open(metadata_path, 'w')
+ json.dump(component, f, indent=4, sort_keys=True)
+ f.close()
+
+ log("Checked out: %r" % (checkoutdir, ))
+
+builtins.register(OstbuildCheckout)
diff --git a/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/ostbuild/pyostbuild/builtin_chroot_compile_one.py
new file mode 100755
index 0000000..9ff1122
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_chroot_compile_one.py
@@ -0,0 +1,223 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,re,subprocess,tempfile,shutil
+from StringIO import StringIO
+import argparse
+import time
+import json
+import hashlib
+
+from . import builtins
+from . import buildutil
+from . import fileutil
+from . import ostbuildrc
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+
+class OstbuildChrootCompileOne(builtins.Builtin):
+ name = "chroot-compile-one"
+ short_description = "Build artifacts from the current source directory in a chroot"
+
+ def _resolve_refs(self, refs):
+ if len(refs) == 0:
+ return []
+ args = ['ostree', '--repo=' + self.repo, 'rev-parse']
+ args.extend(refs)
+ output = run_sync_get_output(args)
+ return output.split('\n')
+
+ def _compose_buildroot(self, component_name, architecture):
+ starttime = time.time()
+
+ rootdir_prefix = os.path.join(self.workdir, 'roots')
+ rootdir = os.path.join(rootdir_prefix, component_name)
+ fileutil.ensure_parent_dir(rootdir)
+
+ # Clean up any leftover root dir
+ rootdir_tmp = rootdir + '.tmp'
+ if os.path.isdir(rootdir_tmp):
+ shutil.rmtree(rootdir_tmp)
+
+ components = self.snapshot['components']
+ component = None
+ build_dependencies = []
+ for component in components:
+ if component['name'] == component_name:
+ break
+ build_dependencies.append(component)
+
+ ref_to_rev = {}
+
+ prefix = self.snapshot['prefix']
+
+ arch_buildroot_name = 'bases/%s/%s-%s-devel' % (self.snapshot['base']['name'],
+ prefix,
+ architecture)
+
+ arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse',
+ arch_buildroot_name]).strip()
+
+ ref_to_rev[arch_buildroot_name] = arch_buildroot_rev
+ checkout_trees = [(arch_buildroot_name, '/')]
+ refs_to_resolve = []
+ for dependency in build_dependencies:
+ buildname = 'components/%s/%s/%s' % (prefix, dependency['name'], architecture)
+ refs_to_resolve.append(buildname)
+ checkout_trees.append((buildname, '/runtime'))
+ checkout_trees.append((buildname, '/devel'))
+
+ resolved_refs = self._resolve_refs(refs_to_resolve)
+ for ref,rev in zip(refs_to_resolve, resolved_refs):
+ ref_to_rev[ref] = rev
+
+ sha = hashlib.sha256()
+
+ (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-buildroot-')
+ f = os.fdopen(fd, 'w')
+ for (branch, subpath) in checkout_trees:
+ f.write(ref_to_rev[branch])
+ f.write('\0')
+ f.write(subpath)
+ f.write('\0')
+ f.close()
+
+ f = open(tmppath)
+ buf = f.read(8192)
+ while buf != '':
+ sha.update(buf)
+ buf = f.read(8192)
+ f.close()
+
+ new_root_cacheid = sha.hexdigest()
+
+ rootdir_cache_path = os.path.join(rootdir_prefix, component_name + '.cacheid')
+
+ if os.path.isdir(rootdir):
+ if os.path.isfile(rootdir_cache_path):
+ f = open(rootdir_cache_path)
+ prev_cache_id = f.read().strip()
+ f.close()
+ if prev_cache_id == new_root_cacheid:
+ log("Reusing previous buildroot")
+ os.unlink(tmppath)
+ return rootdir
+ else:
+ log("New buildroot differs from previous")
+
+ shutil.rmtree(rootdir)
+
+ os.mkdir(rootdir_tmp)
+
+ if len(checkout_trees) > 0:
+ log("composing buildroot from %d parents (last: %r)" % (len(checkout_trees),
+ checkout_trees[-1][0]))
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'checkout', '--user-mode', '--union',
+ '--from-file=' + tmppath, rootdir_tmp])
+
+ os.unlink(tmppath);
+
+ builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
+ os.mkdir(builddir_tmp)
+ os.mkdir(os.path.join(builddir_tmp, 'source'))
+ os.mkdir(os.path.join(builddir_tmp, 'results'))
+ os.rename(rootdir_tmp, rootdir)
+
+ f = open(rootdir_cache_path, 'w')
+ f.write(new_root_cacheid)
+ f.write('\n')
+ f.close()
+
+ endtime = time.time()
+ log("Composed buildroot; %d seconds elapsed" % (int(endtime - starttime),))
+
+ return rootdir
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--snapshot', required=True)
+ parser.add_argument('--name')
+ parser.add_argument('--resultdir')
+ parser.add_argument('--arch', required=True)
+ parser.add_argument('--debug-shell', action='store_true')
+
+ args = parser.parse_args(argv)
+
+ self.parse_config()
+ self.parse_snapshot(args.prefix, args.snapshot)
+
+ if args.name:
+ component_name = args.name
+ else:
+ component_name = self.get_component_from_cwd()
+
+ component = self.get_expanded_component(component_name)
+
+ workdir = self.workdir
+
+ log("Using working directory: %s" % (workdir, ))
+
+ child_tmpdir=os.path.join(workdir, 'tmp')
+ if os.path.isdir(child_tmpdir):
+ log("Cleaning up previous tmpdir: %r" % (child_tmpdir, ))
+ shutil.rmtree(child_tmpdir)
+ fileutil.ensure_dir(child_tmpdir)
+
+ resultdir = args.resultdir
+
+ rootdir = self._compose_buildroot(component_name, args.arch)
+
+ log("Checked out buildroot: %s" % (rootdir, ))
+
+ sourcedir=os.path.join(rootdir, 'ostbuild', 'source', component_name)
+ fileutil.ensure_dir(sourcedir)
+
+ output_metadata = open('_ostbuild-meta.json', 'w')
+ json.dump(component, output_metadata, indent=4, sort_keys=True)
+ output_metadata.close()
+
+ chroot_sourcedir = os.path.join('/ostbuild', 'source', component_name)
+
+ child_args = buildutil.get_base_user_chroot_args()
+ child_args.extend([
+ '--mount-readonly', '/',
+ '--mount-proc', '/proc',
+ '--mount-bind', '/dev', '/dev',
+ '--mount-bind', child_tmpdir, '/tmp',
+ '--mount-bind', os.getcwd(), chroot_sourcedir,
+ '--mount-bind', resultdir, '/ostbuild/results',
+ '--chdir', chroot_sourcedir])
+ if args.debug_shell:
+ child_args.extend([rootdir, '/bin/sh'])
+ else:
+ child_args.extend([rootdir, '/usr/bin/ostbuild',
+ 'compile-one',
+ '--ostbuild-resultdir=/ostbuild/results',
+ '--ostbuild-meta=_ostbuild-meta.json'])
+ env_copy = dict(buildutil.BUILD_ENV)
+ env_copy['PWD'] = chroot_sourcedir
+ run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell)
+
+ recorded_meta_path = os.path.join(resultdir, '_ostbuild-meta.json')
+ recorded_meta_f = open(recorded_meta_path, 'w')
+ json.dump(component, recorded_meta_f, indent=4, sort_keys=True)
+ recorded_meta_f.close()
+
+builtins.register(OstbuildChrootCompileOne)
diff --git a/ostbuild/pyostbuild/builtin_compile_one.py b/ostbuild/pyostbuild/builtin_compile_one.py
new file mode 100755
index 0000000..aa5fc84
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_compile_one.py
@@ -0,0 +1,273 @@
+# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+from StringIO import StringIO
+import json
+from multiprocessing import cpu_count
+import select,time
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+
+PREFIX = '/usr'
+
+_DOC_DIRS = ['usr/share/doc',
+ 'usr/share/gtk-doc',
+ 'usr/share/man',
+ 'usr/share/info']
+
+_DEVEL_DIRS = ['usr/include',
+ 'usr/share/aclocal',
+ 'usr/share/pkgconfig',
+ 'usr/lib/pkgconfig']
+
+class OstbuildCompileOne(builtins.Builtin):
+ name = "compile-one"
+ short_description = "Build artifacts from the current source directory"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+ self.tempfiles = []
+
+ def _has_buildapi_configure_variable(self, name):
+ var = '#buildapi-variable-%s' % (name, )
+ for line in open('configure'):
+ if line.find(var) >= 0:
+ return True
+ return False
+
+ def execute(self, args):
+ self.default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )]
+
+ starttime = time.time()
+
+ uname=os.uname()
+ kernel=uname[0].lower()
+ machine=uname[4]
+ self.build_target='%s-%s' % (machine, kernel)
+
+ self.configargs = ['--build=' + self.build_target,
+ '--prefix=' + PREFIX,
+ '--libdir=' + os.path.join(PREFIX, 'lib'),
+ '--sysconfdir=/etc',
+ '--localstatedir=/var',
+ '--bindir=' + os.path.join(PREFIX, 'bin'),
+ '--sbindir=' + os.path.join(PREFIX, 'sbin'),
+ '--datadir=' + os.path.join(PREFIX, 'share'),
+ '--includedir=' + os.path.join(PREFIX, 'include'),
+ '--libexecdir=' + os.path.join(PREFIX, 'libexec'),
+ '--mandir=' + os.path.join(PREFIX, 'share', 'man'),
+ '--infodir=' + os.path.join(PREFIX, 'share', 'info')]
+ self.makeargs = ['make']
+
+ self.ostbuild_resultdir='_ostbuild-results'
+ self.ostbuild_meta_path='_ostbuild-meta.json'
+
+ chdir = None
+ opt_install = False
+
+ for arg in args:
+ if arg.startswith('--ostbuild-resultdir='):
+ self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):]
+ elif arg.startswith('--ostbuild-meta='):
+ self.ostbuild_meta_path=arg[len('--ostbuild-meta='):]
+ elif arg.startswith('--chdir='):
+ os.chdir(arg[len('--chdir='):])
+ else:
+ self.makeargs.append(arg)
+
+ f = open(self.ostbuild_meta_path)
+ self.metadata = json.load(f)
+ f.close()
+
+ self.configargs.extend(self.metadata.get('config-opts', []))
+
+ if self.metadata.get('rm-configure', False):
+ configure_path = 'configure'
+ if os.path.exists(configure_path):
+ os.unlink(configure_path)
+
+ autogen_script = None
+ if not os.path.exists('configure'):
+ log("No 'configure' script found, looking for autogen/bootstrap")
+ for name in ['autogen', 'autogen.sh', 'bootstrap']:
+ if os.path.exists(name):
+ log("Using bootstrap script '%s'" % (name, ))
+ autogen_script = name
+ if autogen_script is None:
+ fatal("No configure or autogen script detected; unknown buildsystem")
+
+ if autogen_script is not None:
+ env = dict(os.environ)
+ env['NOCONFIGURE'] = '1'
+ run_sync(['./' + autogen_script], env=env)
+ else:
+ log("Using existing 'configure' script")
+
+ builddir = '_build'
+
+ use_builddir = True
+ doesnot_support_builddir = self._has_buildapi_configure_variable('no-builddir')
+ if doesnot_support_builddir:
+ log("Found no-builddir Build API variable; copying source tree to _build")
+ if os.path.isdir('_build'):
+ shutil.rmtree('_build')
+ shutil.copytree('.', '_build', symlinks=True,
+ ignore=shutil.ignore_patterns('_build'))
+ use_builddir = False
+
+ if use_builddir:
+ log("Using build directory %r" % (builddir, ))
+ if not os.path.isdir(builddir):
+ os.mkdir(builddir)
+
+ if use_builddir:
+ args = ['../configure']
+ else:
+ args = ['./configure']
+ args.extend(self.configargs)
+ run_sync(args, cwd=builddir)
+
+ makefile_path = None
+ for name in ['Makefile', 'makefile', 'GNUmakefile']:
+ makefile_path = os.path.join(builddir, name)
+ if os.path.exists(makefile_path):
+ break
+ if makefile_path is None:
+ fatal("No Makefile found")
+
+ args = list(self.makeargs)
+ user_specified_jobs = False
+ for arg in args:
+ if arg == '-j':
+ user_specified_jobs = True
+
+ if not user_specified_jobs:
+ has_notparallel = False
+ for line in open(makefile_path):
+ if line.startswith('.NOTPARALLEL'):
+ has_notparallel = True
+ log("Found .NOTPARALLEL")
+
+ if not has_notparallel:
+ log("Didn't find NOTPARALLEL, using parallel make by default")
+ args.extend(self.default_buildapi_jobs)
+
+ run_sync(args, cwd=builddir)
+
+ tempdir = tempfile.mkdtemp(prefix='ostbuild-destdir-%s' % (self.metadata['name'].replace('/', '_'), ))
+ self.tempfiles.append(tempdir)
+ args = ['make', 'install', 'DESTDIR=' + tempdir]
+ run_sync(args, cwd=builddir)
+
+ runtime_path = os.path.join(self.ostbuild_resultdir, 'runtime')
+ devel_path = os.path.join(self.ostbuild_resultdir, 'devel')
+ doc_path = os.path.join(self.ostbuild_resultdir, 'doc')
+ for artifact_type in ['runtime', 'devel', 'doc']:
+ resultdir = os.path.join(self.ostbuild_resultdir, artifact_type)
+ if os.path.isdir(resultdir):
+ shutil.rmtree(resultdir)
+ os.makedirs(resultdir)
+
+ # Remove /var from the install - components are required to
+ # auto-create these directories on demand.
+ varpath = os.path.join(tempdir, 'var')
+ if os.path.isdir(varpath):
+ shutil.rmtree(varpath)
+
+ # Move symbolic links for shared libraries as well
+ # as static libraries. And delete all .la files.
+ for libdirname in ['lib', 'usr/lib']:
+ path = os.path.join(tempdir, libdirname)
+ if not os.path.isdir(path):
+ continue
+ for filename in os.listdir(path):
+ subpath = os.path.join(path, filename)
+ if filename.endswith('.la'):
+ os.unlink(subpath)
+ continue
+ if not ((filename.endswith('.so')
+ and os.path.islink(filename))
+ or filename.endswith('.a')):
+ continue
+ dest = os.path.join(devel_path, libdirname, filename)
+ self._install_and_unlink(subpath, dest)
+
+ for dirname in _DEVEL_DIRS:
+ dirpath = os.path.join(tempdir, dirname)
+ if os.path.isdir(dirpath):
+ dest = os.path.join(devel_path, dirname)
+ self._install_and_unlink(dirpath, dest)
+
+ for dirname in _DOC_DIRS:
+ dirpath = os.path.join(tempdir, dirname)
+ if os.path.isdir(dirpath):
+ dest = os.path.join(doc_path, dirname)
+ self._install_and_unlink(dirpath, dest)
+
+ for filename in os.listdir(tempdir):
+ src_path = os.path.join(tempdir, filename)
+ dest_path = os.path.join(runtime_path, filename)
+ self._install_and_unlink(src_path, dest_path)
+
+ for tmpname in self.tempfiles:
+ assert os.path.isabs(tmpname)
+ if os.path.isdir(tmpname):
+ shutil.rmtree(tmpname)
+ else:
+ try:
+ os.unlink(tmpname)
+ except OSError, e:
+ pass
+
+ endtime = time.time()
+
+ log("Compilation succeeded; %d seconds elapsed" % (int(endtime - starttime),))
+ log("Results placed in %s" % (self.ostbuild_resultdir, ))
+
+ def _install_and_unlink(self, src, dest):
+ statsrc = os.lstat(src)
+ dirname = os.path.dirname(dest)
+ if not os.path.isdir(dirname):
+ os.makedirs(dirname)
+
+ if stat.S_ISDIR(statsrc.st_mode):
+ if not os.path.isdir(dest):
+ os.mkdir(dest)
+ for filename in os.listdir(src):
+ src_child = os.path.join(src, filename)
+ dest_child = os.path.join(dest, filename)
+
+ self._install_and_unlink(src_child, dest_child)
+ os.rmdir(src)
+ else:
+ try:
+ os.rename(src, dest)
+ except OSError, e:
+ if stat.S_ISLNK(statsrc.st_mode):
+ linkto = os.readlink(src)
+ os.symlink(linkto, dest)
+ else:
+ shutil.copy2(src, dest)
+ os.unlink(src)
+
+builtins.register(OstbuildCompileOne)
diff --git a/ostbuild/pyostbuild/builtin_deploy_qemu.py b/ostbuild/pyostbuild/builtin_deploy_qemu.py
new file mode 100755
index 0000000..f06b669
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_deploy_qemu.py
@@ -0,0 +1,65 @@
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from . import ostbuildrc
+from . import privileged_subproc
+
+class OstbuildDeployQemu(builtins.Builtin):
+ name = "deploy-qemu"
+ short_description = "Extract data from shadow repository to qemu"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--snapshot')
+ parser.add_argument('targets', nargs='*')
+
+ args = parser.parse_args(argv)
+ self.args = args
+
+ self.parse_config()
+ self.parse_snapshot(args.prefix, args.snapshot)
+
+ if len(args.targets) > 0:
+ targets = args.targets
+ else:
+ targets = []
+ prefix = self.snapshot['prefix']
+ for target_component_type in ['runtime', 'devel']:
+ for architecture in self.snapshot['architectures']:
+ name = '%s-%s-%s' % (prefix, architecture, target_component_type)
+ targets.append(name)
+
+ helper = privileged_subproc.PrivilegedSubprocess()
+ shadow_path = os.path.join(self.workdir, 'shadow-repo')
+ child_args = ['ostbuild', 'privhelper-deploy-qemu', shadow_path]
+ child_args.extend(targets)
+ helper.spawn_sync(child_args)
+
+builtins.register(OstbuildDeployQemu)
diff --git a/ostbuild/pyostbuild/builtin_deploy_root.py b/ostbuild/pyostbuild/builtin_deploy_root.py
new file mode 100755
index 0000000..f56d8f8
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_deploy_root.py
@@ -0,0 +1,67 @@
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from . import ostbuildrc
+from . import privileged_subproc
+
+class OstbuildDeployRoot(builtins.Builtin):
+ name = "deploy-root"
+ short_description = "Extract data from shadow repository to system repository"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--snapshot')
+ parser.add_argument('targets', nargs='*')
+
+ args = parser.parse_args(argv)
+ self.args = args
+
+ self.parse_config()
+ self.parse_snapshot(args.prefix, args.snapshot)
+
+ if len(args.targets) > 0:
+ targets = args.targets
+ else:
+ targets = []
+ prefix = self.snapshot['prefix']
+ for target_component_type in ['runtime', 'devel']:
+ for architecture in self.snapshot['architectures']:
+ name = '%s-%s-%s' % (prefix, architecture, target_component_type)
+ targets.append(name)
+
+ helper = privileged_subproc.PrivilegedSubprocess()
+ sys_repo = os.path.join(self.ostree_dir, 'repo')
+ shadow_path = os.path.join(self.workdir, 'shadow-repo')
+ child_args = ['ostree', '--repo=' + sys_repo,
+ 'pull-local', shadow_path]
+ child_args.extend(['trees/' + x for x in targets])
+ helper.spawn_sync(child_args)
+
+builtins.register(OstbuildDeployRoot)
diff --git a/ostbuild/pyostbuild/builtin_git_mirror.py b/ostbuild/pyostbuild/builtin_git_mirror.py
new file mode 100755
index 0000000..6ecdafa
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_git_mirror.py
@@ -0,0 +1,109 @@
+# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil,time
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from . import vcs
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildGitMirror(builtins.Builtin):
+ name = "git-mirror"
+ short_description = "Update internal git mirror for one or more components"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--manifest')
+ parser.add_argument('--src-snapshot')
+ parser.add_argument('--start-at',
+ help="Start at the given component")
+ parser.add_argument('--fetch-skip-secs', type=int, default=0,
+ help="Don't perform a fetch if we have done so in the last N seconds")
+ parser.add_argument('--fetch', action='store_true',
+ help="Also do a git fetch for components")
+ parser.add_argument('-k', '--keep-going', action='store_true',
+ help="Don't exit on fetch failures")
+ parser.add_argument('components', nargs='*')
+
+ args = parser.parse_args(argv)
+ self.parse_config()
+ if args.manifest:
+ self.snapshot = json.load(open(args.manifest))
+ components = map(lambda x: buildutil.resolve_component_meta(self.snapshot, x), self.snapshot['components'])
+ self.snapshot['components'] = components
+ self.snapshot['patches'] = buildutil.resolve_component_meta(self.snapshot, self.snapshot['patches'])
+ else:
+ self.parse_snapshot(args.prefix, args.src_snapshot)
+
+ if len(args.components) == 0:
+ components = []
+ for component in self.snapshot['components']:
+ components.append(component['name'])
+ if 'patches' in self.snapshot:
+ components.append(self.snapshot['patches']['name'])
+ if args.start_at:
+ idx = components.index(args.start_at)
+ components = components[idx:]
+ else:
+ components = args.components
+
+ for name in components:
+ component = self.get_component(name)
+ src = component['src']
+ (keytype, uri) = vcs.parse_src_key(src)
+ branch = component.get('branch')
+ tag = component.get('tag')
+ branch_or_tag = branch or tag
+ mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag)
+
+ if not args.fetch:
+ continue
+
+ if tag is not None:
+ log("Skipping fetch for %s at tag %s" % (name, tag))
+ continue
+
+ curtime = time.time()
+ if args.fetch_skip_secs > 0:
+ last_fetch_path = vcs.get_lastfetch_path(self.mirrordir, keytype, uri, branch_or_tag)
+ try:
+ stbuf = os.stat(last_fetch_path)
+ except OSError, e:
+ stbuf = None
+ if stbuf is not None:
+ mtime = stbuf.st_mtime
+ delta = curtime - mtime
+ if delta < args.fetch_skip_secs:
+ log("Skipping fetch for %s updated in last %d seconds" % (name, delta))
+ continue
+
+ log("Running git fetch for %s" % (name, ))
+ vcs.fetch(self.mirrordir, keytype, uri, branch_or_tag, keep_going=args.keep_going)
+
+builtins.register(OstbuildGitMirror)
diff --git a/ostbuild/pyostbuild/builtin_import_tree.py b/ostbuild/pyostbuild/builtin_import_tree.py
new file mode 100755
index 0000000..9fed209
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_import_tree.py
@@ -0,0 +1,93 @@
+# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildImportTree(builtins.Builtin):
+ name = "import-tree"
+ short_description = "Extract source data from tree into new prefix"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--tree')
+ parser.add_argument('--prefix')
+
+ args = parser.parse_args(argv)
+ self.parse_config()
+ self.parse_snapshot_from_current()
+
+ log("Loading source from tree %r" % (self.snapshot_path, ))
+
+ related_objects = run_sync_get_output(['ostree', '--repo='+ self.repo,
+ 'show', '--print-related',
+ self.active_branch_checksum])
+ ref_to_revision = {}
+ for line in StringIO(related_objects):
+ line = line.strip()
+ (ref, revision) = line.split(' ', 1)
+ ref_to_revision[ref] = revision
+
+ if args.prefix:
+ target_prefix = args.prefix
+ else:
+ target_prefix = self.snapshot['prefix']
+
+ (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-import-tree-')
+ f = os.fdopen(fd, 'w')
+ for (ref, rev) in ref_to_revision.iteritems():
+ if ref.startswith('components/'):
+ ref = ref[len('components/'):]
+ (prefix, subref) = ref.split('/', 1)
+ newref = 'components/%s/%s' % (target_prefix, subref)
+ elif ref.startswith('bases/'):
+ # hack
+ base_key = '/' + self.snapshot['prefix'] + '-'
+ replace_key = '/' + target_prefix + '-'
+ newref = ref.replace(base_key, replace_key)
+ else:
+ fatal("Unhandled ref %r; expected components/ or bases/" % (ref, ))
+
+ f.write('%s %s\n' % (newref, rev))
+ f.close()
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'write-refs'], stdin=open(tmppath))
+
+ self.snapshot['prefix'] = target_prefix
+
+ run_sync(['ostbuild', 'prefix', target_prefix])
+ self.prefix = target_prefix
+
+ db = self.get_src_snapshot_db()
+ path = db.store(self.snapshot)
+ log("Source snapshot: %s" % (path, ))
+
+builtins.register(OstbuildImportTree)
diff --git a/ostbuild/pyostbuild/builtin_init.py b/ostbuild/pyostbuild/builtin_init.py
new file mode 100755
index 0000000..de45eb4
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_init.py
@@ -0,0 +1,58 @@
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+from StringIO import StringIO
+import json
+import select,time
+import argparse
+
+from . import builtins
+from . import ostbuildrc
+from .ostbuildlog import log, fatal
+from . import fileutil
+from .subprocess_helpers import run_sync, run_sync_get_output
+
+class OstbuildInit(builtins.Builtin):
+ name = "init"
+ short_description = "Initialize working state"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+
+ args = parser.parse_args(argv)
+
+ mirrordir = os.path.expanduser(ostbuildrc.get_key('mirrordir'))
+ fileutil.ensure_dir(mirrordir)
+ workdir = os.path.expanduser(ostbuildrc.get_key('workdir'))
+ fileutil.ensure_dir(workdir)
+
+ self.parse_config()
+
+ path = os.path.join(self.workdir, 'shadow-repo')
+ fileutil.ensure_dir(path)
+ if os.path.isdir(os.path.join(path, 'objects')):
+ log("note: shadow repository '%s' already exists" % (path, ))
+ else:
+ run_sync(['ostree', '--repo=' + path, 'init', '--archive'])
+ run_sync(['ostree', '--repo=' + path, 'config', 'set', 'core.parent', '/ostree/repo'])
+ log("Created shadow repository: %s" % (path, ))
+
+builtins.register(OstbuildInit)
diff --git a/ostbuild/pyostbuild/builtin_prefix.py b/ostbuild/pyostbuild/builtin_prefix.py
new file mode 100755
index 0000000..4ffa981
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_prefix.py
@@ -0,0 +1,73 @@
+# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildPrefix(builtins.Builtin):
+ name = "prefix"
+ short_description = "Display or modify \"prefix\" (build target)"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def _set_prefix(self, prefix):
+ f = open(self.path, 'w')
+ f.write(prefix)
+ f.write('\n')
+ f.close()
+ log("Prefix is now %r" % (prefix, ))
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('-a', '--active', action='store_true')
+ parser.add_argument('prefix', nargs='?', default=None)
+
+ args = parser.parse_args(argv)
+
+ self.path = os.path.expanduser('~/.config/ostbuild-prefix')
+ if args.prefix is None and not args.active:
+ if os.path.exists(self.path):
+ f = open(self.path)
+ print "%s" % (f.read().strip(), )
+ f.close()
+ else:
+ log("No currently active prefix")
+ elif args.prefix is not None and args.active:
+ fatal("Can't specify -a with prefix")
+ elif args.prefix is not None:
+ self._set_prefix(args.prefix)
+ else:
+ assert args.active
+
+ self.parse_active_branch()
+
+ active_prefix = self.active_branch_contents['prefix']
+
+ self._set_prefix(active_prefix)
+
+builtins.register(OstbuildPrefix)
diff --git a/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py b/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py
new file mode 100755
index 0000000..9380c46
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py
@@ -0,0 +1,115 @@
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync
+from . import ostbuildrc
+from . import fileutil
+
+class OstbuildPrivhelperDeployQemu(builtins.Builtin):
+ name = "privhelper-deploy-qemu"
+ short_description = "Helper for deploy-qemu"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def _create_qemu_disk(self):
+ log("%s not found, creating" % (self.qemu_path, ))
+ success = False
+ tmppath = self.qemu_path + '.tmp'
+ if os.path.exists(tmppath):
+ os.unlink(tmppath)
+ subprocess.check_call(['qemu-img', 'create', tmppath, '6G'])
+ subprocess.check_call(['mkfs.ext4', '-q', '-F', tmppath])
+
+ subprocess.call(['umount', self.mountpoint], stderr=open('/dev/null', 'w'))
+ try:
+ subprocess.check_call(['mount', '-o', 'loop', tmppath, self.mountpoint])
+
+ for topdir in ['mnt', 'sys', 'root', 'home', 'opt', 'tmp', 'run',
+ 'ostree']:
+ path = os.path.join(self.mountpoint, topdir)
+ fileutil.ensure_dir(path)
+ os.chmod(os.path.join(self.mountpoint, 'root'), 0700)
+ os.chmod(os.path.join(self.mountpoint, 'tmp'), 01777)
+
+ varpath = os.path.join(self.mountpoint, 'ostree', 'var')
+ fileutil.ensure_dir(varpath)
+ modulespath = os.path.join(self.mountpoint, 'ostree', 'modules')
+ fileutil.ensure_dir(modulespath)
+
+ repo_path = os.path.join(self.mountpoint, 'ostree', 'repo')
+ fileutil.ensure_dir(repo_path)
+ subprocess.check_call(['ostree', '--repo=' + repo_path, 'init'])
+ success = True
+ finally:
+ subprocess.call(['umount', self.mountpoint])
+ if success:
+ os.rename(tmppath, self.qemu_path)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--rootdir',
+ help="Directory containing OSTree data (default: /ostree)")
+ parser.add_argument('srcrepo')
+ parser.add_argument('targets', nargs='+')
+
+ args = parser.parse_args(argv)
+
+ if os.geteuid() != 0:
+ fatal("This helper can only be run as root")
+
+ if args.rootdir:
+ self.ostree_dir = args.rootdir
+ else:
+ self.ostree_dir = self.find_ostree_dir()
+ self.qemu_path = os.path.join(self.ostree_dir, "ostree-qemu.img")
+
+ self.mountpoint = os.path.join(self.ostree_dir, 'ostree-qemu-mnt')
+ fileutil.ensure_dir(self.mountpoint)
+
+ if not os.path.exists(self.qemu_path):
+ self._create_qemu_disk()
+
+ subprocess.call(['umount', self.mountpoint], stderr=open('/dev/null', 'w'))
+ repo_path = os.path.join(self.mountpoint, 'ostree', 'repo')
+ try:
+ subprocess.check_call(['mount', '-o', 'loop', self.qemu_path, self.mountpoint])
+ child_args = ['ostree', '--repo=' + repo_path, 'pull-local', args.srcrepo]
+ child_args.extend(['trees/' + x for x in args.targets])
+ run_sync(child_args)
+
+ first_target = args.targets[0]
+ for target in args.targets:
+ run_sync(['ostree', '--repo=' + repo_path, 'checkout', '--atomic-retarget', 'trees/'+ target, target],
+ cwd=os.path.join(self.mountpoint, 'ostree'))
+ current_link_path = os.path.join(self.mountpoint, 'ostree', 'current')
+ os.symlink(first_target, current_link_path + '.tmp')
+ os.rename(current_link_path + '.tmp', current_link_path)
+ finally:
+ subprocess.call(['umount', self.mountpoint])
+
+
+builtins.register(OstbuildPrivhelperDeployQemu)
diff --git a/ostbuild/pyostbuild/builtin_privhelper_run_qemu.py b/ostbuild/pyostbuild/builtin_privhelper_run_qemu.py
new file mode 100755
index 0000000..a3e556e
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_privhelper_run_qemu.py
@@ -0,0 +1,63 @@
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync
+from . import ostbuildrc
+from . import fileutil
+
+class OstbuildPrivhelperRunQemu(builtins.Builtin):
+ name = "privhelper-run-qemu"
+ short_description = "Helper for run-qemu"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('target')
+
+ args = parser.parse_args(argv)
+
+ if os.geteuid() != 0:
+ fatal("This helper can only be run as root")
+
+ self.ostree_dir = self.find_ostree_dir()
+ self.qemu_path = os.path.join(self.ostree_dir, "ostree-qemu.img")
+
+ release = os.uname()[2]
+
+ qemu = 'qemu-kvm'
+ kernel = '/boot/vmlinuz-%s' % (release, )
+ initramfs = '/boot/initramfs-ostree-%s.img' % (release, )
+ memory = '512M'
+ extra_args = 'root=/dev/sda rd.pymouth=0 ostree=%s' % (args.target, )
+
+ args = [qemu, '-kernel', kernel, '-initrd', initramfs,
+ '-hda', self.qemu_path, '-m', memory, '-append', extra_args]
+ log("Running: %s" % (subprocess.list2cmdline(args), ))
+ os.execvp(qemu, args)
+
+builtins.register(OstbuildPrivhelperRunQemu)
diff --git a/ostbuild/pyostbuild/builtin_resolve.py b/ostbuild/pyostbuild/builtin_resolve.py
new file mode 100755
index 0000000..5109fd1
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_resolve.py
@@ -0,0 +1,106 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import copy
+import argparse
+import json
+import time
+import urlparse
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import ostbuildrc
+from . import vcs
+from . import jsondb
+from . import buildutil
+from . import kvfile
+from . import odict
+
+class OstbuildResolve(builtins.Builtin):
+ name = "resolve"
+ short_description = "Expand git revisions in source to exact targets"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--manifest', required=True,
+ help="Path to manifest file")
+ parser.add_argument('--fetch-patches', action='store_true',
+ help="Git fetch the patches")
+ parser.add_argument('--fetch', action='store_true',
+ help="Also perform a git fetch")
+ parser.add_argument('components', nargs='*',
+ help="List of component names to git fetch")
+
+ args = parser.parse_args(argv)
+ self.args = args
+
+ if len(args.components) > 0 and not args.fetch:
+ fatal("Can't specify components without --fetch")
+
+ self.parse_config()
+
+ self.snapshot = json.load(open(args.manifest))
+ self.prefix = self.snapshot['prefix']
+
+ components = map(lambda x: buildutil.resolve_component_meta(self.snapshot, x), self.snapshot['components'])
+ self.snapshot['components'] = components
+
+ unique_component_names = set()
+ for component in components:
+ name = component['name']
+
+ if name in unique_component_names:
+ fatal("Duplicate component name '%s'" % (name, ))
+ unique_component_names.add(name)
+
+ global_patches_meta = buildutil.resolve_component_meta(self.snapshot, self.snapshot['patches'])
+ self.snapshot['patches'] = global_patches_meta
+ (keytype, uri) = vcs.parse_src_key(global_patches_meta['src'])
+ mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, global_patches_meta['branch'])
+ if args.fetch_patches:
+ run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False)
+
+ git_mirror_args = ['ostbuild', 'git-mirror', '--manifest=' + args.manifest]
+ if args.fetch:
+ git_mirror_args.append('--fetch')
+ git_mirror_args.extend(args.components)
+ run_sync(git_mirror_args)
+
+ patch_revision = buildutil.get_git_version_describe(mirrordir, global_patches_meta['branch'])
+ global_patches_meta['revision'] = patch_revision
+
+ for component in components:
+ src = component['src']
+ (keytype, uri) = vcs.parse_src_key(src)
+ branch = component.get('branch')
+ tag = component.get('tag')
+ branch_or_tag = branch or tag
+ mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag)
+ revision = buildutil.get_git_version_describe(mirrordir, branch_or_tag)
+ component['revision'] = revision
+
+ src_db = self.get_src_snapshot_db()
+ path = src_db.store(self.snapshot)
+ log("Source snapshot: %s" % (path, ))
+
+builtins.register(OstbuildResolve)
diff --git a/ostbuild/pyostbuild/builtin_run_qemu.py b/ostbuild/pyostbuild/builtin_run_qemu.py
new file mode 100755
index 0000000..ca33f4f
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_run_qemu.py
@@ -0,0 +1,49 @@
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from . import ostbuildrc
+from . import privileged_subproc
+
+class OstbuildRunQemu(builtins.Builtin):
+ name = "run-qemu"
+ short_description = "Run QEMU image"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('target')
+
+ args = parser.parse_args(argv)
+
+ self.parse_config()
+
+ helper = privileged_subproc.PrivilegedSubprocess()
+ child_args = ['ostbuild', 'privhelper-run-qemu', args.target]
+ helper.spawn_sync(child_args)
+
+builtins.register(OstbuildRunQemu)
diff --git a/ostbuild/pyostbuild/builtin_source_diff.py b/ostbuild/pyostbuild/builtin_source_diff.py
new file mode 100755
index 0000000..0e3ff07
--- /dev/null
+++ b/ostbuild/pyostbuild/builtin_source_diff.py
@@ -0,0 +1,162 @@
+# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from . import vcs
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildSourceDiff(builtins.Builtin):
+ name = "source-diff"
+ short_description = "Show differences in source code between builds"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def _diff(self, name, mirrordir, from_revision, to_revision,
+ diffstat=False):
+ diff_replace_re = re.compile(' [ab]')
+
+ env = dict(os.environ)
+ env['LANG'] = 'C'
+
+ spacename = ' ' + name
+
+ sys.stdout.write('diff of %s revision range %s..%s\n' % (name, from_revision, to_revision))
+ sys.stdout.flush()
+
+ diff_proc = subprocess.Popen(['git', 'diff', from_revision, to_revision],
+ env=env, cwd=mirrordir, stdout=subprocess.PIPE)
+ if diffstat:
+ diffstat_proc = subprocess.Popen(['diffstat', '-p0'],
+ stdin=subprocess.PIPE,
+ stdout=sys.stdout)
+ diff_pipe = diffstat_proc.stdin
+ else:
+ diffstat_proc = None
+ diff_pipe = sys.stdout
+ for line in diff_proc.stdout:
+ if (line.startswith('diff --git ')
+ or line.startswith('--- a/')
+ or line.startswith('+++ b/')
+ or line.startswith('Binary files /dev/null and b/')):
+ line = diff_replace_re.sub(spacename, line)
+ diff_pipe.write(line)
+ else:
+ diff_pipe.write(line)
+ diff_proc.wait()
+ if diffstat_proc is not None:
+ diffstat_proc.stdin.close()
+ diffstat_proc.wait()
+
+ def _log(self, opts, name, mirrordir, from_revision, to_revision):
+ env = dict(os.environ)
+ env['LANG'] = 'C'
+
+ spacename = ' ' + name
+
+ args = ['git', 'log']
+ args.extend(opts)
+ args.append(from_revision + '...' + to_revision)
+ proc = subprocess.Popen(args, env=env, cwd=mirrordir, stdout=subprocess.PIPE)
+ for line in proc.stdout:
+ sys.stdout.write(line)
+ proc.wait()
+
+ def _snapshot_from_rev(self, rev):
+ self.init_repo()
+ text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'cat', rev, '/contents.json'],
+ log_initiation=False)
+ return json.loads(text)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--log', action='store_true')
+ parser.add_argument('--logp', action='store_true')
+ parser.add_argument('--diffstat', action='store_true')
+ parser.add_argument('--rev-from')
+ parser.add_argument('--rev-to')
+ parser.add_argument('--snapshot-from')
+ parser.add_argument('--snapshot-to')
+
+ args = parser.parse_args(argv)
+ self.parse_config()
+
+ to_snap = None
+ from_snap = None
+
+ if args.rev_to:
+ to_snap = self._snapshot_from_rev(args.rev_to)
+ if args.rev_from:
+ from_snap = self._snapshot_from_rev(args.rev_from)
+ if args.snapshot_from:
+ from_snap = json.load(open(args.snapshot_from))
+ if args.snapshot_to:
+ to_snap = json.load(open(args.snapshot_to))
+
+ if to_snap is None:
+ fatal("One of --rev-to/--snapshot-to must be given")
+ if from_snap is None:
+ if args.rev_to:
+ from_snap = self._snapshot_from_rev(args.rev_to + '^')
+ else:
+ fatal("One of --rev-from/--snapshot-from must be given")
+
+ for from_component in from_snap['components']:
+ name = from_component['name']
+ src = from_component['src']
+ (keytype, uri) = vcs.parse_src_key(src)
+ if keytype == 'local':
+ log("Component %r has local URI" % (name, ))
+ continue
+ branch_or_tag = from_component.get('branch') or from_component.get('tag')
+ mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag)
+
+ to_component = self.find_component_in_snapshot(name, to_snap)
+ if to_component is None:
+ log("DELETED COMPONENT: %s" % (name, ))
+ continue
+
+ from_revision = from_component.get('revision')
+ to_revision = to_component.get('revision')
+ if from_revision is None:
+ log("From component %s missing revision" % (name, ))
+ continue
+ if to_revision is None:
+ log("From component %s missing revision" % (name, ))
+ continue
+
+ if from_revision != to_revision:
+ if args.log:
+ self._log([], name, mirrordir, from_revision, to_revision)
+ elif args.logp:
+ self._log(['-p'], name, mirrordir, from_revision, to_revision)
+ else:
+ self._diff(name, mirrordir, from_revision, to_revision,
+ diffstat=args.diffstat)
+
+builtins.register(OstbuildSourceDiff)
diff --git a/ostbuild/pyostbuild/builtins.py b/ostbuild/pyostbuild/builtins.py
new file mode 100755
index 0000000..b5fd104
--- /dev/null
+++ b/ostbuild/pyostbuild/builtins.py
@@ -0,0 +1,237 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import sys
+import stat
+import argparse
+import json
+
+from . import ostbuildrc
+from . import fileutil
+from . import jsondb
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+
+_all_builtins = {}
+
+class Builtin(object):
+ name = None
+ short_description = None
+
+ def __init__(self):
+ self._meta_cache = {}
+ self.prefix = None
+ self.manifest = None
+ self.snapshot = None
+ self.bin_snapshot = None
+ self.repo = None
+ self.ostree_dir = self.find_ostree_dir()
+ (self.active_branch, self.active_branch_checksum) = self._find_active_branch()
+ self._src_snapshots = None
+ self._bin_snapshots = None
+
+ def find_ostree_dir(self):
+ for path in ['/ostree', '/sysroot/ostree']:
+ if os.path.isdir(path):
+ return path
+ return None
+
+ def _find_active_branch(self):
+ if self.ostree_dir is None:
+ return (None, None)
+ current_path = os.path.join(self.ostree_dir, 'current')
+ while True:
+ try:
+ target = os.path.join(self.ostree_dir, current_path)
+ stbuf = os.lstat(target)
+ except OSError, e:
+ current_path = None
+ break
+ if not stat.S_ISLNK(stbuf.st_mode):
+ break
+ current_path = os.readlink(target)
+ if current_path is not None:
+ basename = os.path.basename(current_path)
+ return basename.rsplit('-', 1)
+ else:
+ return (None, None)
+
+ def get_component_from_cwd(self):
+ cwd = os.getcwd()
+ parent = os.path.dirname(cwd)
+ parentparent = os.path.dirname(parent)
+ return '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
+
+ def parse_config(self):
+ self.ostbuildrc = ostbuildrc
+
+ self.mirrordir = os.path.expanduser(ostbuildrc.get_key('mirrordir'))
+ if not os.path.isdir(self.mirrordir):
+ fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, ))
+ self.workdir = os.path.expanduser(ostbuildrc.get_key('workdir'))
+ if not os.path.isdir(self.workdir):
+ fatal("Specified workdir '%s' is not a directory" % (self.workdir, ))
+
+ self.snapshot_dir = os.path.join(self.workdir, 'snapshots')
+ self.patchdir = os.path.join(self.workdir, 'patches')
+
+ def get_component_snapshot(self, name):
+ found = False
+ for content in self.active_branch_contents['contents']:
+ if content['name'] == name:
+ found = True
+ break
+ if not found:
+ fatal("Unknown component '%s'" % (name, ))
+ return content
+
+ def get_component_meta_from_revision(self, revision):
+ text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'cat', revision,
+ '/_ostbuild-meta.json'])
+ return json.loads(text)
+
+ def expand_component(self, component):
+ meta = dict(component)
+ global_patchmeta = self.snapshot.get('patches')
+ if global_patchmeta is not None:
+ component_patch_files = component.get('patches', [])
+ if len(component_patch_files) > 0:
+ patches = dict(global_patchmeta)
+ patches['files'] = component_patch_files
+ meta['patches'] = patches
+ config_opts = list(self.snapshot.get('config-opts', []))
+ config_opts.extend(component.get('config-opts', []))
+ meta['config-opts'] = config_opts
+ return meta
+
+ def find_component_in_snapshot(self, name, snapshot):
+ for component in snapshot['components']:
+ if component['name'] == name:
+ return component
+ return None
+
+ def get_component(self, name, in_snapshot=None):
+ if in_snapshot is None:
+ assert self.snapshot is not None
+ target_snapshot = self.snapshot
+ else:
+ target_snapshot = in_snapshot
+ component = self.find_component_in_snapshot(name, target_snapshot)
+ if (component is None and
+ 'patches' in self.snapshot and
+ self.snapshot['patches']['name'] == name):
+ return self.snapshot['patches']
+ if component is None:
+ fatal("Couldn't find component '%s' in manifest" % (name, ))
+ return component
+
+ def get_expanded_component(self, name):
+ return self.expand_component(self.get_component(name))
+
+ def get_prefix(self):
+ if self.prefix is None:
+ path = os.path.expanduser('~/.config/ostbuild-prefix')
+ if not os.path.exists(path):
+ fatal("No prefix set; use \"ostbuild prefix\" to set one")
+ f = open(path)
+ self.prefix = f.read().strip()
+ f.close()
+ return self.prefix
+
+ def create_db(self, dbsuffix, prefix=None):
+ if prefix is None:
+ target_prefix = self.get_prefix()
+ else:
+ target_prefix = prefix
+ name = '%s-%s' % (target_prefix, dbsuffix)
+ fileutil.ensure_dir(self.snapshot_dir)
+ return jsondb.JsonDB(self.snapshot_dir, prefix=name)
+
+ def get_src_snapshot_db(self):
+ if self._src_snapshots is None:
+ self._src_snapshots = self.create_db('src-snapshot')
+ return self._src_snapshots
+
+ def get_bin_snapshot_db(self):
+ if self._bin_snapshots is None:
+ self._bin_snapshots = self.create_db('bin-snapshot')
+ return self._bin_snapshots
+
+ def init_repo(self):
+ if self.repo is not None:
+ return self.repo
+ repo = ostbuildrc.get_key('repo', default=None)
+ if repo is not None:
+ self.repo = repo
+ else:
+ shadow_path = os.path.join(self.workdir, 'shadow-repo')
+ if os.path.isdir(shadow_path):
+ self.repo = shadow_path
+ else:
+ fatal("No repository configured, and shadow-repo not found. Use \"ostbuild init\" to make one")
+
+ def parse_prefix(self, prefix):
+ if prefix is not None:
+ self.prefix = prefix
+
+ def parse_snapshot(self, prefix, path):
+ self.parse_prefix(prefix)
+ self.init_repo()
+ if path is None:
+ latest_path = self.get_src_snapshot_db().get_latest_path()
+ if latest_path is None:
+ raise Exception("No source snapshot found for prefix %r" % (self.prefix, ))
+ self.snapshot_path = latest_path
+ else:
+ self.snapshot_path = path
+ self.snapshot = json.load(open(self.snapshot_path))
+ key = '00ostbuild-manifest-version'
+ src_ver = self.snapshot[key]
+ if src_ver != 0:
+ fatal("Unhandled %s version \"%d\", expected 0" % (key, src_ver, ))
+ if self.prefix is None:
+ self.prefix = self.snapshot['prefix']
+
+ def parse_snapshot_from_current(self):
+ if self.ostree_dir is None:
+ fatal("/ostree directory not found")
+ repo_path = os.path.join(self.ostree_dir, 'repo')
+ if not os.path.isdir(repo_path):
+ fatal("Repository '%s' doesn't exist" % (repo_path, ))
+ if self.active_branch is None:
+ fatal("No \"current\" link found")
+ tree_path = os.path.join(self.ostree_dir, self.active_branch)
+ self.parse_snapshot(None, os.path.join(tree_path, 'contents.json'))
+
+ def execute(self, args):
+ raise NotImplementedError()
+
+def register(builtin):
+ _all_builtins[builtin.name] = builtin
+
+def get(name):
+ builtin = _all_builtins.get(name)
+ if builtin is not None:
+ return builtin()
+ return None
+
+def get_all():
+ return sorted(_all_builtins.itervalues(), lambda a, b: cmp(a.name, b.name))
diff --git a/ostbuild/pyostbuild/filemonitor.py b/ostbuild/pyostbuild/filemonitor.py
new file mode 100644
index 0000000..0eec07b
--- /dev/null
+++ b/ostbuild/pyostbuild/filemonitor.py
@@ -0,0 +1,64 @@
+#
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import stat
+
+from . import mainloop
+
+_global_filemon = None
+
+class FileMonitor(object):
+ def __init__(self):
+ self._paths = {}
+ self._path_modtimes = {}
+ self._timeout = 1000
+ self._timeout_installed = False
+ self._loop = mainloop.Mainloop.get(None)
+
+ @classmethod
+ def get(cls):
+ global _global_filemon
+ if _global_filemon is None:
+ _global_filemon = cls()
+ return _global_filemon
+
+ def _stat(self, path):
+ try:
+ st = os.stat(path)
+ return st[stat.ST_MTIME]
+ except OSError, e:
+ return None
+
+ def add(self, path, callback):
+ if path not in self._paths:
+ self._paths[path] = []
+ self._path_modtimes[path] = self._stat(path)
+ self._paths[path].append(callback)
+ if not self._timeout_installed:
+ self._timeout_installed = True
+ self._loop.timeout_add(self._timeout, self._check_files)
+
+ def _check_files(self):
+ for (path,callbacks) in self._paths.iteritems():
+ mtime = self._stat(path)
+ orig_mtime = self._path_modtimes[path]
+ if (mtime is not None) and (orig_mtime is None or (mtime > orig_mtime)):
+ self._path_modtimes[path] = mtime
+ for cb in callbacks:
+ cb()
diff --git a/ostbuild/pyostbuild/fileutil.py b/ostbuild/pyostbuild/fileutil.py
new file mode 100644
index 0000000..d9ae91f
--- /dev/null
+++ b/ostbuild/pyostbuild/fileutil.py
@@ -0,0 +1,26 @@
+#
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+
+def ensure_dir(path):
+ if not os.path.isdir(path):
+ os.makedirs(path)
+
+def ensure_parent_dir(path):
+ ensure_dir(os.path.dirname(path))
diff --git a/ostbuild/pyostbuild/jsondb.py b/ostbuild/pyostbuild/jsondb.py
new file mode 100644
index 0000000..810103d
--- /dev/null
+++ b/ostbuild/pyostbuild/jsondb.py
@@ -0,0 +1,112 @@
+#
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import stat
+import time
+import tempfile
+import re
+import shutil
+import hashlib
+import json
+
+class JsonDB(object):
+ def __init__(self, dirpath, prefix):
+ self._dirpath = dirpath
+ self._prefix = prefix
+ self._version_csum_re = re.compile(r'-(\d+)\.(\d+)-([0-9a-f]+).json$')
+
+ def _cmp_match_by_version(self, a, b):
+ # Note this is a reversed comparison; bigger is earlier
+ a_major = a[0]
+ a_minor = a[1]
+ b_major = b[0]
+ b_minor = b[1]
+
+ c = cmp(b_major, a_major)
+ if c == 0:
+ return cmp(b_minor, a_minor)
+ return 0
+
+ def _get_all(self):
+ result = []
+ for fname in os.listdir(self._dirpath):
+ if not (fname.startswith(self._prefix) and fname.endswith('.json')):
+ continue
+
+ path = os.path.join(self._dirpath, fname)
+ match = self._version_csum_re.search(fname)
+ if not match:
+ raise Exception("Invalid file '%s' in JsonDB; doesn't contain version+checksum",
+ path)
+ result.append((int(match.group(1)), int(match.group(2)), match.group(3), fname))
+ result.sort(self._cmp_match_by_version)
+ return result
+
+ def get_latest(self):
+ path = self.get_latest_path()
+ if path is None:
+ return None
+ return json.load(open(path))
+
+ def get_latest_path(self):
+ files = self._get_all()
+ if len(files) == 0:
+ return None
+ return os.path.join(self._dirpath, files[0][3])
+
+ def store(self, obj):
+ files = self._get_all()
+ if len(files) == 0:
+ latest = None
+ else:
+ latest = files[0]
+
+ current_time = time.gmtime()
+
+ (fd, tmppath) = tempfile.mkstemp(suffix='.tmp',
+ prefix='tmp-jsondb-', dir=self._dirpath)
+ os.close(fd)
+ f = open(tmppath, 'w')
+ json.dump(obj, f, indent=4, sort_keys=True)
+ f.close()
+
+ csum = hashlib.sha256()
+ f = open(tmppath)
+ buf = f.read(8192)
+ while buf != '':
+ csum.update(buf)
+ buf = f.read(8192)
+ f.close()
+ digest = csum.hexdigest()
+
+ if latest is not None:
+ if digest == latest[2]:
+ os.unlink(tmppath)
+ return latest[3]
+ latest_version = (latest[0], latest[1])
+ else:
+ latest_version = (current_time.tm_year, 0)
+ target_name = '%s-%d.%d-%s.json' % (self._prefix, current_time.tm_year,
+ latest_version[1] + 1, digest)
+ target_path = os.path.join(self._dirpath, target_name)
+ os.rename(tmppath, target_path)
+ return target_path
+
+
+
diff --git a/ostbuild/pyostbuild/kvfile.py b/ostbuild/pyostbuild/kvfile.py
new file mode 100755
index 0000000..075d4b0
--- /dev/null
+++ b/ostbuild/pyostbuild/kvfile.py
@@ -0,0 +1,23 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+def parse(stream):
+ ret = {}
+ for line in stream:
+ (k,v) = line.split('=', 1)
+ ret[k.strip()] = v.strip()
+ return ret
diff --git a/ostbuild/pyostbuild/main.py b/ostbuild/pyostbuild/main.py
new file mode 100755
index 0000000..0adccd6
--- /dev/null
+++ b/ostbuild/pyostbuild/main.py
@@ -0,0 +1,61 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import sys
+import argparse
+
+from . import builtins
+from . import builtin_build
+from . import builtin_checkout
+from . import builtin_chroot_compile_one
+from . import builtin_compile_one
+from . import builtin_deploy_root
+from . import builtin_deploy_qemu
+from . import builtin_git_mirror
+from . import builtin_import_tree
+from . import builtin_init
+from . import builtin_run_qemu
+from . import builtin_prefix
+from . import builtin_privhelper_deploy_qemu
+from . import builtin_privhelper_run_qemu
+from . import builtin_resolve
+from . import builtin_source_diff
+
+def usage(ecode):
+ print "Builtins:"
+ for builtin in builtins.get_all():
+ if builtin.name.startswith('privhelper'):
+ continue
+ print " %s - %s" % (builtin.name, builtin.short_description)
+ return ecode
+
+def main(args):
+ if len(args) < 1:
+ return usage(1)
+ elif args[0] in ('-h', '--help'):
+ return usage(0)
+ else:
+ builtin = builtins.get(args[0])
+ if builtin is None:
+ print "error: Unknown builtin '%s'" % (args[0], )
+ return usage(1)
+ return builtin.execute(args[1:])
+
+
diff --git a/ostbuild/pyostbuild/mainloop.py b/ostbuild/pyostbuild/mainloop.py
new file mode 100644
index 0000000..40a67be
--- /dev/null
+++ b/ostbuild/pyostbuild/mainloop.py
@@ -0,0 +1,96 @@
+#
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import sys
+import select
+import time
+
+class Mainloop(object):
+ DEFAULT = None
+ def __init__(self):
+ self._running = True
+ self.poll = None
+ self._timeouts = []
+ self._pid_watches = {}
+ self._fd_callbacks = {}
+
+ @classmethod
+ def get(cls, context):
+ if context is None:
+ if cls.DEFAULT is None:
+ cls.DEFAULT = cls()
+ return cls.DEFAULT
+ raise NotImplementedError("Unknown context %r" % (context, ))
+
+ def _ensure_poll(self):
+ if self.poll is None:
+ self.poll = select.poll()
+
+ def watch_fd(self, fd, callback):
+ self._ensure_poll()
+ self.poll.register(fd)
+ self._fd_callbacks[fd] = callback
+
+ def unwatch_fd(self, fd):
+ self.poll.unregister(fd)
+ del self._fd_callbacks[fd]
+
+ def watch_pid(self, pid, callback):
+ self._pid_watches[pid] = callback
+
+ def timeout_add(self, ms, callback):
+ self._timeouts.append((ms, callback))
+
+ def quit(self):
+ self._running = False
+
+ def run_once(self):
+ min_timeout = None
+ if len(self._pid_watches) > 0:
+ min_timeout = 500
+ for (ms, callback) in self._timeouts:
+ if (min_timeout is None) or (ms < min_timeout):
+ min_timeout = ms
+ origtime = time.time() * 1000
+ self._ensure_poll()
+ fds = self.poll.poll(min_timeout)
+ for fd in fds:
+ self._fd_callbacks[fd]()
+ to_delete_pids = []
+ for pid in self._pid_watches:
+ (opid, status) = os.waitpid(pid, os.WNOHANG)
+ if opid == pid:
+ to_delete_pids.append(pid)
+ self._pid_watches[pid](pid, status)
+ for pid in to_delete_pids:
+ del self._pid_watches[pid]
+ newtime = time.time() * 1000
+ diff = int(newtime - origtime)
+ if diff < 0: diff = 0
+ for i,(ms, callback) in enumerate(self._timeouts):
+ remaining_ms = ms - diff
+ if remaining_ms <= 0:
+ callback()
+ else:
+ self._timeouts[i] = (remaining_ms, callback)
+
+ def run(self):
+ self._running = True
+ while self._running:
+ self.run_once()
diff --git a/ostbuild/pyostbuild/odict.py b/ostbuild/pyostbuild/odict.py
new file mode 100644
index 0000000..df703cb
--- /dev/null
+++ b/ostbuild/pyostbuild/odict.py
@@ -0,0 +1,45 @@
+# -*- Mode: Python -*-
+# GObject-Introspection - a framework for introspecting GObject libraries
+# Copyright (C) 2008 Johan Dahlin
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+#
+
+"""odict - an ordered dictionary"""
+
+from UserDict import DictMixin
+
+
+class odict(DictMixin):
+
+ def __init__(self):
+ self._items = {}
+ self._keys = []
+
+ def __setitem__(self, key, value):
+ if key not in self._items:
+ self._keys.append(key)
+ self._items[key] = value
+
+ def __getitem__(self, key):
+ return self._items[key]
+
+ def __delitem__(self, key):
+ del self._items[key]
+ self._keys.remove(key)
+
+ def keys(self):
+ return self._keys[:]
diff --git a/ostbuild/pyostbuild/ostbuildlog.py b/ostbuild/pyostbuild/ostbuildlog.py
new file mode 100755
index 0000000..0f85229
--- /dev/null
+++ b/ostbuild/pyostbuild/ostbuildlog.py
@@ -0,0 +1,35 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import sys
+
+def log(msg, prefix=None):
+ if prefix is None:
+ prefix_target = ''
+ else:
+ prefix_target = prefix
+ fullmsg = '%s: %s%s\n' % (os.path.basename(sys.argv[0]), prefix_target, msg)
+ sys.stdout.write(fullmsg)
+ sys.stdout.flush()
+
+def fatal(msg):
+ log(msg, prefix="FATAL: ")
+ sys.exit(1)
+
diff --git a/ostbuild/pyostbuild/ostbuildrc.py b/ostbuild/pyostbuild/ostbuildrc.py
new file mode 100755
index 0000000..a36febb
--- /dev/null
+++ b/ostbuild/pyostbuild/ostbuildrc.py
@@ -0,0 +1,52 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,ConfigParser
+
+_config = None
+
+def get():
+ global _config
+ if _config is None:
+ configpath = os.path.expanduser('~/.config/ostbuild.cfg')
+ parser = ConfigParser.RawConfigParser()
+ parser.read([configpath])
+
+ _config = {}
+ for (k, v) in parser.items('global'):
+ _config[k.strip()] = v.strip()
+ return _config
+
+# This hack is because we want people to be able to pass None
+# for "default", but still distinguish default=None from default
+# not passed.
+_default_not_supplied = object()
+def get_key(name, provided_args=None, default=_default_not_supplied):
+ global _default_not_supplied
+ config = get()
+ if provided_args:
+ v = provided_args.get(name)
+ if v is not None:
+ return v
+ if default is _default_not_supplied:
+ # Possibly throw a KeyError
+ return config[name]
+ value = config.get(name, _default_not_supplied)
+ if value is _default_not_supplied:
+ return default
+ return value
+
diff --git a/ostbuild/pyostbuild/privileged_subproc.py b/ostbuild/pyostbuild/privileged_subproc.py
new file mode 100755
index 0000000..0116cef
--- /dev/null
+++ b/ostbuild/pyostbuild/privileged_subproc.py
@@ -0,0 +1,39 @@
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess
+
+from .ostbuildlog import log, fatal
+from . import ostbuildrc
+from .subprocess_helpers import run_sync
+
+class PrivilegedSubprocess(object):
+
+ def spawn_sync(self, argv):
+ helper = ostbuildrc.get_key('privileged_exec', default='pkexec')
+
+ handlers = {'pkexec': self._pkexec_spawn_sync}
+
+ handler = handlers.get(helper)
+ if handler is None:
+ fatal("Unrecognized privileged_exec; valid values=%r" % (handlers.keys(),))
+ else:
+ handler(argv)
+
+ def _pkexec_spawn_sync(self, argv):
+ pkexec_argv = ['pkexec'] + argv
+ run_sync(pkexec_argv)
diff --git a/ostbuild/pyostbuild/subprocess_helpers.py b/ostbuild/pyostbuild/subprocess_helpers.py
new file mode 100755
index 0000000..3754900
--- /dev/null
+++ b/ostbuild/pyostbuild/subprocess_helpers.py
@@ -0,0 +1,145 @@
+#!/usr/bin/python
+#
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import sys
+import subprocess
+
+from .ostbuildlog import log, fatal
+from .warningfilter import WarningFilter
+from .mainloop import Mainloop
+
+def _get_env_for_cwd(cwd=None, env=None):
+ # This dance is necessary because we want to keep the PWD
+ # environment variable up to date. Not doing so is a recipie
+ # for triggering edge conditions in pwd lookup.
+ if (cwd is not None) and (env is None or ('PWD' in env)):
+ if env is None:
+ env_copy = os.environ.copy()
+ else:
+ env_copy = env.copy()
+ if ('PWD' in env_copy) and (not cwd.startswith('/')):
+ env_copy['PWD'] = os.path.join(env_copy['PWD'], cwd)
+ else:
+ env_copy['PWD'] = cwd
+ else:
+ env_copy = env
+ return env_copy
+
+def run_sync_get_output(args, cwd=None, env=None, stdout=None, stderr=None, none_on_error=False,
+ log_success=False, log_initiation=False):
+ if log_initiation:
+ log("running: %s" % (subprocess.list2cmdline(args),))
+ env_copy = _get_env_for_cwd(cwd, env)
+ f = open('/dev/null', 'r')
+ if stderr is None:
+ stderr_target = sys.stderr
+ else:
+ stderr_target = stderr
+ proc = subprocess.Popen(args, stdin=f, stdout=subprocess.PIPE, stderr=stderr_target,
+ close_fds=True, cwd=cwd, env=env_copy)
+ f.close()
+ output = proc.communicate()[0].strip()
+ if proc.returncode != 0 and not none_on_error:
+ logfn = fatal
+ elif log_success:
+ logfn = log
+ else:
+ logfn = None
+ if logfn is not None:
+ logfn("cmd '%s' (cwd=%s) exited with code %d, %d bytes of output" % (subprocess.list2cmdline(args),
+ cwd, proc.returncode, len(output)))
+ if proc.returncode == 0:
+ return output
+ return None
+
+def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False,
+ log_success=True, log_initiation=True, stdin=None, stdout=None,
+ stderr=None):
+ if log_initiation:
+ log("running: %s" % (subprocess.list2cmdline(args),))
+
+ env_copy = _get_env_for_cwd(cwd, env)
+
+ if stdin is not None:
+ stdin_target = stdin
+ elif keep_stdin:
+ stdin_target = sys.stdin
+ else:
+ stdin_target = open('/dev/null', 'r')
+
+ if stdout is None:
+ stdout_target = sys.stdout
+ else:
+ stdout_target = stdout
+
+ if stderr is None:
+ stderr_target = sys.stderr
+ else:
+ stderr_target = stderr
+
+ proc = subprocess.Popen(args, stdin=stdin_target, stdout=stdout_target, stderr=stderr_target,
+ close_fds=True, cwd=cwd, env=env_copy)
+ if not keep_stdin:
+ stdin_target.close()
+ returncode = proc.wait()
+ if fatal_on_error and returncode != 0:
+ logfn = fatal
+ elif log_success:
+ logfn = log
+ else:
+ logfn = None
+ if logfn is not None:
+ logfn("pid %d exited with code %d" % (proc.pid, returncode))
+ return returncode
+
+def run_sync_monitor_log_file(args, logfile, cwd=None, env=None,
+ fatal_on_error=True, log_initiation=True):
+ if log_initiation:
+ log("running: %s" % (subprocess.list2cmdline(args),))
+
+ env_copy = _get_env_for_cwd(cwd, env)
+
+ logfile_f = open(logfile, 'w')
+
+ proc = subprocess.Popen(args, stdin=open('/dev/null', 'r'),
+ stdout=logfile_f,
+ stderr=subprocess.STDOUT,
+ close_fds=True, cwd=cwd, env=env_copy)
+ warnfilter = WarningFilter(logfile, sys.stdout)
+
+ warnfilter.start()
+
+ loop = Mainloop.get(None)
+
+ proc_estatus = None
+ def _on_pid_exited(pid, estatus):
+ global proc_estatus
+ proc_estatus = estatus
+ failed = estatus != 0
+ warnfilter.finish(not failed)
+ if fatal_on_error and failed:
+ logfn = fatal
+ else:
+ logfn = log
+ logfn("pid %d exited with code %d" % (pid, estatus))
+ loop.quit()
+ loop.watch_pid(proc.pid, _on_pid_exited)
+ loop.run()
+ return proc_estatus
diff --git a/ostbuild/pyostbuild/vcs.py b/ostbuild/pyostbuild/vcs.py
new file mode 100755
index 0000000..85f2f85
--- /dev/null
+++ b/ostbuild/pyostbuild/vcs.py
@@ -0,0 +1,155 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import re
+import urlparse
+import shutil
+
+from .subprocess_helpers import run_sync_get_output, run_sync
+from . import buildutil
+from .ostbuildlog import log, fatal
+
+def get_mirrordir(mirrordir, keytype, uri, prefix=''):
+ assert keytype == 'git'
+ parsed = urlparse.urlsplit(uri)
+ return os.path.join(mirrordir, prefix, keytype, parsed.scheme, parsed.netloc, parsed.path[1:])
+
+def _fixup_submodule_references(mirrordir, cwd):
+ submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=cwd)
+ submodule_status_lines = submodules_status_text.split('\n')
+ have_submodules = False
+ for line in submodule_status_lines:
+ if line == '': continue
+ have_submodules = True
+ line = line[1:]
+ (sub_checksum, sub_name) = line.split(' ', 1)
+ sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
+ 'submodule.%s.url' % (sub_name, )], cwd=cwd)
+ mirrordir = get_mirrordir(mirrordir, 'git', sub_url)
+ run_sync(['git', 'config', 'submodule.%s.url' % (sub_name, ), 'file://' + mirrordir], cwd=cwd)
+ return have_submodules
+
+def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
+ module_mirror = get_mirrordir(mirrordir, keytype, uri)
+ assert keytype == 'git'
+ checkoutdir_parent=os.path.dirname(dest)
+ if not os.path.isdir(checkoutdir_parent):
+ os.makedirs(checkoutdir_parent)
+ tmp_dest = dest + '.tmp'
+ if os.path.isdir(tmp_dest):
+ shutil.rmtree(tmp_dest)
+ if os.path.islink(dest):
+ os.unlink(dest)
+ if os.path.isdir(dest):
+ if overwrite:
+ shutil.rmtree(dest)
+ else:
+ tmp_dest = dest
+ if not os.path.isdir(tmp_dest):
+ run_sync(['git', 'clone', '-q', '--origin', 'localmirror',
+ '--no-checkout', module_mirror, tmp_dest])
+ run_sync(['git', 'remote', 'add', 'upstream', uri], cwd=tmp_dest)
+ else:
+ run_sync(['git', 'fetch', 'localmirror'], cwd=tmp_dest)
+ run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest)
+ run_sync(['git', 'submodule', 'init'], cwd=tmp_dest)
+ have_submodules = _fixup_submodule_references(mirrordir, tmp_dest)
+ if have_submodules:
+ run_sync(['git', 'submodule', 'update'], cwd=tmp_dest)
+ if tmp_dest != dest:
+ os.rename(tmp_dest, dest)
+ return dest
+
+def clean(keytype, checkoutdir):
+ assert keytype in ('git', 'dirty-git')
+ run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)
+
+def parse_src_key(srckey):
+ idx = srckey.find(':')
+ if idx < 0:
+ raise ValueError("Invalid SRC uri=%s" % (srckey, ))
+ keytype = srckey[:idx]
+ if keytype not in ['git', 'local']:
+ raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
+ uri = srckey[idx+1:]
+ return (keytype, uri)
+
+def get_lastfetch_path(mirrordir, keytype, uri, branch):
+ mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
+ branch_safename = branch.replace('/','_').replace('.', '_')
+ return mirror + '.lastfetch-%s' % (branch_safename, )
+
+def ensure_vcs_mirror(mirrordir, keytype, uri, branch):
+ mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
+ tmp_mirror = mirror + '.tmp'
+ if os.path.isdir(tmp_mirror):
+ shutil.rmtree(tmp_mirror)
+ if not os.path.isdir(mirror):
+ run_sync(['git', 'clone', '--mirror', uri, tmp_mirror])
+ run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror)
+ os.rename(tmp_mirror, mirror)
+ if branch is None:
+ return mirror
+ last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch)
+ if os.path.exists(last_fetch_path):
+ f = open(last_fetch_path)
+ last_fetch_contents = f.read()
+ f.close()
+ last_fetch_contents = last_fetch_contents.strip()
+ else:
+ last_fetch_contents = None
+ current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
+ current_vcs_version = current_vcs_version.strip()
+ if current_vcs_version != last_fetch_contents:
+ log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version))
+ tmp_checkout = buildutil.get_mirrordir(mirrordir, keytype, uri, prefix='_tmp-checkouts')
+ if os.path.isdir(tmp_checkout):
+ shutil.rmtree(tmp_checkout)
+ parent = os.path.dirname(tmp_checkout)
+ if not os.path.isdir(parent):
+ os.makedirs(parent)
+ run_sync(['git', 'clone', '-q', '--no-checkout', mirror, tmp_checkout])
+ run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout)
+ submodules = []
+ submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout)
+ submodule_status_lines = submodules_status_text.split('\n')
+ for line in submodule_status_lines:
+ if line == '': continue
+ line = line[1:]
+ (sub_checksum, sub_name) = line.split(' ', 1)
+ sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
+ 'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout)
+ ensure_vcs_mirror(mirrordir, keytype, sub_url, sub_checksum)
+ shutil.rmtree(tmp_checkout)
+ f = open(last_fetch_path, 'w')
+ f.write(current_vcs_version + '\n')
+ f.close()
+ return mirror
+
+def fetch(mirrordir, keytype, uri, branch, keep_going=False):
+ mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
+ last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch)
+ run_sync(['git', 'fetch'], cwd=mirror, log_initiation=False,
+ fatal_on_error=not keep_going)
+ current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
+ if current_vcs_version is not None:
+ current_vcs_version = current_vcs_version.strip()
+ f = open(last_fetch_path, 'w')
+ f.write(current_vcs_version + '\n')
+ f.close()
+
diff --git a/ostbuild/pyostbuild/warningfilter.py b/ostbuild/pyostbuild/warningfilter.py
new file mode 100644
index 0000000..2461953
--- /dev/null
+++ b/ostbuild/pyostbuild/warningfilter.py
@@ -0,0 +1,113 @@
+#
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import re
+import stat
+import fcntl
+import subprocess
+
+from . import filemonitor
+from . import mainloop
+
+warning_re = re.compile(r'(: ((warning)|(error)|(fatal error)): )|(make(\[[0-9]+\])?: \*\*\*)')
+output_whitelist_re = re.compile(r'^(make(\[[0-9]+\])?: Entering directory)|(ostbuild:)')
+
+_bold_sequence = None
+_normal_sequence = None
+if os.isatty(1):
+ _bold_sequence = subprocess.Popen(['tput', 'bold'], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w')).communicate()[0]
+ _normal_sequence = subprocess.Popen(['tput', 'sgr0'], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w')).communicate()[0]
+def _bold(text):
+ if _bold_sequence is not None:
+ return '%s%s%s' % (_bold_sequence, text, _normal_sequence)
+ else:
+ return text
+
+class WarningFilter(object):
+ def __init__(self, filename, output):
+ self.filename = filename
+ self.output = output
+
+ # inherit globals
+ self._warning_re = warning_re
+ self._nonfilter_re = output_whitelist_re
+
+ self._buf = ''
+ self._warning_count = 0
+ self._filtered_line_count = 0
+ filemon = filemonitor.FileMonitor.get()
+ filemon.add(filename, self._on_changed)
+ self._fd = os.open(filename, os.O_RDONLY)
+ fcntl.fcntl(self._fd, fcntl.F_SETFL, os.O_NONBLOCK)
+
+ def _do_read(self):
+ while True:
+ buf = os.read(self._fd, 4096)
+ if buf == '':
+ break
+ self._buf += buf
+ self._flush()
+
+ def _write_last_log_lines(self):
+ _last_line_limit = 100
+ f = open(self.filename)
+ lines = []
+ for line in f:
+ if line.startswith('ostbuild '):
+ continue
+ lines.append(line)
+ if len(lines) > _last_line_limit:
+ lines.pop(0)
+ f.close()
+ for line in lines:
+ self.output.write('| ')
+ self.output.write(line)
+
+ def _flush(self):
+ while True:
+ p = self._buf.find('\n')
+ if p < 0:
+ break
+ line = self._buf[0:p]
+ self._buf = self._buf[p+1:]
+ match = self._warning_re.search(line)
+ if match:
+ self._warning_count += 1
+ self.output.write(line + '\n')
+ else:
+ match = self._nonfilter_re.search(line)
+ if match:
+ self.output.write(line + '\n')
+ else:
+ self._filtered_line_count += 1
+
+ def _on_changed(self):
+ self._do_read()
+
+ def start(self):
+ self._do_read()
+
+ def finish(self, successful):
+ self._do_read()
+ if not successful:
+ self._write_last_log_lines()
+ pass
+ self.output.write("ostbuild %s: %d warnings\n" % ('success' if successful else _bold('failed'),
+ self._warning_count, ))
+ self.output.write("ostbuild: full log path: %s\n" % (self.filename, ))
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
