[ostree/wip/ostbuild-v2: 4/10] ostbuild: Major rework
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree/wip/ostbuild-v2: 4/10] ostbuild: Major rework
- Date: Wed, 2 May 2012 00:52:57 +0000 (UTC)
commit 15e71e88af6131b20f7369159d5ec860112391fc
Author: Colin Walters <walters verbum org>
Date: Fri Apr 6 15:12:57 2012 -0400
ostbuild: Major rework
Makefile-ostbuild.am | 11 +-
gnomeos/3.4/gnomeos-3.4-src.json | 15 ++-
gnomeos/README-build.md | 42 +++++-
gnomeos/gnomeos-install.sh | 18 ++-
...nomeos-clone-qemu.sh => gnomeos-qemu-create.sh} | 68 +++------
...{gnomeos-clone-qemu.sh => gnomeos-qemu-pull.sh} | 64 ++------
.../{gnomeos-clone-qemu.sh => gnomeos-qemu-run.sh} | 54 +------
gnomeos/gnomeos-setup.sh | 5 +
src/ostbuild/pyostbuild/buildutil.py | 2 +-
...ltin_query_content.py => builtin_bin_to_src.py} | 61 ++++---
...n_query_content.py => builtin_branch_prefix.py} | 54 ++++---
...uiltin_build.py => builtin_build_components.py} | 156 +++++++++----------
src/ostbuild/pyostbuild/builtin_checkout.py | 45 ++++--
.../pyostbuild/builtin_chroot_compile_one.py | 146 +++++++++++++-----
.../pyostbuild/builtin_chroot_run_triggers.py | 47 ------
src/ostbuild/pyostbuild/builtin_compile_one.py | 25 ++--
src/ostbuild/pyostbuild/builtin_compose.py | 110 +++++++++++++
src/ostbuild/pyostbuild/builtin_modify_snapshot.py | 63 ++++++++
...{builtin_query_content.py => builtin_prefix.py} | 58 ++++---
src/ostbuild/pyostbuild/builtin_pull_components.py | 70 ++++++++
src/ostbuild/pyostbuild/builtin_resolve.py | 103 +++++++------
src/ostbuild/pyostbuild/builtins.py | 165 ++++++++++++++++++--
src/ostbuild/pyostbuild/jsondb.py | 110 +++++++++++++
src/ostbuild/pyostbuild/main.py | 10 +-
src/ostbuild/pyostbuild/ostbuildlog.py | 10 +-
src/ostbuild/pyostbuild/subprocess_helpers.py | 6 +-
src/ostbuild/pyostbuild/vcs.py | 2 +-
27 files changed, 1019 insertions(+), 501 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 08eb8c0..a227916 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -22,18 +22,23 @@ bin_SCRIPTS += ostbuild
pyostbuilddir=$(libdir)/ostbuild/pyostbuild
pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/buildutil.py \
- src/ostbuild/pyostbuild/builtin_build.py \
+ src/ostbuild/pyostbuild/builtin_bin_to_src.py \
+ src/ostbuild/pyostbuild/builtin_branch_prefix.py \
+ src/ostbuild/pyostbuild/builtin_build_components.py \
src/ostbuild/pyostbuild/builtin_checkout.py \
+ src/ostbuild/pyostbuild/builtin_compose.py \
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \
- src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py \
src/ostbuild/pyostbuild/builtin_compile_one.py \
- src/ostbuild/pyostbuild/builtin_query_content.py \
+ src/ostbuild/pyostbuild/builtin_pull_components.py \
+ src/ostbuild/pyostbuild/builtin_prefix.py \
src/ostbuild/pyostbuild/builtin_resolve.py \
+ src/ostbuild/pyostbuild/builtin_modify_snapshot.py \
src/ostbuild/pyostbuild/builtin_status.py \
src/ostbuild/pyostbuild/builtins.py \
src/ostbuild/pyostbuild/filemonitor.py \
src/ostbuild/pyostbuild/fileutil.py \
src/ostbuild/pyostbuild/__init__.py \
+ src/ostbuild/pyostbuild/jsondb.py \
src/ostbuild/pyostbuild/kvfile.py \
src/ostbuild/pyostbuild/main.py \
src/ostbuild/pyostbuild/mainloop.py \
diff --git a/gnomeos/3.4/gnomeos-3.4-src.json b/gnomeos/3.4/gnomeos-3.4-src.json
index a229770..f58a5ea 100644
--- a/gnomeos/3.4/gnomeos-3.4-src.json
+++ b/gnomeos/3.4/gnomeos-3.4-src.json
@@ -1,7 +1,10 @@
{
- "name-prefix": "gnomeos-3.4",
+ "prefix": "gnomeos-3.4",
"architectures": ["i686"],
- "base-prefix": "yocto/gnomeos-3.4",
+ "base": {
+ "name": "yocto",
+ "src": "cgwalters:poky"
+ },
"config-opts": ["--disable-static", "--disable-silent-rules"],
@@ -25,13 +28,15 @@
"patches": {"src": "git:/src/ostree",
"branch": "wip/ostbuild-v2",
- "prefix": "gnomeos/3.4"},
+ "subdir": "gnomeos/3.4"},
"components": [
- {"src": "cgwalters:ginitscripts"},
+ {"src": "cgwalters:ginitscripts",
+ "noarch": true},
{"src": "gnome:gtk-doc-stub",
- "component": "devel"},
+ "component": "devel",
+ "noarch": true},
{"src": "git:git://github.com/atgreen/libffi.git"},
diff --git a/gnomeos/README-build.md b/gnomeos/README-build.md
index bd59261..0146c2e 100644
--- a/gnomeos/README-build.md
+++ b/gnomeos/README-build.md
@@ -16,6 +16,9 @@ base "runtime", and one "devel" with all of the development tools like
gcc. We then import that into an OSTree branch
e.g. "bases/yocto/gnomeos-3.4-i686-devel".
+At present, it's still (mostly) possible to put this data on an ext4
+filesystem and boot into it.
+
We also have a Yocto recipe "ostree-native" which generates (as you
might guess) a native binary of ostree. That binary is used to import
into an "archive mode" OSTree repository. You can see it in
@@ -26,26 +29,59 @@ can use "ostbuild" which uses "linux-user-chroot" to chroot inside,
run a build on a source tree, and outputs binaries, which we then add
to the build tree for the next module, and so on.
+The final result of all of this is that the OSTree repository gains
+new commits (which can be downloaded by clients), while still
+retaining old build history.
+
+Yocto details
+-------------
+
+I have a branch of Yocto here:
+
+https://github.com/cgwalters/poky
+
+It has a collection of patches on top of the "Edison" release of
+Yocto, some of which are hacky, others upstreamable. The most
+important part though are the modifications to commit the generated
+root filesystem into OSTree.
+
ostbuild details
----------------
The simple goal of ostbuild is that it only takes as input a
-"manifest" which is basically just a list of components to build. A
-component is a pure metadata file which includes the git repository
+"manifest" which is basically just a list of components to build. You
+can see this here:
+
+http://git.gnome.org/browse/ostree/tree/gnomeos/3.4/gnomeos-3.4-src.json
+
+A component is a pure metadata file which includes the git repository
URL and branch name, as well as ./configure flags (--enable-foo).
There is no support for building from "tarballs" - I want the ability
to review all of the code that goes in, and to efficiently store
-source code updates.
+source code updates. It's also just significantly easier from an
+implementation perspective, versus having to maintain a version
+control abstraction layer.
The result of a build of a component is an OSTree branch like
"artifacts/gnomeos-3.4-i686-devel/libxslt/master". Then, a "compose"
process merges together the individual filesystem trees into the final
branches (e.g. gnomeos-3.4-i686-devel).
+Doing local builds
+------------------
+
+This is where you want to modify one (or a few) components on top of
+what comes from the ostree.gnome.org server, and test the result
+locally. I'm working on this.
+
Doing a full build on your system
---------------------------------
+Following this process is equivalent to what we have set up on the
+ostree.gnome.org build server. It will generate a completely new
+repository.
+
srcdir=/src
builddir=/src/build
diff --git a/gnomeos/gnomeos-install.sh b/gnomeos/gnomeos-install.sh
index 99394be..c37e25a 100755
--- a/gnomeos/gnomeos-install.sh
+++ b/gnomeos/gnomeos-install.sh
@@ -46,16 +46,18 @@ if ! test -d /ostree/repo/objects; then
mkdir -p /ostree
$SRCDIR/gnomeos-setup.sh /ostree
-fi
-cd /ostree
+ cd /ostree
+
+ ostree --repo=repo remote add gnome http://ostree.gnome.org/repo ${BRANCH_PREFIX}{runtime,devel}
+ ostree-pull --repo=repo gnome
+ for branch in runtime devel; do
+ ostree --repo=repo checkout --atomic-retarget ${BRANCH_PREFIX}${branch}
+ done
+ ln -sf ${BRANCH_PREFIX}runtime current
-ostree --repo=repo remote add gnome http://ostree.gnome.org/repo ${BRANCH_PREFIX}{runtime,devel}
-ostree-pull --repo=repo gnome
-for branch in runtime devel; do
- ostree --repo=repo checkout --atomic-retarget ${BRANCH_PREFIX}${branch}
-done
-ln -sf ${BRANCH_PREFIX}runtime current
+ cd ${WORKDIR}
+fi
uname=$(uname -r)
diff --git a/gnomeos/gnomeos-clone-qemu.sh b/gnomeos/gnomeos-qemu-create.sh
similarity index 52%
copy from gnomeos/gnomeos-clone-qemu.sh
copy to gnomeos/gnomeos-qemu-create.sh
index e19abdd..bc0e6a5 100755
--- a/gnomeos/gnomeos-clone-qemu.sh
+++ b/gnomeos/gnomeos-qemu-create.sh
@@ -1,6 +1,7 @@
#!/bin/bash
# -*- indent-tabs-mode: nil; -*-
-# Run built image in QEMU
+# Create ostree-qemu.img file in the current directory, suitable
+# for booting via qemu.
#
# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
#
@@ -33,68 +34,41 @@ EOF
fi
usage () {
- echo "$0 OSTREE_REPO_PATH"
+ echo "$0"
exit 1
}
-ARCH=i686
-BRANCH_PREFIX="gnomeos-3.4-${ARCH}-"
-
-OBJ=gnomeos-fs.img
+OBJ=ostree-qemu.img
if ! test -f ${OBJ}; then
- cat <<EOF
-Create gnomeos-fs.img like this:
-$ qemu-img create gnomeos-fs.img 6G
-$ mkfs.ext4 -q -F gnomeos-fs.img
-EOF
- exit 1
-fi
-
-if ! test -d ${WORKDIR}/repo/objects; then
- cat <<EOF
-No ./repo/objects found
-EOF
- exit 1
+ # Hardcoded 6 gigabyte filesystem size here; 6 gigabytes should be
+ # enough for everybody.
+ qemu-img create $OBJ 6G
+ mkfs.ext4 -q -F $OBJ
fi
mkdir -p fs
umount fs || true
sleep 1 # Avoid Linux kernel bug, pretty sure it's the new RCU pathname lookup
-mount -o loop gnomeos-fs.img fs
+mount -o loop ostree-qemu.img fs
cd fs
-TOPROOT_BIND_MOUNTS="home root tmp"
-
-for d in $TOPROOT_BIND_MOUNTS; do
- if ! test -d $d; then
- mkdir -m 0755 $d
- fi
-done
-chmod a=rwxt tmp
-
-if ! test -d ostree; then
- mkdir -p ostree
-
+if ! test -d ./ostree/repo/objects; then
+ mkdir -p ./ostree
+
$SRCDIR/gnomeos-setup.sh $(pwd)/ostree
fi
-rsync -a -H -v ${WORKDIR}/repo ${WORKDIR}/current ${WORKDIR}/modules ${WORKDIR}/var ${WORKDIR}/gnomeos-3.4-* ./ostree
+mkdir -p ./run ./home ./root ./sys
+mkdir -p ./tmp
+chmod 01777 ./tmp
-current_uname=$(uname -r)
+mkdir -p $(pwd)/ostree/modules
+rsync -a -H -v --delete /ostree/modules/ ./ostree/modules/
-cd ${WORKDIR}
-
-sync
+cd ..
umount fs
-rmdir fs
-
-ARGS="$@"
-if ! echo $ARGS | grep -q 'ostree='; then
- ARGS="ostree=current $ARGS"
-fi
-ARGS="rd.plymouth=0 root=/dev/sda $ARGS"
-KERNEL=/boot/vmlinuz-${current_uname}
-INITRD_ARG="-initrd /boot/initramfs-ostree-${current_uname}.img"
-exec qemu-kvm -kernel ${KERNEL} ${INITRD_ARG} -hda gnomeos-fs.img -net user -net nic,model=virtio -m 512M -append "$ARGS" -monitor stdio
+cat << EOF
+Next, run gnomeos-qemu-pull.sh to copy data.
+EOF
diff --git a/gnomeos/gnomeos-clone-qemu.sh b/gnomeos/gnomeos-qemu-pull.sh
similarity index 51%
copy from gnomeos/gnomeos-clone-qemu.sh
copy to gnomeos/gnomeos-qemu-pull.sh
index e19abdd..66d9547 100755
--- a/gnomeos/gnomeos-clone-qemu.sh
+++ b/gnomeos/gnomeos-qemu-pull.sh
@@ -33,68 +33,38 @@ EOF
fi
usage () {
- echo "$0 OSTREE_REPO_PATH"
+ cat <<EOF
+usage: $0 SRC_REPO_PATH CURRENT_REF [REFS...]
+EOF
exit 1
}
-ARCH=i686
-BRANCH_PREFIX="gnomeos-3.4-${ARCH}-"
+SRC_REPO_PATH=$1
+test -n "$SRC_REPO_PATH" || usage
+shift
-OBJ=gnomeos-fs.img
-if ! test -f ${OBJ}; then
- cat <<EOF
-Create gnomeos-fs.img like this:
-$ qemu-img create gnomeos-fs.img 6G
-$ mkfs.ext4 -q -F gnomeos-fs.img
-EOF
- exit 1
-fi
+CURRENT_REF=$1
+test -n "$CURRENT_REF" || usage
+shift
-if ! test -d ${WORKDIR}/repo/objects; then
+if ! test -f ostree-qemu.img; then
cat <<EOF
-No ./repo/objects found
+ostree-qemu.img not found; You must run gnomeos-qemu-create.sh first
EOF
- exit 1
fi
mkdir -p fs
umount fs || true
sleep 1 # Avoid Linux kernel bug, pretty sure it's the new RCU pathname lookup
-mount -o loop gnomeos-fs.img fs
+mount -o loop ostree-qemu.img fs
cd fs
+ostree --repo=${SRC_REPO_PATH} local-clone ./ostree/repo ${CURRENT_REF} "$@"
-TOPROOT_BIND_MOUNTS="home root tmp"
-
-for d in $TOPROOT_BIND_MOUNTS; do
- if ! test -d $d; then
- mkdir -m 0755 $d
- fi
-done
-chmod a=rwxt tmp
-
-if ! test -d ostree; then
- mkdir -p ostree
-
- $SRCDIR/gnomeos-setup.sh $(pwd)/ostree
-fi
-
-rsync -a -H -v ${WORKDIR}/repo ${WORKDIR}/current ${WORKDIR}/modules ${WORKDIR}/var ${WORKDIR}/gnomeos-3.4-* ./ostree
-
-current_uname=$(uname -r)
+cd ostree
+ostree --repo=./repo checkout --atomic-retarget ${CURRENT_REF}
+ln -sf ${CURRENT_REF} ${CURRENT_REF}.tmplink
+mv -T ${CURRENT_REF}.tmplink current
cd ${WORKDIR}
-
-sync
umount fs
-rmdir fs
-
-ARGS="$@"
-if ! echo $ARGS | grep -q 'ostree='; then
- ARGS="ostree=current $ARGS"
-fi
-ARGS="rd.plymouth=0 root=/dev/sda $ARGS"
-KERNEL=/boot/vmlinuz-${current_uname}
-INITRD_ARG="-initrd /boot/initramfs-ostree-${current_uname}.img"
-
-exec qemu-kvm -kernel ${KERNEL} ${INITRD_ARG} -hda gnomeos-fs.img -net user -net nic,model=virtio -m 512M -append "$ARGS" -monitor stdio
diff --git a/gnomeos/gnomeos-clone-qemu.sh b/gnomeos/gnomeos-qemu-run.sh
similarity index 55%
rename from gnomeos/gnomeos-clone-qemu.sh
rename to gnomeos/gnomeos-qemu-run.sh
index e19abdd..db93588 100755
--- a/gnomeos/gnomeos-clone-qemu.sh
+++ b/gnomeos/gnomeos-qemu-run.sh
@@ -33,62 +33,16 @@ EOF
fi
usage () {
- echo "$0 OSTREE_REPO_PATH"
+ echo "$0"
exit 1
}
-ARCH=i686
-BRANCH_PREFIX="gnomeos-3.4-${ARCH}-"
-
-OBJ=gnomeos-fs.img
-if ! test -f ${OBJ}; then
- cat <<EOF
-Create gnomeos-fs.img like this:
-$ qemu-img create gnomeos-fs.img 6G
-$ mkfs.ext4 -q -F gnomeos-fs.img
-EOF
- exit 1
-fi
-
-if ! test -d ${WORKDIR}/repo/objects; then
- cat <<EOF
-No ./repo/objects found
-EOF
- exit 1
+if ! test -f ostree-qemu.img; then
+ echo "ostree-qemu.img not found; You must run gnomeos-qemu-create.sh first"
fi
-mkdir -p fs
-umount fs || true
-sleep 1 # Avoid Linux kernel bug, pretty sure it's the new RCU pathname lookup
-mount -o loop gnomeos-fs.img fs
-
-cd fs
-
-TOPROOT_BIND_MOUNTS="home root tmp"
-
-for d in $TOPROOT_BIND_MOUNTS; do
- if ! test -d $d; then
- mkdir -m 0755 $d
- fi
-done
-chmod a=rwxt tmp
-
-if ! test -d ostree; then
- mkdir -p ostree
-
- $SRCDIR/gnomeos-setup.sh $(pwd)/ostree
-fi
-
-rsync -a -H -v ${WORKDIR}/repo ${WORKDIR}/current ${WORKDIR}/modules ${WORKDIR}/var ${WORKDIR}/gnomeos-3.4-* ./ostree
-
current_uname=$(uname -r)
-cd ${WORKDIR}
-
-sync
-umount fs
-rmdir fs
-
ARGS="$@"
if ! echo $ARGS | grep -q 'ostree='; then
ARGS="ostree=current $ARGS"
@@ -97,4 +51,4 @@ ARGS="rd.plymouth=0 root=/dev/sda $ARGS"
KERNEL=/boot/vmlinuz-${current_uname}
INITRD_ARG="-initrd /boot/initramfs-ostree-${current_uname}.img"
-exec qemu-kvm -kernel ${KERNEL} ${INITRD_ARG} -hda gnomeos-fs.img -net user -net nic,model=virtio -m 512M -append "$ARGS" -monitor stdio
+exec qemu-kvm -kernel ${KERNEL} ${INITRD_ARG} -hda ostree-qemu.img -net user -net nic,model=virtio -m 512M -append "$ARGS" -monitor stdio
diff --git a/gnomeos/gnomeos-setup.sh b/gnomeos/gnomeos-setup.sh
index e7c1ad4..77fb893 100755
--- a/gnomeos/gnomeos-setup.sh
+++ b/gnomeos/gnomeos-setup.sh
@@ -58,6 +58,11 @@ fi
mkdir -p ./var/lib/gdm
chown 2:2 ./var/lib/gdm
+mkdir -p ./var/log/gdm
+chown 2:2 ./var/log/gdm
+chmod 01770 ./var/log/gdm
+
+mkdir -p ./var/lib/AccountsService
touch ./var/shadow
chmod 0600 ./var/shadow
diff --git a/src/ostbuild/pyostbuild/buildutil.py b/src/ostbuild/pyostbuild/buildutil.py
index 96e0e40..386a35c 100755
--- a/src/ostbuild/pyostbuild/buildutil.py
+++ b/src/ostbuild/pyostbuild/buildutil.py
@@ -40,7 +40,7 @@ def parse_src_key(srckey):
if idx < 0:
raise ValueError("Invalid SRC uri=%s" % (srckey, ))
keytype = srckey[:idx]
- if keytype not in ['git']:
+ if keytype not in ['git', 'dirty-git']:
raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
uri = srckey[idx+1:]
return (keytype, uri)
diff --git a/src/ostbuild/pyostbuild/builtin_query_content.py b/src/ostbuild/pyostbuild/builtin_bin_to_src.py
similarity index 50%
copy from src/ostbuild/pyostbuild/builtin_query_content.py
copy to src/ostbuild/pyostbuild/builtin_bin_to_src.py
index 89f5839..b8cbb0a 100755
--- a/src/ostbuild/pyostbuild/builtin_query_content.py
+++ b/src/ostbuild/pyostbuild/builtin_bin_to_src.py
@@ -28,40 +28,49 @@ from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
-class OstbuildQueryContent(builtins.Builtin):
- name = "query-content"
- short_description = "Output metadata from a component"
+class OstbuildBinToSrc(builtins.Builtin):
+ name = "bin-to-src"
+ short_description = "Turn a binary snapshot into a source snapshot"
def __init__(self):
builtins.Builtin.__init__(self)
+ def bin_snapshot_to_src(self, bin_snapshot):
+ del bin_snapshot['00ostree-bin-snapshot-version']
+
+ src_snapshot = dict(bin_snapshot)
+ src_snapshot['00ostree-src-snapshot-version'] = 0
+
+ all_architectures = src_snapshot['architecture-buildroots'].keys()
+ # Arbitrarily take first architecture
+ first_arch = all_architectures[0]
+
+ bin_components = src_snapshot['components']
+ src_components = {}
+ src_snapshot['components'] = src_components
+ for archname,rev in bin_components.iteritems():
+ (name, arch) = archname.rsplit('/', 1)
+ if arch != first_arch:
+ continue
+ meta = dict(self.get_component_meta_from_revision(rev))
+ del meta['name']
+ src_components[name] = meta
+
+ for target in src_snapshot['targets']:
+ del target['base']['ostree-revision']
+
+ return src_snapshot
+
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--branch', required=True)
- parser.add_argument('--component')
+ parser.add_argument('--prefix')
+ parser.add_argument('--bin-snapshot')
args = parser.parse_args(argv)
- self.args = args
self.parse_config()
+ self.parse_bin_snapshot(args.prefix, args.bin_snapshot)
- contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'cat', args.branch, 'contents.json'])
-
- if args.component is None:
- sys.stdout.write(contents_json_text)
- else:
- contents = json.loads(contents_json_text)
- contents_list = contents['contents']
- found = False
- for content in contents_list:
- if content['name'] != args.component:
- found = True
- break
- if not found:
- fatal("Unknown component '%s'" % (args.component, ))
- ostbuildmeta_json = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'cat', content['ostree-revision'],
- '/_ostbuild-meta.json'])
- sys.stdout.write(ostbuildmeta_json)
+ snapshot = self.bin_snapshot_to_src(self.bin_snapshot)
+ json.dump(snapshot, sys.stdout, indent=4, sort_keys=True)
-builtins.register(OstbuildQueryContent)
+builtins.register(OstbuildBinToSrc)
diff --git a/src/ostbuild/pyostbuild/builtin_query_content.py b/src/ostbuild/pyostbuild/builtin_branch_prefix.py
similarity index 53%
copy from src/ostbuild/pyostbuild/builtin_query_content.py
copy to src/ostbuild/pyostbuild/builtin_branch_prefix.py
index 89f5839..5aaf939 100755
--- a/src/ostbuild/pyostbuild/builtin_query_content.py
+++ b/src/ostbuild/pyostbuild/builtin_branch_prefix.py
@@ -28,40 +28,44 @@ from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
-class OstbuildQueryContent(builtins.Builtin):
- name = "query-content"
- short_description = "Output metadata from a component"
+class OstbuildBranchPrefix(builtins.Builtin):
+ name = "prefix-branch"
+ short_description = "Copy current source snapshot to new prefix"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--branch', required=True)
- parser.add_argument('--component')
+ parser.add_argument('--prefix')
+ parser.add_argument('--src-snapshot')
+ parser.add_argument('newprefix')
args = parser.parse_args(argv)
- self.args = args
self.parse_config()
+ self.parse_snapshot(args.prefix, args.src_snapshot)
- contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'cat', args.branch, 'contents.json'])
+ if args.newprefix == self.prefix:
+ fatal("Specified prefix %r matches active prefix" % (args.newprefix, ))
+
+ db = self.create_db('src-snapshot', prefix=args.newprefix)
+
+ log("Branching from source snapshot %r" % (self.snapshot_path, ))
+
+ orig_prefix = self.prefix
+
+ forked_snapshot = dict(self.snapshot)
+ forked_snapshot['prefix'] = args.newprefix
+
+ for target in forked_snapshot['targets']:
+ name = target['name']
+ if not name.startswith(orig_prefix):
+ fatal("Mismatched name %r in snapshot" % (name, ))
+ target['name'] = name.replace(orig_prefix, args.newprefix)
- if args.component is None:
- sys.stdout.write(contents_json_text)
- else:
- contents = json.loads(contents_json_text)
- contents_list = contents['contents']
- found = False
- for content in contents_list:
- if content['name'] != args.component:
- found = True
- break
- if not found:
- fatal("Unknown component '%s'" % (args.component, ))
- ostbuildmeta_json = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'cat', content['ostree-revision'],
- '/_ostbuild-meta.json'])
- sys.stdout.write(ostbuildmeta_json)
+ db.store(forked_snapshot)
+
+ run_sync(['ostbuild', 'prefix', args.newprefix],
+ log_initiation=False, log_success=False)
-builtins.register(OstbuildQueryContent)
+builtins.register(OstbuildBranchPrefix)
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build_components.py
similarity index 64%
rename from src/ostbuild/pyostbuild/builtin_build.py
rename to src/ostbuild/pyostbuild/builtin_build_components.py
index 772aaca..5e35c0f 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build_components.py
@@ -36,9 +36,9 @@ from . import vcs
class BuildOptions(object):
pass
-class OstbuildBuild(builtins.Builtin):
- name = "build"
- short_description = "Rebuild all artifacts from the given manifest"
+class OstbuildBuildComponents(builtins.Builtin):
+ name = "build-components"
+ short_description = "Build multiple components from given source snapshot"
def __init__(self):
builtins.Builtin.__init__(self)
@@ -49,7 +49,8 @@ class OstbuildBuild(builtins.Builtin):
args = ['setarch', architecture]
else:
args = []
- args.extend(['ostbuild', 'chroot-compile-one'])
+ args.extend(['ostbuild', 'chroot-compile-one',
+ '--snapshot=' + self.snapshot_path])
return args
def _launch_debug_shell(self, architecture, buildroot, cwd=None):
@@ -60,10 +61,10 @@ class OstbuildBuild(builtins.Builtin):
run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
fatal("Exiting after debug shell")
- def _build_one_component(self, name, component):
+ def _build_one_component(self, basename, component, architecture):
branch = component['branch']
- architecture = component['architecture']
+ name = '%s/%s' % (basename, architecture)
buildname = 'components/%s' % (name, )
current_vcs_version = component['revision']
@@ -73,7 +74,7 @@ class OstbuildBuild(builtins.Builtin):
stderr=open('/dev/null', 'w'),
none_on_error=True)
if previous_build_version is not None:
- log("Previous build of '%s' is %s" % (buildname, previous_build_version))
+ log("Previous build of '%s' is %s" % (name, previous_build_version))
previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', previous_build_version,
@@ -92,19 +93,15 @@ class OstbuildBuild(builtins.Builtin):
else:
log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
else:
- log("No previous build for '%s' found" % (buildname, ))
+ log("No previous build for '%s' found" % (name, ))
checkoutdir = os.path.join(self.workdir, 'src')
- component_src = os.path.join(checkoutdir, name)
- run_sync(['ostbuild', 'checkout', '--clean', '--overwrite', name], cwd=checkoutdir)
+ component_src = os.path.join(checkoutdir, basename)
+ run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
+ '--clean', '--overwrite', basename], cwd=checkoutdir)
artifact_meta = dict(component)
- metadata_path = os.path.join(component_src, '_ostbuild-meta.json')
- f = open(metadata_path, 'w')
- json.dump(artifact_meta, f, indent=4, sort_keys=True)
- f.close()
-
logdir = os.path.join(self.workdir, 'logs', name)
fileutil.ensure_dir(logdir)
log_path = os.path.join(logdir, 'compile.log')
@@ -116,7 +113,7 @@ class OstbuildBuild(builtins.Builtin):
log("Logging to %s" % (log_path, ))
f = open(log_path, 'w')
chroot_args = self._get_ostbuild_chroot_args(architecture)
- chroot_args.extend(['--pristine', '--name=' + name])
+ chroot_args.extend(['--pristine', '--name=' + basename, '--arch=' + architecture])
if self.buildopts.shell_on_failure:
ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
if ecode != 0:
@@ -146,69 +143,48 @@ class OstbuildBuild(builtins.Builtin):
os.unlink(statoverride_path)
return True
- def _compose(self, target):
- base_name = 'bases/%s' % (target['base']['name'], )
- branch_to_rev = {}
- branch_to_subtrees = {}
+ def _resolve_refs(self, refs):
+ args = ['ostree', '--repo=' + self.repo, 'rev-parse']
+ args.extend(refs)
+ output = run_sync_get_output(args)
+ return output.split('\n')
- contents = [base_name]
- branch_to_subtrees[base_name] = ['/']
- base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', base_name])
+ def _save_bin_snapshot(self, components, component_architectures):
+ bin_snapshot = dict(self.snapshot)
- branch_to_rev[base_name] = base_revision
+ del bin_snapshot['00ostree-src-snapshot-version']
+ bin_snapshot['00ostree-bin-snapshot-version'] = 0
- args = ['ostree', '--repo=' + self.repo, 'rev-parse']
- for component in target['contents']:
- name = component['name']
- contents.append(name)
- args.append('components/%s' % (name, ))
- branch_to_subtrees[name] = component['trees']
- branch_revs_text = run_sync_get_output(args)
- branch_revs = branch_revs_text.split('\n')
-
- for (content, rev) in zip(target['contents'], branch_revs):
- name = content['name']
- branch_to_rev[name] = rev
-
- compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
- if os.path.isdir(compose_rootdir):
- shutil.rmtree(compose_rootdir)
- os.mkdir(compose_rootdir)
-
- resolved_base = dict(target['base'])
- resolved_base['ostree-revision'] = base_revision
- resolved_contents = list(target['contents'])
- for component in resolved_contents:
- component['ostree-revision'] = branch_to_rev[component['name']]
- metadata = {'source': 'ostbuild compose v0',
- 'base': resolved_base,
- 'contents': resolved_contents}
-
- for branch in contents:
- branch_rev = branch_to_rev[branch]
- subtrees = branch_to_subtrees[branch]
- for subtree in subtrees:
- run_sync(['ostree', '--repo=' + self.repo,
- 'checkout', '--user-mode',
- '--union', '--subpath=' + subtree,
- branch_rev, compose_rootdir])
-
- contents_path = os.path.join(compose_rootdir, 'contents.json')
- f = open(contents_path, 'w')
- json.dump(metadata, f, indent=4, sort_keys=True)
- f.close()
-
- run_sync(['ostree', '--repo=' + self.repo,
- 'commit', '-b', target['name'], '-s', 'Compose',
- '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
- '--skip-if-unchanged'], cwd=compose_rootdir)
+ for target in bin_snapshot['targets']:
+ base = target['base']
+ base_name = 'bases/%s' % (base['name'], )
+ base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', base_name])
+ base['ostree-revision'] = base_revision
+
+ component_refs = []
+ for name in components.iterkeys():
+ for architecture in component_architectures[name]:
+ component_refs.append('components/%s/%s' % (name, architecture))
+
+ new_components = {}
+ resolved_refs = self._resolve_refs(component_refs)
+ for name,rev in zip(components.iterkeys(), resolved_refs):
+ for architecture in component_architectures[name]:
+ archname = '%s/%s' % (name, architecture)
+ new_components[archname] = rev
+
+ bin_snapshot['components'] = new_components
+
+ path = self.get_bin_snapshot_db().store(bin_snapshot)
+ log("Binary snapshot: %s" % (path, ))
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--skip-built', action='store_true')
- parser.add_argument('--recompose', action='store_true')
- parser.add_argument('--skip-compose', action='store_true')
+ parser.add_argument('--prefix')
+ parser.add_argument('--src-snapshot')
+ parser.add_argument('--compose', action='store_true')
parser.add_argument('--start-at')
parser.add_argument('--shell-on-failure', action='store_true')
parser.add_argument('--debug-shell', action='store_true')
@@ -218,17 +194,28 @@ class OstbuildBuild(builtins.Builtin):
self.args = args
self.parse_config()
- self.parse_snapshot()
+ self.parse_snapshot(args.prefix, args.src_snapshot)
+
+ log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
self.buildopts = BuildOptions()
self.buildopts.shell_on_failure = args.shell_on_failure
self.buildopts.skip_built = args.skip_built
+ required_components = {}
+ component_architectures = {}
+ for target in self.snapshot['targets']:
+ for tree_content in target['contents']:
+ (name, arch) = tree_content['name'].rsplit('/', 1)
+ required_components[name] = self.snapshot['components'][name]
+ if name not in component_architectures:
+ component_architectures[name] = set([arch])
+ else:
+ component_architectures[name].add(arch)
+
build_component_order = []
- if args.recompose:
- pass
- elif len(args.components) == 0:
- tsorted = buildutil.tsort_components(self.snapshot['components'], 'build-depends')
+ if len(args.components) == 0:
+ tsorted = buildutil.tsort_components(required_components, 'build-depends')
tsorted.reverse()
build_component_order = tsorted
else:
@@ -253,11 +240,14 @@ class OstbuildBuild(builtins.Builtin):
start_at_index = 0
for component_name in build_component_order[start_at_index:]:
- component = self.snapshot['components'].get(component_name)
- self._build_one_component(component_name, component)
+ component = required_components[component_name]
+ architectures = component_architectures[component_name]
+ for architecture in architectures:
+ self._build_one_component(component_name, component, architecture)
+
+ self._save_bin_snapshot(required_components, component_architectures)
- if not args.skip_compose:
- for target in self.snapshot['targets']:
- self._compose(target)
+ if args.compose:
+ run_sync(['ostbuild', 'compose', '--prefix=' + self.prefix])
-builtins.register(OstbuildBuild)
+builtins.register(OstbuildBuildComponents)
diff --git a/src/ostbuild/pyostbuild/builtin_checkout.py b/src/ostbuild/pyostbuild/builtin_checkout.py
index 553e505..27281be 100755
--- a/src/ostbuild/pyostbuild/builtin_checkout.py
+++ b/src/ostbuild/pyostbuild/builtin_checkout.py
@@ -40,6 +40,9 @@ class OstbuildCheckout(builtins.Builtin):
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--overwrite', action='store_true')
+ parser.add_argument('--prefix')
+ parser.add_argument('--snapshot')
+ parser.add_argument('-a', '--active-tree', action='store_true')
parser.add_argument('--clean', action='store_true')
parser.add_argument('components', nargs='*')
@@ -47,28 +50,41 @@ class OstbuildCheckout(builtins.Builtin):
self.args = args
self.parse_config()
- self.parse_snapshot()
if len(args.components) > 0:
checkout_components = args.components
else:
checkout_components = [os.path.basename(os.getcwd())]
+ if args.active_tree:
+ self.parse_active_branch()
+ else:
+ self.parse_snapshot(args.prefix, args.snapshot)
+
for component_name in checkout_components:
found = False
- component = self.snapshot['components'].get(component_name)
- if component is None:
- fatal("Unknown component %r" % (component_name, ))
+ component = self.get_component_meta(component_name)
(keytype, uri) = buildutil.parse_src_key(component['src'])
checkoutdir = os.path.join(os.getcwd(), component_name)
fileutil.ensure_parent_dir(checkoutdir)
- component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
- component['revision'],
- overwrite=args.overwrite)
+ is_dirty = (keytype == 'dirty-git')
+ if is_dirty:
+ # Kind of a hack, but...
+ if os.path.lexists(checkoutdir):
+ os.unlink(checkoutdir)
+ os.symlink(uri, checkoutdir)
+ else:
+ vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
+ component['revision'],
+ overwrite=args.overwrite)
+
if args.clean:
- vcs.clean(keytype, checkoutdir)
+ if is_dirty:
+ log("note: ignoring --clean argument due to \"dirty-git:\" specification")
+ else:
+ vcs.clean(keytype, checkoutdir)
patches = component.get('patches')
if patches is not None:
@@ -78,15 +94,20 @@ class OstbuildCheckout(builtins.Builtin):
self.patchdir, patches['branch'],
overwrite=True)
- patch_prefix = patches.get('prefix', None)
- if patch_prefix is not None:
- patchdir = os.path.join(self.patchdir, patch_prefix)
+ patch_subdir = patches.get('subdir', None)
+ if patch_subdir is not None:
+ patchdir = os.path.join(self.patchdir, patch_subdir)
else:
patchdir = self.patchdir
for patch in patches['files']:
patch_path = os.path.join(patchdir, patch)
run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=checkoutdir)
+
+ metadata_path = os.path.join(checkoutdir, '_ostbuild-meta.json')
+ f = open(metadata_path, 'w')
+ json.dump(component, f, indent=4, sort_keys=True)
+ f.close()
- print "Checked out: %r" % (component_src, )
+ log("Checked out: %r" % (checkoutdir, ))
builtins.register(OstbuildCheckout)
diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
index 0c6a96c..9bb8c98 100755
--- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
@@ -18,7 +18,9 @@
import os,sys,re,subprocess,tempfile,shutil
from StringIO import StringIO
import argparse
+import time
import json
+import hashlib
from . import builtins
from . import buildutil
@@ -31,47 +33,133 @@ class OstbuildChrootCompileOne(builtins.Builtin):
name = "chroot-compile-one"
short_description = "Build artifacts from the current source directory in a chroot"
- def _compose_buildroot(self, component_name, dirpath):
+ def _resolve_refs(self, refs):
+ args = ['ostree', '--repo=' + self.repo, 'rev-parse']
+ args.extend(refs)
+ output = run_sync_get_output(args)
+ return output.split('\n')
+
+ def _compose_buildroot(self, component_name, architecture):
+ starttime = time.time()
+
+ rootdir_prefix = os.path.join(self.workdir, 'roots')
+ rootdir = os.path.join(rootdir_prefix, component_name)
+ fileutil.ensure_parent_dir(rootdir)
+
+ # Clean up any leftover root dir
+ rootdir_tmp = rootdir + '.tmp'
+ if os.path.isdir(rootdir_tmp):
+ shutil.rmtree(rootdir_tmp)
+
components = self.snapshot['components']
dependencies = buildutil.build_depends(component_name, components)
component = components.get(component_name)
- base_devel_name = 'bases/%s-%s-%s' % (self.snapshot['base-prefix'],
- component['architecture'],
- 'devel')
+ buildroots = self.snapshot['architecture-buildroots']
+ base_devel_name = 'bases/' + buildroots[architecture]
+
+ refs_to_resolve = [base_devel_name]
checkout_trees = [(base_devel_name, '/')]
for dependency_name in dependencies:
- buildname = 'components/%s' % (dependency_name, )
+ buildname = 'components/%s/%s' % (dependency_name, architecture)
+ refs_to_resolve.append(buildname)
checkout_trees.append((buildname, '/runtime'))
checkout_trees.append((buildname, '/devel'))
+ resolved_refs = self._resolve_refs(refs_to_resolve)
+ ref_to_rev = {}
+ for ref,rev in zip(refs_to_resolve, resolved_refs):
+ ref_to_rev[ref] = rev
+
+ link_cache_dir = os.path.join(self.workdir, 'link-cache')
+ fileutil.ensure_dir(link_cache_dir)
+
+ sha = hashlib.sha256()
+
+ (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-buildroot-')
+ f = os.fdopen(fd, 'w')
+ for (branch, subpath) in checkout_trees:
+ f.write(ref_to_rev[branch])
+ f.write('\0')
+ f.write(subpath)
+ f.write('\0')
+ f.close()
+
+ f = open(tmppath)
+ buf = f.read(8192)
+ while buf != '':
+ sha.update(buf)
+ buf = f.read(8192)
+ f.close()
+
+ new_root_cacheid = sha.hexdigest()
+
+ rootdir_cache_path = os.path.join(rootdir_prefix, component_name + '.cacheid')
+
+ if os.path.isdir(rootdir):
+ if os.path.isfile(rootdir_cache_path):
+ f = open(rootdir_cache_path)
+ prev_cache_id = f.read().strip()
+ f.close()
+ if prev_cache_id == new_root_cacheid:
+ log("Reusing previous buildroot")
+ os.unlink(tmppath)
+ return rootdir
+ else:
+ log("New buildroot differs from previous")
+
+ shutil.rmtree(rootdir)
+
+ os.mkdir(rootdir_tmp)
+
+ if len(checkout_trees) > 0:
+ log("composing buildroot from %d parents (last: %r)" % (len(checkout_trees),
+ checkout_trees[-1][0]))
+
link_cache_dir = os.path.join(self.workdir, 'link-cache')
fileutil.ensure_dir(link_cache_dir)
- for (branch, rootpath) in checkout_trees:
- run_sync(['ostree', '--repo=' + self.repo,
- 'checkout', '--user-mode', '--link-cache=' + link_cache_dir,
- '--union', '--subpath=' + rootpath,
- branch, dirpath])
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'checkout', '--link-cache=' + link_cache_dir,
+ '--user-mode', '--union', '--from-stdin', rootdir_tmp],
+ stdin=open(tmppath))
+
+ os.unlink(tmppath);
+
+ builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
+ os.mkdir(builddir_tmp)
+ os.mkdir(os.path.join(builddir_tmp, 'source'))
+ os.mkdir(os.path.join(builddir_tmp, 'results'))
+ os.rename(rootdir_tmp, rootdir)
+
+ f = open(rootdir_cache_path, 'w')
+ f.write(new_root_cacheid)
+ f.write('\n')
+ f.close()
+
+ endtime = time.time()
+ log("Composed buildroot; %d seconds elapsed" % (int(endtime - starttime),))
+
+ return rootdir
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--pristine', action='store_true')
+ parser.add_argument('--prefix')
+ parser.add_argument('--snapshot', required=True)
parser.add_argument('--name')
+ parser.add_argument('--arch', required=True)
parser.add_argument('--debug-shell', action='store_true')
args = parser.parse_args(argv)
self.parse_config()
- self.parse_snapshot()
+ self.parse_snapshot(args.prefix, args.snapshot)
if args.name:
component_name = args.name
else:
- cwd = os.getcwd()
- parent = os.path.dirname(cwd)
- parentparent = os.path.dirname(parent)
- component_name = '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
+ component_name = self.get_component_from_cwd()
components = self.snapshot['components']
component = components.get(component_name)
@@ -93,37 +181,16 @@ class OstbuildChrootCompileOne(builtins.Builtin):
shutil.rmtree(child_tmpdir)
fileutil.ensure_dir(child_tmpdir)
- resultdir = os.path.join(self.workdir, 'results', component_name)
+ resultdir = os.path.join(self.workdir, 'results', component_name, args.arch)
if os.path.isdir(resultdir):
shutil.rmtree(resultdir)
fileutil.ensure_dir(resultdir)
- rootdir_prefix = os.path.join(workdir, 'roots')
- fileutil.ensure_dir(rootdir_prefix)
- rootdir = os.path.join(rootdir_prefix, component_name)
- fileutil.ensure_parent_dir(rootdir)
- if os.path.isdir(rootdir):
- shutil.rmtree(rootdir)
-
- rootdir_tmp = rootdir + '.tmp'
- builddir = os.path.join(rootdir, 'ostbuild');
- if os.path.isdir(rootdir_tmp):
- shutil.rmtree(rootdir_tmp)
- os.mkdir(rootdir_tmp)
-
- self._compose_buildroot(component_name, rootdir_tmp)
+ rootdir = self._compose_buildroot(component_name, args.arch)
- child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp]
- run_sync(child_args)
-
- builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
- os.mkdir(builddir_tmp)
- os.mkdir(os.path.join(builddir_tmp, 'source'))
- os.mkdir(os.path.join(builddir_tmp, 'results'))
- os.rename(rootdir_tmp, rootdir)
log("Checked out buildroot: %s" % (rootdir, ))
- sourcedir=os.path.join(builddir, 'source', component_name)
+ sourcedir=os.path.join(rootdir, 'ostbuild', 'source', component_name)
fileutil.ensure_dir(sourcedir)
output_metadata = open('_ostbuild-meta.json', 'w')
@@ -148,7 +215,6 @@ class OstbuildChrootCompileOne(builtins.Builtin):
'compile-one',
'--ostbuild-resultdir=/ostbuild/results',
'--ostbuild-meta=_ostbuild-meta.json'])
- child_args.extend(self.metadata.get('config-opts', []))
env_copy = dict(buildutil.BUILD_ENV)
env_copy['PWD'] = chroot_sourcedir
run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell)
diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py
index f703d0d..eef08df 100755
--- a/src/ostbuild/pyostbuild/builtin_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_compile_one.py
@@ -57,6 +57,8 @@ class OstbuildCompileOne(builtins.Builtin):
def execute(self, args):
self.default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )]
+
+ starttime = time.time()
uname=os.uname()
kernel=uname[0].lower()
@@ -77,32 +79,28 @@ class OstbuildCompileOne(builtins.Builtin):
'--infodir=' + os.path.join(PREFIX, 'share', 'info')]
self.makeargs = ['make']
- self.ostbuild_resultdir=None
- self.ostbuild_meta=None
+ self.ostbuild_resultdir='_ostbuild-results'
+ self.ostbuild_meta_path='_ostbuild-meta.json'
chdir = None
+ opt_install = False
for arg in args:
if arg.startswith('--ostbuild-resultdir='):
self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):]
elif arg.startswith('--ostbuild-meta='):
- self.ostbuild_meta=arg[len('--ostbuild-meta='):]
+ self.ostbuild_meta_path=arg[len('--ostbuild-meta='):]
elif arg.startswith('--chdir='):
os.chdir(arg[len('--chdir='):])
- elif arg.startswith('--'):
- self.configargs.append(arg)
else:
self.makeargs.append(arg)
- if self.ostbuild_resultdir is None:
- fatal("Must specify --ostbuild-resultdir=")
- if self.ostbuild_meta is None:
- fatal("Must specify --ostbuild-meta=")
-
- f = open(self.ostbuild_meta)
+ f = open(self.ostbuild_meta_path)
self.metadata = json.load(f)
f.close()
+ self.configargs.extend(self.metadata.get('config-opts', []))
+
if self.metadata.get('rm-configure', False):
configure_path = 'configure'
if os.path.exists(configure_path):
@@ -235,6 +233,11 @@ class OstbuildCompileOne(builtins.Builtin):
except OSError, e:
pass
+ endtime = time.time()
+
+ log("Compliation succeeded; %d seconds elapsed" % (int(endtime - starttime),))
+ log("Results placed in %s" % (self.ostbuild_resultdir, ))
+
def _install_and_unlink(self, src, dest):
statsrc = os.lstat(src)
dirname = os.path.dirname(dest)
diff --git a/src/ostbuild/pyostbuild/builtin_compose.py b/src/ostbuild/pyostbuild/builtin_compose.py
new file mode 100755
index 0000000..52dbb8f
--- /dev/null
+++ b/src/ostbuild/pyostbuild/builtin_compose.py
@@ -0,0 +1,110 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from .subprocess_helpers import run_sync_monitor_log_file
+from . import ostbuildrc
+from . import buildutil
+from . import fileutil
+from . import kvfile
+from . import odict
+from . import vcs
+
+class OstbuildCompose(builtins.Builtin):
+ name = "compose"
+ short_description = "Build complete trees from components"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def _compose_one_target(self, bin_snapshot, target):
+ components = bin_snapshot['components']
+ base = target['base']
+ base_name = 'bases/%s' % (base['name'], )
+ base_revision = target['base']['ostree-revision']
+
+ compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
+ if os.path.isdir(compose_rootdir):
+ shutil.rmtree(compose_rootdir)
+ os.mkdir(compose_rootdir)
+
+ compose_contents = [(base_revision, '/')]
+ for tree_content in target['contents']:
+ name = tree_content['name']
+ rev = components[name]
+ subtrees = tree_content['trees']
+ for subpath in subtrees:
+ compose_contents.append((rev, subpath))
+
+ (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+ f = os.fdopen(fd, 'w')
+ for (branch, subpath) in compose_contents:
+ f.write(branch)
+ f.write('\0')
+ f.write(subpath)
+ f.write('\0')
+ f.close()
+
+ link_cache_dir = os.path.join(self.workdir, 'link-cache')
+ fileutil.ensure_dir(link_cache_dir)
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'checkout', '--link-cache=' + link_cache_dir,
+ '--user-mode', '--no-triggers',
+ '--union', '--from-stdin', compose_rootdir],
+ stdin=open(tmppath))
+ os.unlink(tmppath)
+
+ contents_path = os.path.join(compose_rootdir, 'contents.json')
+ f = open(contents_path, 'w')
+ json.dump(bin_snapshot, f, indent=4, sort_keys=True)
+ f.close()
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'commit', '-b', target['name'], '-s', 'Compose',
+ '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
+ '--skip-if-unchanged'], cwd=compose_rootdir)
+ shutil.rmtree(compose_rootdir)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--bin-snapshot')
+
+ args = parser.parse_args(argv)
+ self.args = args
+
+ self.parse_config()
+ self.parse_bin_snapshot(args.prefix, args.bin_snapshot)
+
+ log("Using binary snapshot: %s" % (os.path.basename(self.bin_snapshot_path), ))
+
+ for target in self.bin_snapshot['targets']:
+ log("Composing target %r from %u components" % (target['name'],
+ len(target['contents'])))
+ self._compose_one_target(self.bin_snapshot, target)
+
+builtins.register(OstbuildCompose)
diff --git a/src/ostbuild/pyostbuild/builtin_modify_snapshot.py b/src/ostbuild/pyostbuild/builtin_modify_snapshot.py
new file mode 100755
index 0000000..b438a9a
--- /dev/null
+++ b/src/ostbuild/pyostbuild/builtin_modify_snapshot.py
@@ -0,0 +1,63 @@
+# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+from StringIO import StringIO
+import json
+import select,time
+import argparse
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+
+class OstbuildModifySnapshot(builtins.Builtin):
+ name = "modify-snapshot"
+ short_description = "Change the current source snapshot"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--src-snapshot')
+
+ args = parser.parse_args(argv)
+
+ self.parse_config()
+ self.parse_snapshot(args.prefix, args.src_snapshot)
+
+ component_name = self.get_component_from_cwd()
+ current_meta = self.get_component_meta(component_name)
+
+ new_snapshot = dict(self.snapshot)
+ new_meta = dict(current_meta)
+ if 'patches' in new_meta:
+ del new_meta['patches']
+ new_meta['src'] = "dirty-git:%s" % (os.getcwd(), )
+ new_meta['revision'] = run_sync_get_output(['git', 'rev-parse', 'HEAD'])
+
+ new_snapshot['components'][component_name] = new_meta
+
+ db = self.get_src_snapshot_db()
+ path = db.store(new_snapshot)
+ log("Replaced %s with %s %s" % (component_name, new_meta['src'],
+ new_meta['revision']))
+ log("New source snapshot: %s" % (path, ))
+
+builtins.register(OstbuildModifySnapshot)
diff --git a/src/ostbuild/pyostbuild/builtin_query_content.py b/src/ostbuild/pyostbuild/builtin_prefix.py
similarity index 52%
rename from src/ostbuild/pyostbuild/builtin_query_content.py
rename to src/ostbuild/pyostbuild/builtin_prefix.py
index 89f5839..4ffa981 100755
--- a/src/ostbuild/pyostbuild/builtin_query_content.py
+++ b/src/ostbuild/pyostbuild/builtin_prefix.py
@@ -28,40 +28,46 @@ from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
-class OstbuildQueryContent(builtins.Builtin):
- name = "query-content"
- short_description = "Output metadata from a component"
+class OstbuildPrefix(builtins.Builtin):
+ name = "prefix"
+ short_description = "Display or modify \"prefix\" (build target)"
def __init__(self):
builtins.Builtin.__init__(self)
+ def _set_prefix(self, prefix):
+ f = open(self.path, 'w')
+ f.write(prefix)
+ f.write('\n')
+ f.close()
+ log("Prefix is now %r" % (prefix, ))
+
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--branch', required=True)
- parser.add_argument('--component')
+ parser.add_argument('-a', '--active', action='store_true')
+ parser.add_argument('prefix', nargs='?', default=None)
args = parser.parse_args(argv)
- self.args = args
- self.parse_config()
- contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'cat', args.branch, 'contents.json'])
-
- if args.component is None:
- sys.stdout.write(contents_json_text)
+ self.path = os.path.expanduser('~/.config/ostbuild-prefix')
+ if args.prefix is None and not args.active:
+ if os.path.exists(self.path):
+ f = open(self.path)
+ print "%s" % (f.read().strip(), )
+ f.close()
+ else:
+ log("No currently active prefix")
+ elif args.prefix is not None and args.active:
+ fatal("Can't specify -a with prefix")
+ elif args.prefix is not None:
+ self._set_prefix(args.prefix)
else:
- contents = json.loads(contents_json_text)
- contents_list = contents['contents']
- found = False
- for content in contents_list:
- if content['name'] != args.component:
- found = True
- break
- if not found:
- fatal("Unknown component '%s'" % (args.component, ))
- ostbuildmeta_json = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'cat', content['ostree-revision'],
- '/_ostbuild-meta.json'])
- sys.stdout.write(ostbuildmeta_json)
+ assert args.active
+
+ self.parse_active_branch()
+
+ active_prefix = self.active_branch_contents['prefix']
+
+ self._set_prefix(active_prefix)
-builtins.register(OstbuildQueryContent)
+builtins.register(OstbuildPrefix)
diff --git a/src/ostbuild/pyostbuild/builtin_pull_components.py b/src/ostbuild/pyostbuild/builtin_pull_components.py
new file mode 100755
index 0000000..7464b1e
--- /dev/null
+++ b/src/ostbuild/pyostbuild/builtin_pull_components.py
@@ -0,0 +1,70 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import copy
+import argparse
+import json
+import time
+import urlparse
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from . import ostbuildrc
+from . import buildutil
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import kvfile
+from . import odict
+
+class OstbuildPullComponents(builtins.Builtin):
+ name = "pull-components"
+ short_description = "Download the component data for active branch"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('targets', nargs='*')
+
+ args = parser.parse_args(argv)
+
+ self.parse_active_branch()
+
+ if len(args.targets) == 0:
+ targets = [self.active_branch]
+ else:
+ targets = args.targets
+
+ tree_contents_list = []
+ for target in targets:
+ tree_contents_path = os.path.join(self.ostree_dir, target, 'contents.json')
+ tree_contents = json.load(open(tree_contents_path))
+ tree_contents_list.append(tree_contents)
+ revisions = set()
+ for tree_contents in tree_contents_list:
+ for content_item in tree_contents['contents']:
+ revisions.add(content_item['ostree-revision'])
+ args = ['ostree-pull', '--repo=' + self.repo]
+ # FIXME FIXME - don't hardcode origin here
+ args.append('gnome')
+ for revision in revisions:
+ args.append(revision)
+ run_sync(args)
+
+builtins.register(OstbuildPullComponents)
diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py
index 3486396..4ddd1ad 100755
--- a/src/ostbuild/pyostbuild/builtin_resolve.py
+++ b/src/ostbuild/pyostbuild/builtin_resolve.py
@@ -19,6 +19,7 @@ import os,sys,subprocess,tempfile,re,shutil
import copy
import argparse
import json
+import time
import urlparse
from StringIO import StringIO
@@ -26,6 +27,7 @@ from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import ostbuildrc
+from . import jsondb
from . import buildutil
from . import kvfile
from . import odict
@@ -38,6 +40,7 @@ class OstbuildResolve(builtins.Builtin):
builtins.Builtin.__init__(self)
def _ensure_vcs_mirror(self, name, keytype, uri, branch):
+ # FIXME - remove "name" from parameter list here - hash uri?
mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri)
tmp_mirror = mirror + '.tmp'
if os.path.isdir(tmp_mirror):
@@ -135,6 +138,7 @@ class OstbuildResolve(builtins.Builtin):
self.args = args
self.parse_config()
+ self.repo = ostbuildrc.get_key('repo')
manifest_path = self.ostbuildrc.get_key('manifest')
self.manifest = json.load(open(manifest_path))
@@ -192,52 +196,52 @@ class OstbuildResolve(builtins.Builtin):
component['patches'] = copy.deepcopy(global_patches_meta)
component['patches']['files'] = patch_files
- name_prefix = snapshot['name-prefix']
- base_prefix = snapshot['base-prefix']
-
manifest_architectures = snapshot['architectures']
+ ostree_prefix = snapshot['prefix']
+ base_prefix = '%s/%s' % (snapshot['base']['name'], ostree_prefix)
+
+ snapshot['architecture-buildroots'] = {}
+ for architecture in manifest_architectures:
+ snapshot['architecture-buildroots'][architecture] = '%s-%s-devel' % (base_prefix, architecture)
+
components_by_name = {}
component_ordering = []
- build_prev_component_by_arch = {}
- runtime_prev_component_by_arch = {}
- runtime_components_by_arch = {}
- devel_components_by_arch = {}
- for architecture in manifest_architectures:
- runtime_components_by_arch[architecture] = []
- devel_components_by_arch[architecture] = []
+ build_prev_component = None
+ runtime_prev_component = None
+ runtime_components = []
+ devel_components = []
- for component in component_source_list:
- component_architectures = component.get('architectures', manifest_architectures)
- for architecture in component_architectures:
- component_binary = copy.deepcopy(component)
- source_name = component['name']
- binary_name = '%s/%s/%s' % (name_prefix, architecture, source_name)
- component_binary['name'] = binary_name
- component_binary['architecture'] = architecture
+ builds = {}
- components_by_name[binary_name] = component_binary
+ for component in component_source_list:
+ base_name = component['name']
+ name = '%s/%s' % (ostree_prefix, base_name)
+ component['name'] = name
- prev_component = build_prev_component_by_arch.get(architecture)
- if prev_component is not None:
- component_binary['build-depends'] = [prev_component['name']]
- build_prev_component_by_arch[architecture] = component_binary
+ components_by_name[name] = component
- is_runtime = component.get('component', 'runtime') == 'runtime'
+ if build_prev_component is not None:
+ component['build-depends'] = [build_prev_component['name']]
+ build_prev_component = component
- prev_component = runtime_prev_component_by_arch.get(architecture)
- if prev_component is not None:
- component_binary['runtime-depends'] = [prev_component['name']]
+ is_runtime = component.get('component', 'runtime') == 'runtime'
- if is_runtime:
- runtime_prev_component_by_arch[architecture] = component_binary
+ if runtime_prev_component is not None:
+ component['runtime-depends'] = [runtime_prev_component['name']]
- if is_runtime:
- runtime_components_by_arch[architecture].append(component_binary)
- devel_components_by_arch[architecture].append(component_binary)
+ if is_runtime:
+ runtime_prev_component = component
+ runtime_components.append(component)
+ devel_components.append(component)
- if 'architectures' in component_binary:
- del component_binary['architectures']
+ is_noarch = component.get('noarch', False)
+ if is_noarch:
+ # Just use the first specified architecture
+ component_arches = [manifest_architectures[0]]
+ else:
+ component_arches = component.get('architectures', manifest_architectures)
+ builds[name] = component_arches
# We expanded these keys
del snapshot['config-opts']
@@ -247,26 +251,27 @@ class OstbuildResolve(builtins.Builtin):
targets_list = []
snapshot['targets'] = targets_list
- for architecture in manifest_architectures:
- for target_component_type in ['runtime', 'devel']:
+ for target_component_type in ['runtime', 'devel']:
+ for architecture in manifest_architectures:
target = {}
targets_list.append(target)
- target['name'] = '%s-%s-%s' % (name_prefix, architecture, target_component_type)
+ target['name'] = '%s-%s-%s' % (ostree_prefix, architecture, target_component_type)
base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
- base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', 'bases/%s' % (base_ref, )])
target['base'] = {'name': base_ref}
if target_component_type == 'runtime':
- target_components = runtime_components_by_arch[architecture]
+ target_components = runtime_components
else:
- target_components = devel_components_by_arch[architecture]
+ target_components = devel_components
contents = []
for component in target_components:
- name = component['name']
- component_ref = {'name': name}
+ builds_for_component = builds[component['name']]
+ if architecture not in builds_for_component:
+ continue
+ binary_name = '%s/%s' % (component['name'], architecture)
+ component_ref = {'name': binary_name}
if target_component_type == 'runtime':
component_ref['trees'] = ['/runtime']
else:
@@ -278,10 +283,12 @@ class OstbuildResolve(builtins.Builtin):
del component['name']
snapshot['components'] = components_by_name
- out_snapshot = os.path.join(self.workdir, '%s-snapshot.json' % (name_prefix, ))
- f = open(out_snapshot, 'w')
- json.dump(snapshot, f, indent=4, sort_keys=True)
- f.close()
- print "Created: %s" % (out_snapshot, )
+ snapshot['00ostree-src-snapshot-version'] = 0
+
+ current_time = time.time()
+
+ src_db = self.get_src_snapshot_db()
+ path = src_db.store(snapshot)
+ log("Source snapshot: %s" % (path, ))
builtins.register(OstbuildResolve)
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index 64f7be4..c71431f 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -19,11 +19,15 @@
import os
import sys
+import stat
import argparse
import json
from . import ostbuildrc
+from . import fileutil
+from . import jsondb
from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
_all_builtins = {}
@@ -31,26 +35,167 @@ class Builtin(object):
name = None
short_description = None
+ def __init__(self):
+ self._meta_cache = {}
+ self.prefix = None
+ self.manifest = None
+ self.snapshot = None
+ self.bin_snapshot = None
+ self.repo = None
+ self.ostree_dir = self._find_ostree_dir()
+ (self.active_branch, self.active_branch_checksum) = self._find_active_branch()
+ self._src_snapshots = None
+ self._bin_snapshots = None
+
+ def _find_ostree_dir(self):
+ for path in ['/ostree', '/sysroot/ostree']:
+ if os.path.isdir(path):
+ return path
+ return None
+
+ def _find_active_branch(self):
+ if self.ostree_dir is None:
+ return (None, None)
+ current_path = os.path.join(self.ostree_dir, 'current')
+ while True:
+ try:
+ target = os.path.join(self.ostree_dir, current_path)
+ stbuf = os.lstat(target)
+ except OSError, e:
+ current_path = None
+ break
+ if not stat.S_ISLNK(stbuf.st_mode):
+ break
+ current_path = os.readlink(target)
+ if current_path is not None:
+ basename = os.path.basename(current_path)
+ return basename.rsplit('-', 1)
+ else:
+ return (None, None)
+
+ def get_component_from_cwd(self):
+ cwd = os.getcwd()
+ parent = os.path.dirname(cwd)
+ parentparent = os.path.dirname(parent)
+ return '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
+
def parse_config(self):
self.ostbuildrc = ostbuildrc
- self.repo = ostbuildrc.get_key('repo')
+
self.mirrordir = ostbuildrc.get_key('mirrordir')
if not os.path.isdir(self.mirrordir):
fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, ))
self.workdir = ostbuildrc.get_key('workdir')
if not os.path.isdir(self.workdir):
fatal("Specified workdir '%s' is not a directory" % (self.workdir, ))
+
+ self.snapshot_dir = os.path.join(self.workdir, 'snapshots')
self.patchdir = os.path.join(self.workdir, 'patches')
- def parse_manifest(self):
- self.manifest_path = ostbuildrc.get_key('manifest')
- self.manifest = json.load(open(self.manifest_path))
- self.name_prefix = self.manifest['name-prefix']
+ def parse_active_branch(self):
+ if self.ostree_dir is None:
+ fatal("/ostree directory not found")
+ repo_path = os.path.join(self.ostree_dir, 'repo')
+ if not os.path.isdir(repo_path):
+ fatal("Repository '%s' doesn't exist" % (repo_path, ))
+ self.repo = repo_path
+ if self.active_branch is None:
+ fatal("No \"current\" link found")
+ branch_path = os.path.join(self.ostree_dir, self.active_branch)
+ contents_path = os.path.join(branch_path, 'contents.json')
+ f = open(contents_path)
+ self.active_branch_contents = json.load(f)
+ f.close()
+
+ def get_component_snapshot(self, name):
+ found = False
+ for content in self.active_branch_contents['contents']:
+ if content['name'] == name:
+ found = True
+ break
+ if not found:
+ fatal("Unknown component '%s'" % (name, ))
+ return content
+
+ def get_component_meta_from_revision(self, revision):
+ text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'cat', revision,
+ '/_ostbuild-meta.json'])
+ return json.loads(text)
+
+ def get_component_meta(self, name):
+ assert self.repo is not None
- def parse_snapshot(self):
- self.parse_manifest()
- snapshot_path = os.path.join(self.workdir, '%s-snapshot.json' % (self.name_prefix, ))
- self.snapshot = json.load(open(snapshot_path))
+ if self.snapshot is not None:
+ return self.snapshot['components'][name]
+
+ meta = self._meta_cache.get(name)
+ if meta is None:
+ content = self.get_component_snapshot(name)
+ meta = self.get_component_meta_from_revision(content['ostree-revision'])
+ self._meta_cache[name] = meta
+ return meta
+
+ def get_prefix(self):
+ if self.prefix is None:
+ path = os.path.expanduser('~/.config/ostbuild-prefix')
+ if not os.path.exists(path):
+ fatal("No prefix set; use \"ostbuild prefix\" to set one")
+ f = open(path)
+ self.prefix = f.read().strip()
+ f.close()
+ return self.prefix
+
+ def create_db(self, dbsuffix, prefix=None):
+ if prefix is None:
+ target_prefix = self.get_prefix()
+ else:
+ target_prefix = prefix
+ name = '%s-%s' % (target_prefix, dbsuffix)
+ fileutil.ensure_dir(self.snapshot_dir)
+ return jsondb.JsonDB(self.snapshot_dir, prefix=name)
+
+ def get_src_snapshot_db(self):
+ if self._src_snapshots is None:
+ self._src_snapshots = self.create_db('src-snapshot')
+ return self._src_snapshots
+
+ def get_bin_snapshot_db(self):
+ if self._bin_snapshots is None:
+ self._bin_snapshots = self.create_db('bin-snapshot')
+ return self._bin_snapshots
+
+ def parse_snapshot(self, prefix, path):
+ if prefix is not None:
+ self.prefix = prefix
+ self.repo = ostbuildrc.get_key('repo')
+ if path is None:
+ latest_path = self.get_src_snapshot_db().get_latest_path()
+ if latest_path is None:
+ raise Exception("No source snapshot found for prefix %r" % (self.prefix, ))
+ self.snapshot_path = latest_path
+ else:
+ self.snapshot_path = path
+ self.snapshot = json.load(open(self.snapshot_path))
+ src_ver = self.snapshot['00ostree-src-snapshot-version']
+ if src_ver != 0:
+ fatal("Unhandled 00ostree-src-snapshot-version \"%d\", expected 0", src_ver)
+
+ def parse_bin_snapshot(self, prefix, path):
+ if prefix is not None:
+ self.prefix = prefix
+ self.repo = ostbuildrc.get_key('repo')
+ if path is None:
+ latest_path = self.get_bin_snapshot_db().get_latest_path()
+ if latest_path is None:
+ raise Exception("No binary snapshot found for prefix %r" % (self.prefix, ))
+ self.bin_snapshot_path = latest_path
+ else:
+ self.bin_snapshot_path = path
+ self.bin_snapshot = json.load(open(self.bin_snapshot_path))
+ bin_ver = self.bin_snapshot['00ostree-bin-snapshot-version']
+ if bin_ver != 0:
+ fatal("Unhandled 00ostree-bin-snapshot-version \"%d\", expected 0", bin_ver)
def execute(self, args):
raise NotImplementedError()
@@ -65,4 +210,4 @@ def get(name):
return None
def get_all():
- return _all_builtins.itervalues()
+ return sorted(_all_builtins.itervalues(), lambda a, b: cmp(a.name, b.name))
diff --git a/src/ostbuild/pyostbuild/jsondb.py b/src/ostbuild/pyostbuild/jsondb.py
new file mode 100644
index 0000000..aecee35
--- /dev/null
+++ b/src/ostbuild/pyostbuild/jsondb.py
@@ -0,0 +1,110 @@
+#
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import stat
+import time
+import tempfile
+import re
+import shutil
+import hashlib
+import json
+
+class JsonDB(object):
+ def __init__(self, dirpath, prefix):
+ self._dirpath = dirpath
+ self._prefix = prefix
+ self._version_csum_re = re.compile(r'-(\d+)\.(\d+)-([0-9a-f]+).json$')
+
+ def _cmp_match_by_version(self, a, b):
+ # Note this is a reversed comparison; bigger is earlier
+ a_major = a[0]
+ a_minor = a[1]
+ b_major = b[0]
+ b_minor = b[1]
+
+ c = cmp(b_major, a_major)
+ if c == 0:
+ return cmp(b_minor, a_minor)
+ return 0
+
+ def _get_all(self):
+ result = []
+ for fname in os.listdir(self._dirpath):
+ if not (fname.startswith(self._prefix) and fname.endswith('.json')):
+ continue
+
+ path = os.path.join(self._dirpath, fname)
+ match = self._version_csum_re.search(fname)
+ if not match:
+ raise Exception("Invalid file '%s' in JsonDB; doesn't contain version+checksum",
+ path)
+ result.append((int(match.group(1)), int(match.group(2)), match.group(3), fname))
+ result.sort(self._cmp_match_by_version)
+ return result
+
+ def get_latest(self):
+ path = self.get_latest_path()
+ if path is None:
+ return None
+ return json.load(open(path))
+
+ def get_latest_path(self):
+ files = self._get_all()
+ if len(files) == 0:
+ return None
+ return os.path.join(self._dirpath, files[0][3])
+
+ def store(self, obj):
+ files = self._get_all()
+ if len(files) == 0:
+ latest = None
+ else:
+ latest = files[0]
+
+ current_time = time.gmtime()
+
+ (fd, tmppath) = tempfile.mkstemp(suffix='.tmp', prefix='tmp-jsondb-')
+ os.close(fd)
+ f = open(tmppath, 'w')
+ json.dump(obj, f, indent=4, sort_keys=True)
+ f.close()
+
+ csum = hashlib.sha256()
+ f = open(tmppath)
+ buf = f.read(8192)
+ while buf != '':
+ csum.update(buf)
+ buf = f.read(8192)
+ f.close()
+ digest = csum.hexdigest()
+
+ if latest is not None:
+ if digest == latest[2]:
+ return latest[3]
+ latest_version = (latest[0], latest[1])
+ else:
+ latest_version = (current_time.tm_year, 0)
+ target_name = '%s-%d.%d-%s.json' % (self._prefix, current_time.tm_year,
+ latest_version[1] + 1, digest)
+ target_path = os.path.join(self._dirpath, target_name)
+ os.rename(tmppath, target_path)
+ return target_path
+
+
+
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 842cb5b..63827c1 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -22,13 +22,17 @@ import sys
import argparse
from . import builtins
-from . import builtin_build
+from . import builtin_bin_to_src
+from . import builtin_build_components
+from . import builtin_branch_prefix
from . import builtin_checkout
from . import builtin_chroot_compile_one
-from . import builtin_chroot_run_triggers
+from . import builtin_compose
from . import builtin_compile_one
-from . import builtin_query_content
+from . import builtin_pull_components
+from . import builtin_prefix
from . import builtin_resolve
+from . import builtin_modify_snapshot
from . import builtin_status
def usage(ecode):
diff --git a/src/ostbuild/pyostbuild/ostbuildlog.py b/src/ostbuild/pyostbuild/ostbuildlog.py
index a4af2a6..0f85229 100755
--- a/src/ostbuild/pyostbuild/ostbuildlog.py
+++ b/src/ostbuild/pyostbuild/ostbuildlog.py
@@ -20,12 +20,16 @@
import os
import sys
-def log(msg):
- fullmsg = '%s: %s\n' % (os.path.basename(sys.argv[0]), msg)
+def log(msg, prefix=None):
+ if prefix is None:
+ prefix_target = ''
+ else:
+ prefix_target = prefix
+ fullmsg = '%s: %s%s\n' % (os.path.basename(sys.argv[0]), prefix_target, msg)
sys.stdout.write(fullmsg)
sys.stdout.flush()
def fatal(msg):
- log(msg)
+ log(msg, prefix="FATAL: ")
sys.exit(1)
diff --git a/src/ostbuild/pyostbuild/subprocess_helpers.py b/src/ostbuild/pyostbuild/subprocess_helpers.py
index 116d0fb..3754900 100755
--- a/src/ostbuild/pyostbuild/subprocess_helpers.py
+++ b/src/ostbuild/pyostbuild/subprocess_helpers.py
@@ -70,14 +70,16 @@ def run_sync_get_output(args, cwd=None, env=None, stdout=None, stderr=None, none
return None
def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False,
- log_success=True, log_initiation=True, stdout=None,
+ log_success=True, log_initiation=True, stdin=None, stdout=None,
stderr=None):
if log_initiation:
log("running: %s" % (subprocess.list2cmdline(args),))
env_copy = _get_env_for_cwd(cwd, env)
- if keep_stdin:
+ if stdin is not None:
+ stdin_target = stdin
+ elif keep_stdin:
stdin_target = sys.stdin
else:
stdin_target = open('/dev/null', 'r')
diff --git a/src/ostbuild/pyostbuild/vcs.py b/src/ostbuild/pyostbuild/vcs.py
index 10c7e39..0716122 100755
--- a/src/ostbuild/pyostbuild/vcs.py
+++ b/src/ostbuild/pyostbuild/vcs.py
@@ -71,5 +71,5 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
return dest
def clean(keytype, checkoutdir):
- assert keytype == 'git'
+ assert keytype in ('git', 'dirty-git')
run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
