[ostree/wip/multibuild] gnomeos: Lots more ostbuild work
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree/wip/multibuild] gnomeos: Lots more ostbuild work
- Date: Wed, 28 Dec 2011 16:33:04 +0000 (UTC)
commit 1cec9578f9f1158e254568ba09f7c976ebf4bee6
Author: Colin Walters <walters verbum org>
Date: Wed Dec 28 11:32:25 2011 -0500
gnomeos: Lots more ostbuild work
Makefile-ostbuild.am | 1 +
gnomeos/yocto/commit-yocto-build.sh | 10 +++--
src/ostbuild/pyostbuild/buildutil.py | 10 ++---
src/ostbuild/pyostbuild/builtin_build.py | 41 ++++++++++----------
.../pyostbuild/builtin_chroot_compile_one.py | 4 +-
.../pyostbuild/builtin_commit_artifacts.py | 4 +-
src/ostbuild/pyostbuild/builtin_compile_one.py | 14 +++---
src/ostbuild/pyostbuild/subprocess_helpers.py | 38 ++++++++++++++++++
8 files changed, 80 insertions(+), 42 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index e848322..42074e2 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -28,6 +28,7 @@ pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/builtin_commit_artifacts.py \
src/ostbuild/pyostbuild/builtin_compile_one.py \
src/ostbuild/pyostbuild/builtins.py \
+ src/ostbuild/pyostbuild/__init__.py \
src/ostbuild/pyostbuild/kvfile.py \
src/ostbuild/pyostbuild/main.py \
src/ostbuild/pyostbuild/ostbuildlog.py \
diff --git a/gnomeos/yocto/commit-yocto-build.sh b/gnomeos/yocto/commit-yocto-build.sh
index f622d36..6c79d7d 100755
--- a/gnomeos/yocto/commit-yocto-build.sh
+++ b/gnomeos/yocto/commit-yocto-build.sh
@@ -26,16 +26,18 @@ BRANCH=$1
test -n "$BRANCH" || usage
shift
-ARCH=x86
+YOCTO_ARCH=x86
+MACHINE=i686
+REF="bases/gnomeos-3.4-yocto-${MACHINE}-${BRANCH}"
OSTREE_VER=$(cd $SCRIPT_SRCDIR && git describe)
BUILDDIR=$WORKDIR/tmp-eglibc
OSTREE_REPO=$WORKDIR/repo
-BUILD_TAR=$BUILDDIR/deploy/images/gnomeos-contents-$BRANCH-qemu${ARCH}.tar.gz
+BUILD_TAR=$BUILDDIR/deploy/images/gnomeos-contents-$BRANCH-qemu${YOCTO_ARCH}.tar.gz
BUILD_TIME=$(date -r $BUILD_TAR)
-ostree --repo=${OSTREE_REPO} commit --skip-if-unchanged -s "Build from OSTree ${OSTREE_VER}" -b "gnomeos-yocto-$ARCH-$BRANCH" --tree=tar=${BUILD_TAR}
-ostree --repo=${OSTREE_REPO} diff "gnomeos-yocto-$ARCH-$BRANCH"^ "gnomeos-yocto-$ARCH-$BRANCH"
+ostree --repo=${OSTREE_REPO} commit --skip-if-unchanged -s "Build from OSTree ${OSTREE_VER}" -b "${REF}" --tree=tar=${BUILD_TAR}
+ostree --repo=${OSTREE_REPO} diff "${REF}"^ "${REF}" || true
diff --git a/src/ostbuild/pyostbuild/buildutil.py b/src/ostbuild/pyostbuild/buildutil.py
index 2c0b021..26bea32 100755
--- a/src/ostbuild/pyostbuild/buildutil.py
+++ b/src/ostbuild/pyostbuild/buildutil.py
@@ -30,11 +30,9 @@ def parse_artifact_name(artifact_basename):
'version': match.group(5),
'type': match.group(6)}
-def branch_name_for_build(buildroot, name, branch):
- return 'artifacts/%s/%s/%s' % (buildroot, name, branch)
-
def branch_name_for_artifact(a):
- return branch_name_for_build(a['buildroot'],
- a['name'],
- a['branch'])
+ return 'artifacts/%s/%s/%s/%s' % (a['buildroot'],
+ a['name'],
+ a['branch'],
+ a['type'])
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index c063f30..6950108 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -21,7 +21,7 @@ import json
from . import builtins
from .ostbuildlog import log, fatal
-from .subprocess_helpers import run_sync
+from .subprocess_helpers import run_sync, run_sync_get_output
from . import ostbuildrc
from . import buildutil
from . import kvfile
@@ -80,25 +80,21 @@ class OstbuildBuild(builtins.Builtin):
component_src = self._ensure_vcs_checkout(name, keytype, uri, branch)
buildroot = '%s-%s-devel' % (self.manifest['name'], architecture)
branchname = 'artifacts/%s/%s/%s' % (buildroot, name, branch)
- try:
- previous_commit_version = subprocess.check_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', branchname],
- stderr=open('/dev/null', 'w'))
- previous_commit_version = previous_commit_version.strip()
- log("Previous build of '%s' is %s" % (branchname, previous_commit_version))
- except subprocess.CalledProcessError, e:
- previous_commit_version = None
- log("No previous build for '%s' found" % (branchname, ))
+ current_buildroot_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', buildroot])
+ current_buildroot_version = current_buildroot_version.strip()
+ previous_commit_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', branchname],
+ stderr=open('/dev/null', 'w'),
+ none_on_error=True)
if previous_commit_version is not None:
- previous_artifact_version = subprocess.check_output(['ostree', '--repo=' + self.repo,
- 'show', '--print-metadata-key=ostbuild-artifact-version', previous_commit_version])
+ log("Previous build of '%s' is %s" % (branchname, previous_commit_version))
+ previous_artifact_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'show', '--print-metadata-key=ostbuild-artifact-version', previous_commit_version])
previous_artifact_version = previous_artifact_version.strip()
- previous_buildroot_version = subprocess.check_output(['ostree', '--repo=' + self.repo,
- 'show', '--print-metadata-key=ostbuild-buildroot-version', previous_commit_version])
+ previous_buildroot_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'show', '--print-metadata-key=ostbuild-buildroot-version', previous_commit_version])
previous_buildroot_version = previous_buildroot_version.strip()
- current_buildroot_version = subprocess.check_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', buildroot])
- current_buildroot_version = current_buildroot_version.strip()
previous_vcs_version = self._parse_artifact_vcs_version(previous_artifact_version)
current_vcs_version = self._get_vcs_version_from_checkout(name)
@@ -116,8 +112,10 @@ class OstbuildBuild(builtins.Builtin):
return
else:
log("Buildroot is now '%s'" % (current_buildroot_version, ))
+ else:
+ log("No previous build for '%s' found" % (branchname, ))
- component_resultdir = os.path.join(self.workdir, 'name', 'results')
+ component_resultdir = os.path.join(self.workdir, name, 'results')
if os.path.isdir(component_resultdir):
shutil.rmtree(component_resultdir)
os.makedirs(component_resultdir)
@@ -136,13 +134,14 @@ class OstbuildBuild(builtins.Builtin):
artifact_files = []
for name in os.listdir(component_resultdir):
if name.startswith('artifact-'):
+ log("Generated artifact file: %s" % (name, ))
artifact_files.append(os.path.join(component_resultdir, name))
assert len(artifact_files) >= 1 and len(artifact_files) <= 2
run_sync(['ostbuild', 'commit-artifacts',
'--repo=' + self.repo] + artifact_files)
artifacts = []
for filename in artifact_files:
- parsed = buildutil.parse_artifact_name(name)
+ parsed = buildutil.parse_artifact_name(os.path.basename(filename))
artifacts.append(parsed)
def _sort_artifact(a, b):
if a['type'] == b['type']:
@@ -154,7 +153,7 @@ class OstbuildBuild(builtins.Builtin):
return artifacts
def _compose(self, suffix, artifacts):
- compose_contents = [self.manifest['base'] + '-' + suffix]
+ compose_contents = ['bases/' + self.manifest['base'] + '-' + suffix]
compose_contents.extend(artifacts)
child_args = ['ostree', '--repo=' + self.repo, 'compose',
'-b', self.manifest['name'] + '-' + suffix, '-s', 'Compose']
@@ -199,6 +198,6 @@ class OstbuildBuild(builtins.Builtin):
devel_branches = map(buildutil.branch_name_for_artifact, devel_artifacts)
self._compose(architecture + '-devel', devel_branches)
runtime_branches = map(buildutil.branch_name_for_artifact, runtime_artifacts)
- self._compose(architecture + '-runtime', runtime_artifacts)
+ self._compose(architecture + '-runtime', runtime_branches)
builtins.register(OstbuildBuild)
diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
index cbb4fe2..8c18965 100755
--- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
@@ -67,7 +67,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
workdir_is_tmp = (args.workdir is None)
if workdir_is_tmp:
- workdir = tempfile.mkdtemp(prefix='ostree-chroot-compile-')
+ workdir = tempfile.mkdtemp(prefix='ostbuild-chroot-compile-')
else:
workdir = args.workdir
@@ -136,7 +136,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
'/bin/sh']
if not args.debug_shell:
child_args += ['-c',
- 'cd "%s" && ostbuild compile-one OSTBUILD_RESULTDIR=/ostbuild/results OSTBUILD_META=_ostbuild-meta' % (chroot_sourcedir, )
+ 'cd "%s" && ostbuild compile-one --ostbuild-resultdir=/ostbuild/results --ostbuild-meta=_ostbuild-meta' % (chroot_sourcedir, )
]
run_sync(child_args, env=BUILD_ENV)
diff --git a/src/ostbuild/pyostbuild/builtin_commit_artifacts.py b/src/ostbuild/pyostbuild/builtin_commit_artifacts.py
index 19ff99b..6f6f63c 100644
--- a/src/ostbuild/pyostbuild/builtin_commit_artifacts.py
+++ b/src/ostbuild/pyostbuild/builtin_commit_artifacts.py
@@ -51,8 +51,8 @@ class OstbuildCommitArtifacts(builtins.Builtin):
run_sync(['ostree', '--repo=' + args.repo,
'commit', '-b', branch_name, '-s', 'Build ' + parsed['version'],
- '--add-metadata-string=ostree-buildroot-version=' + parsed['buildroot_version'],
- '--add-metadata-string=ostree-artifact-version=' + parsed['version'],
+ '--add-metadata-string=ostbuild-buildroot-version=' + parsed['buildroot_version'],
+ '--add-metadata-string=ostbuild-artifact-version=' + parsed['version'],
'--skip-if-unchanged', '--tar-autocreate-parents', '--tree=tar=' + arg])
builtins.register(OstbuildCommitArtifacts)
diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py
index 3d083fd..ff87464 100755
--- a/src/ostbuild/pyostbuild/builtin_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_compile_one.py
@@ -90,9 +90,9 @@ class OstbuildCompileOne(builtins.Builtin):
for arg in args:
if arg.startswith('--ostbuild-resultdir='):
- self.ostbuild_resultdir=arg[len('ostbuild-resultdir='):]
- elif arg.startswith('ostbuild-meta='):
- self.ostbuild_meta=arg[len('ostbuild-meta='):]
+ self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):]
+ elif arg.startswith('--ostbuild-meta='):
+ self.ostbuild_meta=arg[len('--ostbuild-meta='):]
elif arg.startswith('--'):
self.configargs.append(arg)
else:
@@ -101,10 +101,10 @@ class OstbuildCompileOne(builtins.Builtin):
self.metadata = {}
if self.ostbuild_meta is None:
- output = subprocess.check_output(['ostbuild-autodiscover-meta'])
+ output = subprocess.check_output(['ostbuild', 'autodiscover-meta'])
ostbuild_meta_f = StringIO(output)
else:
- ostbuild_meta_f = open(ostbuild_meta)
+ ostbuild_meta_f = open(self.ostbuild_meta)
for line in ostbuild_meta_f:
(k,v) = line.split('=', 1)
@@ -196,7 +196,7 @@ class OstbuildCompileOne(builtins.Builtin):
artifact_prefix='artifact-%s,%s,%s,%s,%s' % (root_name, root_version, name, branch, version)
- tempdir = tempfile.mkdtemp(prefix='ostree-build-%s-' % (name,))
+ tempdir = tempfile.mkdtemp(prefix='ostbuild-%s-' % (name,))
self.tempfiles.append(tempdir)
args = ['make', 'install', 'DESTDIR=' + tempdir]
run_sync(args, cwd=builddir)
@@ -247,7 +247,7 @@ class OstbuildCompileOne(builtins.Builtin):
def make_artifact(self, name, from_files, tempdir=None, resultdir=None):
targz_name = name + '.tar.gz'
- (fd,filelist_temp)=tempfile.mkstemp(prefix='ostree-filelist-%s' % (name, ))
+ (fd,filelist_temp)=tempfile.mkstemp(prefix='ostbuild-filelist-%s' % (name, ))
os.close(fd)
self.tempfiles.append(filelist_temp)
f = open(filelist_temp, 'w')
diff --git a/src/ostbuild/pyostbuild/subprocess_helpers.py b/src/ostbuild/pyostbuild/subprocess_helpers.py
index 9d2476a..a05679e 100755
--- a/src/ostbuild/pyostbuild/subprocess_helpers.py
+++ b/src/ostbuild/pyostbuild/subprocess_helpers.py
@@ -23,6 +23,44 @@ import subprocess
from .ostbuildlog import log, fatal
+def _get_env_for_cwd(cwd=None, env=None):
+ # This dance is necessary because we want to keep the PWD
+ # environment variable up to date. Not doing so is a recipie
+ # for triggering edge conditions in pwd lookup.
+ if (cwd is not None) and (env is None or ('PWD' in env)):
+ if env is None:
+ env_copy = os.environ.copy()
+ else:
+ env_copy = env.copy()
+ if ('PWD' in env_copy) and (not cwd.startswith('/')):
+ env_copy['PWD'] = os.path.join(env_copy['PWD'], cwd)
+ else:
+ env_copy['PWD'] = cwd
+ else:
+ env_copy = env
+ return env_copy
+
+def run_sync_get_output(args, cwd=None, env=None, stderr=None, none_on_error=False):
+ log("running: %r" % (args,))
+ env_copy = _get_env_for_cwd(cwd, env)
+ f = open('/dev/null', 'r')
+ if stderr is None:
+ stderr_target = sys.stderr
+ else:
+ stderr_target = stderr
+ proc = subprocess.Popen(args, stdin=f, stdout=subprocess.PIPE, stderr=stderr_target,
+ close_fds=True, cwd=cwd, env=env_copy)
+ f.close()
+ output = proc.communicate()[0].strip()
+ if proc.returncode != 0 and not none_on_error:
+ logfn = fatal
+ else:
+ logfn = log
+ logfn("pid %d exited with code %d, %d bytes of output" % (proc.pid, proc.returncode, len(output)))
+ if proc.returncode == 0:
+ return output
+ return None
+
def run_sync(args, cwd=None, env=None):
log("running: %r" % (args,))
f = open('/dev/null', 'r')
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
