[ostree] ostbuild: Rework again, split into components.json and targets.json
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree] ostbuild: Rework again, split into components.json and targets.json
- Date: Tue, 13 Mar 2012 14:42:36 +0000 (UTC)
commit 4d1d6789c22af1d409fbd953f6aeea03e137c686
Author: Colin Walters <walters verbum org>
Date: Sat Mar 10 19:27:27 2012 -0500
ostbuild: Rework again, split into components.json and targets.json
The "resolve" builtin now does a lot more heavy lifting; we expand the
manifest.json, and "build" consequently is less intelligent now, more
of a low-level implementation.
Makefile-ostbuild.am | 1 +
gnomeos/3.4/manifest.json | 2 +-
src/ostbuild/pyostbuild/buildutil.py | 63 ++++++---
src/ostbuild/pyostbuild/builtin_build.py | 142 ++++++++-----------
src/ostbuild/pyostbuild/builtin_checkout.py | 15 ++-
.../pyostbuild/builtin_chroot_compile_one.py | 63 ++++----
src/ostbuild/pyostbuild/builtin_compile_one.py | 9 +-
src/ostbuild/pyostbuild/builtin_resolve.py | 151 ++++++++++++++------
src/ostbuild/pyostbuild/builtin_status.py | 10 +-
src/ostbuild/pyostbuild/builtins.py | 13 ++
src/ostbuild/pyostbuild/fileutil.py | 26 ++++
src/ostbuild/pyostbuild/vcs.py | 4 +
12 files changed, 303 insertions(+), 196 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index f45eb0a..cc5ed5e 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -32,6 +32,7 @@ pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/builtin_status.py \
src/ostbuild/pyostbuild/builtins.py \
src/ostbuild/pyostbuild/filemonitor.py \
+ src/ostbuild/pyostbuild/fileutil.py \
src/ostbuild/pyostbuild/__init__.py \
src/ostbuild/pyostbuild/kvfile.py \
src/ostbuild/pyostbuild/main.py \
diff --git a/gnomeos/3.4/manifest.json b/gnomeos/3.4/manifest.json
index 79dcdbc..16ad724 100644
--- a/gnomeos/3.4/manifest.json
+++ b/gnomeos/3.4/manifest.json
@@ -1,7 +1,7 @@
{
"name-prefix": "gnomeos-3.4",
"architectures": ["i686"],
- "base-prefix": "bases/yocto/gnomeos-3.4",
+ "base-prefix": "yocto/gnomeos-3.4",
"config-opts": ["--disable-static", "--disable-silent-rules"],
diff --git a/src/ostbuild/pyostbuild/buildutil.py b/src/ostbuild/pyostbuild/buildutil.py
index 79f0129..814e5e5 100755
--- a/src/ostbuild/pyostbuild/buildutil.py
+++ b/src/ostbuild/pyostbuild/buildutil.py
@@ -19,6 +19,7 @@ import os
import re
import urlparse
import tempfile
+import StringIO
from .subprocess_helpers import run_sync_get_output
@@ -75,23 +76,51 @@ def get_git_version_describe(dirpath, commit=None):
version = run_sync_get_output(args, cwd=dirpath)
return version.strip()
-def manifest_target(manifest):
- name = manifest['name']
- is_runtime = name.endswith('-runtime')
- # HACK - we should really name builds just like e.g. gnomeos-3.4-i686
- if is_runtime:
- return name[:-len('-runtime')] + '-devel'
- return name
-
-def manifest_buildname(manifest, component):
- return 'artifacts/%s/%s/%s' % (manifest_target(manifest),
- component['name'],
- component['branch'])
-
-def manifest_buildroot_name(manifest, component):
- return 'buildroots/%s/%s/%s' % (manifest_target (manifest),
- component['name'],
- component['branch'])
+def ref_to_unix_name(ref):
+ return ref.replace('/', '.')
+
+def tsort_components(components, key):
+ (fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-tsort-')
+ f = os.fdopen(fd, 'w')
+ for name,component in components.iteritems():
+ build_prev = component.get(key)
+ if (build_prev is not None and len(build_prev) > 0):
+ for dep_name in build_prev:
+ f.write('%s %s\n' % (name, dep_name))
+ f.close()
+
+ output = run_sync_get_output(['tsort', path])
+ os.unlink(path)
+ output_stream = StringIO.StringIO(output)
+ result = []
+ for line in output_stream:
+ result.append(line.strip())
+ return result
+
+def _recurse_depends(depkey, component_name, components, dep_names):
+ component = components[component_name]
+ depends = component.get(depkey)
+ if (depends is None or len(depends) == 0):
+ return
+ for depname in depends:
+ dep_names.add(depname)
+ _recurse_depends(depkey, depname, components, dep_names)
+
+def _sorted_depends(deptype, component_name, components):
+ dep_names = set()
+ _recurse_depends(deptype, component_name, components, dep_names)
+ dep_components = {}
+ for component_name in dep_names:
+ dep_components[component_name] = components[component_name]
+ result = tsort_components(dep_components, deptype)
+ result.reverse()
+ return result
+
+def build_depends(component_name, components):
+ return _sorted_depends('build-depends', component_name, components)
+
+def runtime_depends(component_name, components):
+ return _sorted_depends('runtime-depends', component_name, components)
def find_component_in_manifest(manifest, component_name):
for component in manifest['components']:
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index 2473660..88073bb 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -28,6 +28,7 @@ from .subprocess_helpers import run_sync, run_sync_get_output
from .subprocess_helpers import run_sync_monitor_log_file
from . import ostbuildrc
from . import buildutil
+from . import fileutil
from . import kvfile
from . import odict
from . import vcs
@@ -91,16 +92,13 @@ class OstbuildBuild(builtins.Builtin):
return result
- def _build_one_component(self, meta):
- name = meta['name']
- branch = meta['branch']
- architecture = meta['arch']
+ def _build_one_component(self, name, component):
+ branch = component['branch']
+ architecture = component['architecture']
- target = buildutil.manifest_target(self.manifest)
- buildname = buildutil.manifest_buildname(self.manifest, meta)
- buildroot_name = buildutil.manifest_buildroot_name(self.manifest, meta)
+ buildname = 'components/%s' % (name, )
- current_vcs_version = meta['revision']
+ current_vcs_version = component['revision']
previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', buildname],
@@ -130,31 +128,27 @@ class OstbuildBuild(builtins.Builtin):
checkoutdir = os.path.join(self.workdir, 'src')
component_src = os.path.join(checkoutdir, name)
- run_sync(['ostbuild', 'checkout', '--overwrite', '--manifest=' + self.manifest_path, name], cwd=checkoutdir)
+ run_sync(['ostbuild', 'checkout', '--clean', '--overwrite', name], cwd=checkoutdir)
- artifact_meta = dict(meta)
+ artifact_meta = dict(component)
metadata_path = os.path.join(component_src, '_ostbuild-meta.json')
f = open(metadata_path, 'w')
json.dump(artifact_meta, f, indent=4, sort_keys=True)
f.close()
- logdir = os.path.join(self.workdir, 'logs', 'compile', name)
- old_logdir = os.path.join(self.workdir, 'old-logs', 'compile', name)
- if not os.path.isdir(logdir):
- os.makedirs(logdir)
- if not os.path.isdir(old_logdir):
- os.makedirs(old_logdir)
- log_path = os.path.join(logdir, '%s.log' % (name, ))
+ logdir = os.path.join(self.workdir, 'logs', name)
+ fileutil.ensure_dir(logdir)
+ log_path = os.path.join(logdir, 'compile.log')
if os.path.isfile(log_path):
curtime = int(time.time())
- saved_name = '%s-%d.log' % (name, int(time.time()),)
- os.rename(log_path, os.path.join(old_logdir, saved_name))
+ saved_name = os.path.join(logdir, 'compile-prev.log')
+ os.rename(log_path, saved_name)
log("Logging to %s" % (log_path, ))
f = open(log_path, 'w')
chroot_args = self._get_ostbuild_chroot_args(architecture)
- chroot_args.extend(['--pristine', '--manifest=' + self.manifest_path])
+ chroot_args.extend(['--pristine', '--name=' + name])
if self.buildopts.shell_on_failure:
ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
if ecode != 0:
@@ -184,53 +178,45 @@ class OstbuildBuild(builtins.Builtin):
os.unlink(statoverride_path)
return True
- def _compose(self, components):
- base = self.manifest['base']
- base_branch = base['branch']
- base_revision = base['revision']
- # HACK
- manifest_build_name = self.manifest['name']
- is_runtime = manifest_build_name.endswith('-runtime')
-
+ def _compose(self, target):
+ base_name = 'bases/%s' % (target['base']['name'], )
branch_to_rev = {}
branch_to_subtrees = {}
- component_branches = []
- for component in components:
- branch = buildutil.manifest_buildname(self.manifest, component)
- component_branches.append(branch)
+ contents = [base_name]
+ branch_to_subtrees[base_name] = ['/']
+ base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', base_name])
- args = ['ostree', '--repo=' + self.repo,
- 'rev-parse']
- args.extend(component_branches)
+ branch_to_rev[base_name] = base_revision
+
+ args = ['ostree', '--repo=' + self.repo, 'rev-parse']
+ for component in target['contents']:
+ name = component['name']
+ contents.append(name)
+ args.append('components/%s' % (name, ))
+ branch_to_subtrees[name] = component['trees']
branch_revs_text = run_sync_get_output(args)
branch_revs = branch_revs_text.split('\n')
- for (branch, rev) in zip(component_branches, branch_revs):
- branch_to_rev[branch] = rev
-
- contents = [base_branch]
- branch_to_subtrees[base_branch] = ['/']
- branch_to_rev[base_branch] = base_revision
+ for (content, rev) in zip(target['contents'], branch_revs):
+ name = content['name']
+ branch_to_rev[name] = rev
- for branch in component_branches:
- contents.append(branch)
- subtrees = ['/runtime']
- branch_to_subtrees[branch] = subtrees
- if not is_runtime:
- # For now just hardcode docs going in devel
- subtrees.append('/doc')
- subtrees.append('/devel')
-
- compose_rootdir = os.path.join(self.workdir, 'roots', self.manifest['name'])
+ compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
if os.path.isdir(compose_rootdir):
shutil.rmtree(compose_rootdir)
os.mkdir(compose_rootdir)
- metadata_contents = []
+ resolved_base = dict(target['base'])
+ resolved_base['ostree-revision'] = base_revision
+ resolved_contents = list(target['contents'])
+ for component in resolved_contents:
+ component['ostree-revision'] = branch_to_rev[component['name']]
metadata = {'source': 'ostbuild compose v0',
- 'base': base,
- 'contents': metadata_contents}
+ 'base': resolved_base,
+ 'contents': resolved_contents}
+
for branch in contents:
branch_rev = branch_to_rev[branch]
subtrees = branch_to_subtrees[branch]
@@ -239,18 +225,14 @@ class OstbuildBuild(builtins.Builtin):
'checkout', '--user-mode',
'--union', '--subpath=' + subtree,
branch_rev, compose_rootdir])
- branch_meta = {'name': branch,
- 'rev': branch_rev,
- 'subtrees': subtrees}
- metadata_contents.append(branch_meta)
- contents_path = os.path.join(compose_rootdir, 'contents.json')
+ contents_path = os.path.join(compose_rootdir, 'manifest.json')
f = open(contents_path, 'w')
json.dump(metadata, f, indent=4, sort_keys=True)
f.close()
run_sync(['ostree', '--repo=' + self.repo,
- 'commit', '-b', self.manifest['name'], '-s', 'Compose',
+ 'commit', '-b', target['name'], '-s', 'Compose',
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--skip-if-unchanged'], cwd=compose_rootdir)
@@ -259,7 +241,6 @@ class OstbuildBuild(builtins.Builtin):
parser.add_argument('--skip-built', action='store_true')
parser.add_argument('--recompose', action='store_true')
parser.add_argument('--start-at')
- parser.add_argument('--manifest', required=True)
parser.add_argument('--shell-on-failure', action='store_true')
parser.add_argument('--debug-shell', action='store_true')
parser.add_argument('components', nargs='*')
@@ -268,37 +249,33 @@ class OstbuildBuild(builtins.Builtin):
self.args = args
self.parse_config()
+ self.parse_components_and_targets()
self.buildopts = BuildOptions()
self.buildopts.shell_on_failure = args.shell_on_failure
self.buildopts.skip_built = args.skip_built
- self.manifest_path = args.manifest
- self.manifest = json.load(open(args.manifest))
-
- components = self.manifest['components']
+ build_component_order = []
if args.recompose:
- build_components = []
+ pass
elif len(args.components) == 0:
- build_components = components
+ tsorted = buildutil.tsort_components(self.components, 'build-depends')
+ tsorted.reverse()
+ build_component_order = tsorted
else:
- build_components = []
+ if args.start_at is not None:
+ fatal("Can't specify --start-at with component list")
for name in args.components:
found = False
- for child in components:
- if child['name'] == name:
- found = True
- build_components.append(child)
- break
- if not found:
+ component = self.components.get(name)
+ if component is None:
fatal("Unknown component %r" % (name, ))
+ build_component_order.append(name)
start_at_index = -1
if args.start_at is not None:
- if build_components != components:
- fatal("Can't specify --start-at with component list")
- for i,component in enumerate(build_components):
- if component['name'] == args.start_at:
+ for i,component_name in enumerate(build_component_order):
+ if component_name == args.start_at:
start_at_index = i
break
if start_at_index == -1:
@@ -306,10 +283,11 @@ class OstbuildBuild(builtins.Builtin):
else:
start_at_index = 0
- for component in build_components[start_at_index:]:
- index = components.index(component)
- self._build_one_component(component)
+ for component_name in build_component_order[start_at_index:]:
+ component = self.components.get(component_name)
+ self._build_one_component(component_name, component)
- self._compose(components)
+ for target in self.targets['targets']:
+ self._compose(target)
builtins.register(OstbuildBuild)
diff --git a/src/ostbuild/pyostbuild/builtin_checkout.py b/src/ostbuild/pyostbuild/builtin_checkout.py
index d45a742..b768fe6 100755
--- a/src/ostbuild/pyostbuild/builtin_checkout.py
+++ b/src/ostbuild/pyostbuild/builtin_checkout.py
@@ -26,6 +26,7 @@ from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import ostbuildrc
from . import buildutil
+from . import fileutil
from . import odict
from . import vcs
@@ -38,16 +39,15 @@ class OstbuildCheckout(builtins.Builtin):
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--manifest', required=True)
parser.add_argument('--overwrite', action='store_true')
+ parser.add_argument('--clean', action='store_true')
parser.add_argument('components', nargs='*')
args = parser.parse_args(argv)
self.args = args
self.parse_config()
-
- self.manifest = json.load(open(args.manifest))
+ self.parse_components_and_targets()
if len(args.components) > 0:
checkout_components = args.components
@@ -56,17 +56,20 @@ class OstbuildCheckout(builtins.Builtin):
for component_name in checkout_components:
found = False
- component = buildutil.find_component_in_manifest(self.manifest,
- component_name)
+ component = self.components.get(component_name)
if component is None:
fatal("Unknown component %r" % (component_name, ))
(keytype, uri) = buildutil.parse_src_key(component['src'])
- checkoutdir = os.path.join(os.getcwd(), component['name'])
+ checkoutdir = os.path.join(os.getcwd(), component_name)
+ fileutil.ensure_parent_dir(checkoutdir)
component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
component['revision'],
overwrite=args.overwrite)
+ if args.clean:
+ vcs.clean(keytype, checkoutdir)
+
patches = component.get('patches')
if patches is not None:
(patches_keytype, patches_uri) = buildutil.parse_src_key(patches['src'])
diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
index 6569a39..a77d6cf 100755
--- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
@@ -22,6 +22,7 @@ import json
from . import builtins
from . import buildutil
+from . import fileutil
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
@@ -29,16 +30,16 @@ class OstbuildChrootCompileOne(builtins.Builtin):
name = "chroot-compile-one"
short_description = "Build artifacts from the current source directory in a chroot"
- def _compose_buildroot(self, component, dirpath):
- components = self.manifest['components']
- index = components.index(component)
- dependencies = components[:index]
+ def _compose_buildroot(self, component_name, dirpath):
+ dependencies = buildutil.build_depends(component_name, self.components)
+ component = self.components.get(component_name)
- base = self.manifest['base']
- base_revision = base['revision']
- checkout_trees = [(base_revision, '/')]
- for dep in dependencies:
- buildname = buildutil.manifest_buildname(self.manifest, dep)
+ base_devel_name = 'bases/%s-%s-%s' % (self.manifest['base-prefix'],
+ component['architecture'],
+ 'devel')
+ checkout_trees = [(base_devel_name, '/')]
+ for dependency_name in dependencies:
+ buildname = 'components/%s' % (dependency_name, )
checkout_trees.append((buildname, '/runtime'))
checkout_trees.append((buildname, '/devel'))
@@ -50,31 +51,32 @@ class OstbuildChrootCompileOne(builtins.Builtin):
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
- parser.add_argument('--manifest', required=True)
parser.add_argument('--pristine', action='store_true')
+ parser.add_argument('--name')
parser.add_argument('--debug-shell', action='store_true')
args = parser.parse_args(argv)
self.parse_config()
+ self.parse_components_and_targets()
- component_name = os.path.basename(os.getcwd())
- self.manifest = json.load(open(args.manifest))
+ if args.name:
+ component_name = args.name
+ else:
+ cwd = os.getcwd()
+ parent = os.path.dirname(cwd)
+ parentparent = os.path.dirname(parent)
+ component_name = '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
- component = buildutil.find_component_in_manifest(self.manifest, component_name)
- self.metadata = component
+ component = self.components.get(component_name)
if component is None:
fatal("Couldn't find component '%s' in manifest" % (component_name, ))
+ self.metadata = dict(component)
+ self.metadata['name'] = component_name
if not args.pristine:
self.metadata['src'] = 'dirty:worktree'
self.metadata['revision'] = 'dirty-worktree'
- architecture = os.uname()[4]
-
- if 'name' not in self.metadata:
- sys.stderr.write('Missing required key "%s" in metadata' % (k, ))
- sys.exit(1)
-
workdir = self.workdir
log("Using working directory: %s" % (workdir, ))
@@ -83,17 +85,17 @@ class OstbuildChrootCompileOne(builtins.Builtin):
if os.path.isdir(child_tmpdir):
log("Cleaning up previous tmpdir: %r" % (child_tmpdir, ))
shutil.rmtree(child_tmpdir)
- os.mkdir(child_tmpdir)
+ fileutil.ensure_dir(child_tmpdir)
- resultdir = os.path.join(self.workdir, 'results', component['name'])
+ resultdir = os.path.join(self.workdir, 'results', component_name)
if os.path.isdir(resultdir):
shutil.rmtree(resultdir)
- os.makedirs(resultdir)
+ fileutil.ensure_dir(resultdir)
rootdir_prefix = os.path.join(workdir, 'roots')
- if not os.path.isdir(rootdir_prefix):
- os.makedirs(rootdir_prefix)
- rootdir = os.path.join(rootdir_prefix, component['name'])
+ fileutil.ensure_dir(rootdir_prefix)
+ rootdir = os.path.join(rootdir_prefix, component_name)
+ fileutil.ensure_parent_dir(rootdir)
if os.path.isdir(rootdir):
shutil.rmtree(rootdir)
@@ -103,7 +105,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
shutil.rmtree(rootdir_tmp)
os.mkdir(rootdir_tmp)
- self._compose_buildroot(component, rootdir_tmp)
+ self._compose_buildroot(component_name, rootdir_tmp)
child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp]
run_sync(child_args)
@@ -115,15 +117,14 @@ class OstbuildChrootCompileOne(builtins.Builtin):
os.rename(rootdir_tmp, rootdir)
log("Checked out buildroot: %s" % (rootdir, ))
- sourcedir=os.path.join(builddir, 'source', self.metadata['name'])
- if not os.path.isdir(sourcedir):
- os.mkdir(sourcedir)
+ sourcedir=os.path.join(builddir, 'source', component_name)
+ fileutil.ensure_dir(sourcedir)
output_metadata = open('_ostbuild-meta.json', 'w')
json.dump(self.metadata, output_metadata, indent=4, sort_keys=True)
output_metadata.close()
- chroot_sourcedir = os.path.join('/ostbuild', 'source', self.metadata['name'])
+ chroot_sourcedir = os.path.join('/ostbuild', 'source', component_name)
ostbuild_user_chroot_path = buildutil.find_user_chroot_path()
diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py
index 03805ca..f703d0d 100755
--- a/src/ostbuild/pyostbuild/builtin_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_compile_one.py
@@ -103,10 +103,6 @@ class OstbuildCompileOne(builtins.Builtin):
self.metadata = json.load(f)
f.close()
- for k in ['name', 'revision']:
- if k not in self.metadata:
- fatal('Missing required key "%s" in metadata' % (k, ))
-
if self.metadata.get('rm-configure', False):
configure_path = 'configure'
if os.path.exists(configure_path):
@@ -180,10 +176,7 @@ class OstbuildCompileOne(builtins.Builtin):
run_sync(args, cwd=builddir)
- name = self.metadata['name']
- assert ',' not in name
-
- tempdir = tempfile.mkdtemp(prefix='ostbuild-%s-' % (name,))
+ tempdir = tempfile.mkdtemp(prefix='ostbuild-destdir-%s' % (self.metadata['name'].replace('/', '_'), ))
self.tempfiles.append(tempdir)
args = ['make', 'install', 'DESTDIR=' + tempdir]
run_sync(args, cwd=builddir)
diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py
index 3f4b566..6defbae 100755
--- a/src/ostbuild/pyostbuild/builtin_resolve.py
+++ b/src/ostbuild/pyostbuild/builtin_resolve.py
@@ -16,6 +16,7 @@
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
+import copy
import argparse
import json
import urlparse
@@ -138,16 +139,18 @@ class OstbuildResolve(builtins.Builtin):
manifest_path = self.ostbuildrc.get_key('manifest')
self.manifest = json.load(open(manifest_path))
- self.resolved_components = map(self._resolve_component_meta, self.manifest['components'])
+ snapshot = copy.deepcopy(self.manifest)
+ component_source_list = map(self._resolve_component_meta, self.manifest['components'])
+ del snapshot['components']
if args.fetch:
if len(args.components) == 0:
- fetch_components = map(lambda x: x['name'], self.resolved_components)
+ fetch_components = map(lambda x: x['name'], component_source_list)
else:
fetch_components = args.components
for component_name in fetch_components:
found = False
- for component in self.resolved_components:
+ for component in component_source_list:
if component['name'] == component_name:
found = True
break
@@ -159,76 +162,132 @@ class OstbuildResolve(builtins.Builtin):
run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False)
else:
fetch_components = []
-
+
global_patches_meta = self._resolve_component_meta(self.manifest['patches'])
(keytype, uri) = self._parse_src_key(global_patches_meta['src'])
mirrordir = self._ensure_vcs_mirror(global_patches_meta['name'], keytype, uri, global_patches_meta['branch'])
revision = buildutil.get_git_version_describe(mirrordir, global_patches_meta['branch'])
global_patches_meta['revision'] = revision
- for component in self.resolved_components:
+ unique_component_names = set()
+ for component in component_source_list:
(keytype, uri) = self._parse_src_key(component['src'])
name = component['name']
+
+ if name in unique_component_names:
+ fatal("Duplicate component name '%s'" % (name, ))
+ unique_component_names.add(name)
+
mirrordir = self._ensure_vcs_mirror(name, keytype, uri, component['branch'])
revision = buildutil.get_git_version_describe(mirrordir,
component['branch'])
component['revision'] = revision
- if 'component' not in component:
- component['component'] = 'runtime'
-
config_opts = list(self.manifest['config-opts'])
config_opts.extend(component.get('config-opts', []))
component['config-opts'] = config_opts
patch_files = component.get('patches')
if patch_files is not None:
- component['patches'] = dict(global_patches_meta)
+ component['patches'] = copy.deepcopy(global_patches_meta)
component['patches']['files'] = patch_files
- self.manifest['components'] = self.resolved_components
+ name_prefix = snapshot['name-prefix']
+ del snapshot['name-prefix']
+ base_prefix = snapshot['base-prefix']
+ del snapshot['base-prefix']
- # We expanded these keys
- del self.manifest['config-opts']
- del self.manifest['vcsconfig']
- del self.manifest['patches']
+ manifest_architectures = snapshot['architectures']
- manifest_architectures = self.manifest['architectures']
- del self.manifest['architectures']
+ components_by_name = {}
+ component_ordering = []
+ build_prev_component_by_arch = {}
+ runtime_prev_component_by_arch = {}
+ runtime_components_by_arch = {}
+ devel_components_by_arch = {}
for architecture in manifest_architectures:
- arch_manifest = dict(self.manifest)
- for component in arch_manifest['components']:
- component['arch'] = architecture
-
- runtime_components = filter(lambda x: x.get('component', 'runtime') == 'runtime', arch_manifest['components'])
- devel_components = arch_manifest['components']
- for component in arch_manifest['components']:
- if 'component' in component:
- del component['component']
-
- for component_type in ['runtime', 'devel']:
- snapshot = dict(arch_manifest)
- if component_type == 'runtime':
- snapshot['components'] = runtime_components
- else:
- snapshot['components'] = devel_components
+ runtime_components_by_arch[architecture] = []
+ devel_components_by_arch[architecture] = []
+
+ for component in component_source_list:
+ component_architectures = component.get('architectures', manifest_architectures)
+ for architecture in component_architectures:
+ component_binary = copy.deepcopy(component)
+ source_name = component['name']
+ binary_name = '%s/%s/%s' % (name_prefix, architecture, source_name)
+ component_binary['name'] = binary_name
+ component_binary['architecture'] = architecture
+
+ components_by_name[binary_name] = component_binary
+
+ prev_component = build_prev_component_by_arch.get(architecture)
+ if prev_component is not None:
+ component_binary['build-depends'] = [prev_component['name']]
+ build_prev_component_by_arch[architecture] = component_binary
+
+ is_runtime = component.get('component', 'runtime') == 'runtime'
- name_prefix = snapshot['name-prefix']
- del snapshot['name-prefix']
- base_prefix = snapshot['base-prefix']
- del snapshot['base-prefix']
+ prev_component = runtime_prev_component_by_arch.get(architecture)
+ if prev_component is not None:
+ component_binary['runtime-depends'] = [prev_component['name']]
- snapshot['name'] = '%s-%s-%s' % (name_prefix, architecture, component_type)
+ if is_runtime:
+ runtime_prev_component_by_arch[architecture] = component_binary
- base_ref = '%s-%s-%s' % (base_prefix, architecture, component_type)
+ if is_runtime:
+ runtime_components_by_arch[architecture].append(component_binary)
+ devel_components_by_arch[architecture].append(component_binary)
+
+ if 'architectures' in component_binary:
+ del component_binary['architectures']
+
+ # We expanded these keys
+ del snapshot['config-opts']
+ del snapshot['vcsconfig']
+ del snapshot['patches']
+ del snapshot['architectures']
+
+ targets_json = {}
+ targets_list = []
+ targets_json['targets'] = targets_list
+ for architecture in manifest_architectures:
+ for target_component_type in ['runtime', 'devel']:
+ target = {}
+ targets_list.append(target)
+ target['name'] = '%s-%s-%s' % (name_prefix, architecture, target_component_type)
+
+ base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', base_ref])
- snapshot['base'] = {'branch': base_ref,
- 'revision': base_revision}
- out_snapshot = os.path.join(self.workdir, snapshot['name'] + '.snapshot')
- f = open(out_snapshot, 'w')
- json.dump(snapshot, f, indent=4, sort_keys=True)
- f.close()
- print "Created: %s" % (out_snapshot, )
+ 'rev-parse', 'bases/%s' % (base_ref, )])
+ target['base'] = {'name': base_ref}
+
+ if target_component_type == 'runtime':
+ target_components = runtime_components_by_arch[architecture]
+ else:
+ target_components = devel_components_by_arch[architecture]
+
+ contents = []
+ for component in target_components:
+ name = component['name']
+ component_ref = {'name': name}
+ if target_component_type == 'runtime':
+ component_ref['trees'] = ['/runtime']
+ else:
+ component_ref['trees'] = ['/runtime', '/devel', '/doc']
+ contents.append(component_ref)
+ target['contents'] = contents
+ out_targets = os.path.join(self.workdir, '%s-targets.json' % (name_prefix, ))
+ f = open(out_targets, 'w')
+ json.dump(targets_json, f, indent=4, sort_keys=True)
+ f.close()
+ print "Created: %s" % (out_targets, )
+
+ out_components = os.path.join(self.workdir, '%s-components.json' % (name_prefix, ))
+ f = open(out_components, 'w')
+ for component in components_by_name.itervalues():
+ del component['name']
+ json.dump(components_by_name, f, indent=4, sort_keys=True)
+ f.close()
+ print "Created: %s" % (out_components, )
builtins.register(OstbuildResolve)
diff --git a/src/ostbuild/pyostbuild/builtin_status.py b/src/ostbuild/pyostbuild/builtin_status.py
index 4b85c9d..f4585d5 100755
--- a/src/ostbuild/pyostbuild/builtin_status.py
+++ b/src/ostbuild/pyostbuild/builtin_status.py
@@ -42,14 +42,14 @@ class OstbuildStatus(builtins.Builtin):
args = parser.parse_args(argv)
self.parse_config()
- self.manifest = json.load(open(args.manifest))
+ self.parse_components_and_targets()
- for component in self.manifest['components']:
- branch = buildutil.manifest_buildname(self.manifest, component)
+ for name,component in self.components.iteritems():
+ buildname = 'components/%s' % (name, )
build_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
'show',
'--print-metadata-key=ostbuild-artifact-version',
- branch],
+ buildname],
none_on_error=True)
if build_revision is None:
build_revision = '(not built)'
@@ -57,7 +57,7 @@ class OstbuildStatus(builtins.Builtin):
build_status = '(needs build)'
else:
build_status = 'ok'
- sys.stdout.write('{:<40} {:<95} {:<10}\n'.format(component['name'],
+ sys.stdout.write('{:<40} {:<95} {:<10}\n'.format(name,
build_revision, build_status))
builtins.register(OstbuildStatus)
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index 4a2baee..60f5278 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -20,6 +20,7 @@
import os
import sys
import argparse
+import json
from . import ostbuildrc
from .ostbuildlog import log, fatal
@@ -41,6 +42,18 @@ class Builtin(object):
fatal("Specified workdir '%s' is not a directory" % (self.workdir, ))
self.patchdir = os.path.join(self.workdir, 'patches')
+ def parse_manifest(self):
+ self.manifest_path = ostbuildrc.get_key('manifest')
+ self.manifest = json.load(open(self.manifest_path))
+ self.name_prefix = self.manifest['name-prefix']
+
+ def parse_components_and_targets(self):
+ self.parse_manifest()
+ components_path = os.path.join(self.workdir, '%s-components.json' % (self.name_prefix, ))
+ self.components = json.load(open(components_path))
+ targets_path = os.path.join(self.workdir, '%s-targets.json' % (self.name_prefix, ))
+ self.targets = json.load(open(targets_path))
+
def execute(self, args):
raise NotImplementedError()
diff --git a/src/ostbuild/pyostbuild/fileutil.py b/src/ostbuild/pyostbuild/fileutil.py
new file mode 100644
index 0000000..d9ae91f
--- /dev/null
+++ b/src/ostbuild/pyostbuild/fileutil.py
@@ -0,0 +1,26 @@
+#
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+
+def ensure_dir(path):
+ if not os.path.isdir(path):
+ os.makedirs(path)
+
+def ensure_parent_dir(path):
+ ensure_dir(os.path.dirname(path))
diff --git a/src/ostbuild/pyostbuild/vcs.py b/src/ostbuild/pyostbuild/vcs.py
index 2479d62..10c7e39 100755
--- a/src/ostbuild/pyostbuild/vcs.py
+++ b/src/ostbuild/pyostbuild/vcs.py
@@ -69,3 +69,7 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
if tmp_dest != dest:
os.rename(tmp_dest, dest)
return dest
+
+def clean(keytype, checkoutdir):
+ assert keytype == 'git'
+ run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
