[ostree/wip/ostbuild-v2] wip
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree/wip/ostbuild-v2] wip
- Date: Wed, 18 Apr 2012 19:16:12 +0000 (UTC)
commit 36ec4facb701f8aa8daadc846be80f74179b8a23
Author: Colin Walters <walters verbum org>
Date: Wed Apr 18 08:57:54 2012 -0400
wip
Makefile-ostbuild.am | 2 +
src/ostbuild/pyostbuild/builtin_build.py | 117 ++++++---------------------
src/ostbuild/pyostbuild/builtin_compose.py | 119 ++++++++++++++++++++++++++++
src/ostbuild/pyostbuild/builtin_resolve.py | 11 +--
src/ostbuild/pyostbuild/builtins.py | 38 +++++++++-
src/ostbuild/pyostbuild/jsondb.py | 109 +++++++++++++++++++++++++
src/ostbuild/pyostbuild/main.py | 1 +
7 files changed, 299 insertions(+), 98 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 2109e61..11e49e2 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -24,6 +24,7 @@ pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/buildutil.py \
src/ostbuild/pyostbuild/builtin_build.py \
src/ostbuild/pyostbuild/builtin_checkout.py \
+ src/ostbuild/pyostbuild/builtin_compose.py \
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \
src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py \
src/ostbuild/pyostbuild/builtin_compile_one.py \
@@ -36,6 +37,7 @@ pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/filemonitor.py \
src/ostbuild/pyostbuild/fileutil.py \
src/ostbuild/pyostbuild/__init__.py \
+ src/ostbuild/pyostbuild/jsondb.py \
src/ostbuild/pyostbuild/kvfile.py \
src/ostbuild/pyostbuild/main.py \
src/ostbuild/pyostbuild/mainloop.py \
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index c044601..aae6481 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -149,72 +149,34 @@ class OstbuildBuild(builtins.Builtin):
output = run_sync_get_output(args)
return output.split('\n')
- def _compose_one_target(self, bin_snapshot, target):
- components = bin_snapshot['components']
- base = target['base']
- base_name = 'bases/%s' % (base['name'], )
- base_revision = target['base']['ostree-revision']
-
- compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
- if os.path.isdir(compose_rootdir):
- shutil.rmtree(compose_rootdir)
- os.mkdir(compose_rootdir)
-
- resolved_base = dict(target['base'])
- resolved_base['ostree-revision'] = base_revision
- resolved_contents = list(target['contents'])
- for tree_content_item in resolved_contents:
- name = tree_content_item['name']
- rev = components[name]['ostree-revision']
- tree_content_item['ostree-revision'] = rev
-
- metadata = {'source': 'ostbuild compose v0',
- 'base': resolved_base,
- 'contents': resolved_contents}
-
- # Copy all other keys
- for k,v in bin_snapshot.iteritems():
- if k in ['components', 'targets']:
- continue
- metadata[k] = v
-
- compose_contents = [(base_revision, '/')]
- for tree_content in resolved_contents:
- rev = tree_content['ostree-revision']
- subtrees = tree_content['trees']
- for subpath in subtrees:
- compose_contents.append((rev, subpath))
-
- (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
- f = os.fdopen(fd, 'w')
- for (branch, subpath) in compose_contents:
- f.write(branch)
- f.write('\0')
- f.write(subpath)
- f.write('\0')
- f.close()
-
- run_sync(['ostree', '--repo=' + self.repo,
- 'checkout', '--user-mode', '--no-triggers',
- '--union', '--from-stdin', compose_rootdir],
- stdin=open(tmppath))
- os.unlink(tmppath)
-
- contents_path = os.path.join(compose_rootdir, 'contents.json')
- f = open(contents_path, 'w')
- json.dump(metadata, f, indent=4, sort_keys=True)
- f.close()
-
- run_sync(['ostree', '--repo=' + self.repo,
- 'commit', '-b', target['name'], '-s', 'Compose',
- '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
- '--skip-if-unchanged'], cwd=compose_rootdir)
+ def _save_bin_snapshot(self):
+ bin_snapshot = dict(self.snapshot)
+
+ for target in bin_snapshot['targets']:
+ base = target['base']
+ base_name = 'bases/%s' % (base['name'], )
+ base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', base_name])
+ base['ostree-revision'] = base_revision
+
+ component_refs = []
+ for name in bin_snapshot['components'].iterkeys():
+ component_refs.append('components/%s' % (name, ))
+
+ new_components = {}
+ resolved_refs = self._resolve_refs(component_refs)
+ for name,rev in zip(bin_snapshot['components'].iterkeys(), resolved_refs):
+ new_components[name] = {'ostree-revision': rev}
+
+ bin_snapshot['components'] = new_components
+
+ path = self.get_bin_snapshot_db().store(bin_snapshot)
+ print "Binary snapshot: %s" % (path, )
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--skip-built', action='store_true')
- parser.add_argument('--snapshot', required=True)
- parser.add_argument('--recompose', action='store_true')
+ parser.add_argument('--src-snapshot')
parser.add_argument('--skip-compose', action='store_true')
parser.add_argument('--start-at')
parser.add_argument('--shell-on-failure', action='store_true')
@@ -225,16 +187,14 @@ class OstbuildBuild(builtins.Builtin):
self.args = args
self.parse_config()
- self.parse_snapshot(args.snapshot)
+ self.parse_snapshot(args.src_snapshot)
self.buildopts = BuildOptions()
self.buildopts.shell_on_failure = args.shell_on_failure
self.buildopts.skip_built = args.skip_built
build_component_order = []
- if args.recompose:
- pass
- elif len(args.components) == 0:
+ if len(args.components) == 0:
tsorted = buildutil.tsort_components(self.snapshot['components'], 'build-depends')
tsorted.reverse()
build_component_order = tsorted
@@ -263,29 +223,6 @@ class OstbuildBuild(builtins.Builtin):
component = self.snapshot['components'].get(component_name)
self._build_one_component(component_name, component)
- if not args.skip_compose:
-
- bin_snapshot = dict(self.snapshot)
-
- for target in bin_snapshot['targets']:
- base = target['base']
- base_name = 'bases/%s' % (base['name'], )
- base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
- 'rev-parse', base_name])
- base['ostree-revision'] = base_revision
-
- component_refs = []
- for name in bin_snapshot['components'].iterkeys():
- component_refs.append('components/%s' % (name, ))
-
- new_components = {}
- resolved_refs = self._resolve_refs(component_refs)
- for name,rev in zip(bin_snapshot['components'].iterkeys(), resolved_refs):
- new_components[name] = {'ostree-revision': rev}
-
- bin_snapshot['components'] = new_components
-
- for target in bin_snapshot['targets']:
- self._compose_one_target(bin_snapshot, target)
+ self._save_bin_snapshot()
builtins.register(OstbuildBuild)
diff --git a/src/ostbuild/pyostbuild/builtin_compose.py b/src/ostbuild/pyostbuild/builtin_compose.py
new file mode 100755
index 0000000..51be34a
--- /dev/null
+++ b/src/ostbuild/pyostbuild/builtin_compose.py
@@ -0,0 +1,119 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from .subprocess_helpers import run_sync_monitor_log_file
+from . import ostbuildrc
+from . import buildutil
+from . import fileutil
+from . import kvfile
+from . import odict
+from . import vcs
+
+class OstbuildCompose(builtins.Builtin):
+ name = "compose"
+ short_description = "Build complete trees from components"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+
+ def _compose_one_target(self, bin_snapshot, target):
+ components = bin_snapshot['components']
+ base = target['base']
+ base_name = 'bases/%s' % (base['name'], )
+ base_revision = target['base']['ostree-revision']
+
+ compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
+ if os.path.isdir(compose_rootdir):
+ shutil.rmtree(compose_rootdir)
+ os.mkdir(compose_rootdir)
+
+ resolved_base = dict(target['base'])
+ resolved_base['ostree-revision'] = base_revision
+ resolved_contents = list(target['contents'])
+ for tree_content_item in resolved_contents:
+ name = tree_content_item['name']
+ rev = components[name]['ostree-revision']
+ tree_content_item['ostree-revision'] = rev
+
+ metadata = {'source': 'ostbuild compose v0',
+ 'base': resolved_base,
+ 'contents': resolved_contents}
+
+ # Copy all other keys
+ for k,v in bin_snapshot.iteritems():
+ if k in ['components', 'targets']:
+ continue
+ metadata[k] = v
+
+ compose_contents = [(base_revision, '/')]
+ for tree_content in resolved_contents:
+ rev = tree_content['ostree-revision']
+ subtrees = tree_content['trees']
+ for subpath in subtrees:
+ compose_contents.append((rev, subpath))
+
+ (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+ f = os.fdopen(fd, 'w')
+ for (branch, subpath) in compose_contents:
+ f.write(branch)
+ f.write('\0')
+ f.write(subpath)
+ f.write('\0')
+ f.close()
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'checkout', '--user-mode', '--no-triggers',
+ '--union', '--from-stdin', compose_rootdir],
+ stdin=open(tmppath))
+ os.unlink(tmppath)
+
+ contents_path = os.path.join(compose_rootdir, 'contents.json')
+ f = open(contents_path, 'w')
+ json.dump(metadata, f, indent=4, sort_keys=True)
+ f.close()
+
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'commit', '-b', target['name'], '-s', 'Compose',
+ '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
+ '--skip-if-unchanged'], cwd=compose_rootdir)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--bin-snapshot')
+
+ args = parser.parse_args(argv)
+ self.args = args
+
+ self.parse_config()
+ self.parse_bin_snapshot(args.bin_snapshot)
+
+ for target in self.bin_snapshot['targets']:
+ print "Composing target %r from %u components" % (target['name'],
+ len(target['contents']))
+ self._compose_one_target(self.bin_snapshot, target)
+
+builtins.register(OstbuildCompose)
diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py
index ea70ea5..4575b45 100755
--- a/src/ostbuild/pyostbuild/builtin_resolve.py
+++ b/src/ostbuild/pyostbuild/builtin_resolve.py
@@ -27,6 +27,7 @@ from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import ostbuildrc
+from . import jsondb
from . import buildutil
from . import kvfile
from . import odict
@@ -195,9 +196,7 @@ class OstbuildResolve(builtins.Builtin):
component['patches']['files'] = patch_files
name_prefix = snapshot['name-prefix']
- del snapshot['name-prefix']
base_prefix = snapshot['base-prefix']
- del snapshot['base-prefix']
manifest_architectures = snapshot['architectures']
@@ -288,10 +287,8 @@ class OstbuildResolve(builtins.Builtin):
current_time = time.time()
- out_snapshot = os.path.join(self.workdir, '%s-src-snapshot-%d.json' % (name_prefix, int(current_time)))
- f = open(out_snapshot, 'w')
- json.dump(snapshot, f, indent=4, sort_keys=True)
- f.close()
- print "Created: %s" % (out_snapshot, )
+ src_db = self.get_src_snapshot_db()
+ path = src_db.store(snapshot)
+ print "Source snapshot: %s" % (path, )
builtins.register(OstbuildResolve)
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index 984c751..253e85a 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -24,6 +24,7 @@ import argparse
import json
from . import ostbuildrc
+from . import jsondb
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
@@ -37,9 +38,12 @@ class Builtin(object):
self._meta_cache = {}
self.manifest = None
self.snapshot = None
+ self.bin_snapshot = None
self.repo = None
self.ostree_dir = self._find_ostree_dir()
(self.active_branch, self.active_branch_checksum) = self._find_active_branch()
+ self._src_snapshots = None
+ self._bin_snapshots = None
def _find_ostree_dir(self):
for path in ['/ostree', '/sysroot/ostree']:
@@ -120,11 +124,43 @@ class Builtin(object):
self._meta_cache[name] = meta
return meta
+ def _create_db(self, dbsuffix):
+ prefix = ostbuildrc.get_key('prefix')
+ name = '%s-%s' % (prefix, dbsuffix)
+ return jsondb.JsonDB(self.workdir, prefix=name)
+
+ def get_src_snapshot_db(self):
+ if self._src_snapshots is None:
+ self._src_snapshots = self._create_db('src-snapshot')
+ return self._src_snapshots
+
+ def get_bin_snapshot_db(self):
+ if self._bin_snapshots is None:
+ self._bin_snapshots = self._create_db('bin-snapshot')
+ return self._bin_snapshots
+
def parse_snapshot(self, path):
self.repo = ostbuildrc.get_key('repo')
- self.snapshot_path = path
+ if path is None:
+ latest_path = self.get_src_snapshot_db().get_latest_path()
+ if latest_path is None:
+ raise Exception("No source snapshot found")
+ self.snapshot_path = latest_path
+ else:
+ self.snapshot_path = path
self.snapshot = json.load(open(self.snapshot_path))
+ def parse_bin_snapshot(self, path):
+ self.repo = ostbuildrc.get_key('repo')
+ if path is None:
+ latest_path = self.get_bin_snapshot_db().get_latest_path()
+ if latest_path is None:
+ raise Exception("No binary snapshot found")
+ self.bin_snapshot_path = latest_path
+ else:
+ self.bin_snapshot_path = path
+ self.bin_snapshot = json.load(open(self.bin_snapshot_path))
+
def execute(self, args):
raise NotImplementedError()
diff --git a/src/ostbuild/pyostbuild/jsondb.py b/src/ostbuild/pyostbuild/jsondb.py
new file mode 100644
index 0000000..9655fbf
--- /dev/null
+++ b/src/ostbuild/pyostbuild/jsondb.py
@@ -0,0 +1,109 @@
+#
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os
+import stat
+import time
+import tempfile
+import re
+import shutil
+import hashlib
+import json
+
+class JsonDB(object):
+ def __init__(self, dirpath, prefix):
+ self._dirpath = dirpath
+ self._prefix = prefix
+ self._version_csum_re = re.compile(r'-(\d+)\.(\d+)-([0-9a-f]+).json$')
+
+ def _cmp_match_by_version(self, a, b):
+ a_major = a[0]
+ a_minor = a[1]
+ b_major = b[0]
+ b_minor = b[1]
+
+ c = cmp(a_major, b_major)
+ if c == 0:
+ return cmp(a_minor, b_minor)
+ return 0
+
+ def _get_all(self):
+ result = []
+ for fname in os.listdir(self._dirpath):
+ if not fname.startswith(self._prefix):
+ continue
+
+ path = os.path.join(self._dirpath, fname)
+ match = self._version_csum_re.search(fname)
+ if not match:
+ raise Exception("Invalid file '%s' in JsonDB; doesn't contain version+checksum",
+ path)
+ result.append((int(match.group(1)), int(match.group(2)), match.group(3), fname))
+ result.sort(self._cmp_match_by_version)
+ return result
+
+ def get_latest(self):
+ path = self.get_latest_path()
+ if path is None:
+ return None
+ return json.load(open(path))
+
+ def get_latest_path(self):
+ files = self._get_all()
+ if len(files) == 0:
+ return None
+ return os.path.join(self._dirpath, files[0][3])
+
+ def store(self, obj):
+ files = self._get_all()
+ if len(files) == 0:
+ latest = None
+ else:
+ latest = files[0]
+
+ current_time = time.gmtime()
+
+ (fd, tmppath) = tempfile.mkstemp(suffix='.tmp', prefix='tmp-jsondb-')
+ os.close(fd)
+ f = open(tmppath, 'w')
+ json.dump(obj, f, indent=4, sort_keys=True)
+ f.close()
+
+ csum = hashlib.sha256()
+ f = open(tmppath)
+ buf = f.read(8192)
+ while buf != '':
+ csum.update(buf)
+ buf = f.read(8192)
+ f.close()
+ digest = csum.hexdigest()
+
+ if latest is not None:
+ if digest == latest[2]:
+ return latest[3]
+ latest_version = (latest[0], latest[1])
+ else:
+ latest_version = (current_time.tm_year, 0)
+ target_name = '%s-%d.%d-%s.json' % (self._prefix, current_time.tm_year,
+ latest_version[1] + 1, digest)
+ target_path = os.path.join(self._dirpath, target_name)
+ os.rename(tmppath, target_path)
+ return target_path
+
+
+
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 5b48d71..0b2fd24 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -26,6 +26,7 @@ from . import builtin_build
from . import builtin_checkout
from . import builtin_chroot_compile_one
from . import builtin_chroot_run_triggers
+from . import builtin_compose
from . import builtin_compile_one
from . import builtin_pull_components
from . import builtin_query_content
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
