[ostree/wip/ostbuild-v2: 1/4] ostbuild: Major rework

From: Colin Walters <walters src gnome org>
To: commits-list gnome org
Cc:
Subject: [ostree/wip/ostbuild-v2: 1/4] ostbuild: Major rework
Date: Thu, 19 Apr 2012 03:15:14 +0000 (UTC)
commit 1a1531970bc7a3659362526e40710ebbb694df1e
Author: Colin Walters <walters verbum org>
Date:   Fri Apr 6 15:12:57 2012 -0400

    ostbuild: Major rework

 Makefile-ostbuild.am                               |    2 +
 gnomeos/3.4/gnomeos-3.4-src.json                   |    3 +-
 gnomeos/README-build.md                            |   42 +++++++++-
 src/ostbuild/pyostbuild/builtin_build.py           |   44 +++++++---
 src/ostbuild/pyostbuild/builtin_checkout.py        |   16 +++-
 .../pyostbuild/builtin_chroot_compile_one.py       |   45 +++++++---
 src/ostbuild/pyostbuild/builtin_compile_one.py     |   25 +++---
 src/ostbuild/pyostbuild/builtin_pull_components.py |   70 +++++++++++++++
 src/ostbuild/pyostbuild/builtin_query_content.py   |   65 ++++++++++----
 .../pyostbuild/builtin_replace_component.py        |   67 ++++++++++++++
 src/ostbuild/pyostbuild/builtin_resolve.py         |   12 +++-
 src/ostbuild/pyostbuild/builtins.py                |   91 ++++++++++++++++++--
 src/ostbuild/pyostbuild/main.py                    |    2 +
 src/ostbuild/pyostbuild/subprocess_helpers.py      |    6 +-
 14 files changed, 413 insertions(+), 77 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 08eb8c0..2109e61 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -28,7 +28,9 @@ pyostbuild_PYTHON =					\
 	src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py	\
 	src/ostbuild/pyostbuild/builtin_compile_one.py	\
 	src/ostbuild/pyostbuild/builtin_query_content.py	\
+	src/ostbuild/pyostbuild/builtin_pull_components.py	\
 	src/ostbuild/pyostbuild/builtin_resolve.py	\
+	src/ostbuild/pyostbuild/builtin_replace_component.py	\
 	src/ostbuild/pyostbuild/builtin_status.py	\
 	src/ostbuild/pyostbuild/builtins.py		\
 	src/ostbuild/pyostbuild/filemonitor.py		\
diff --git a/gnomeos/3.4/gnomeos-3.4-src.json b/gnomeos/3.4/gnomeos-3.4-src.json
index fcf7e9b..6864e0c 100644
--- a/gnomeos/3.4/gnomeos-3.4-src.json
+++ b/gnomeos/3.4/gnomeos-3.4-src.json
@@ -566,7 +566,8 @@
 		{"src": "gnome:libsoup",
 		 "config-args": ["--disable-tls-check"]},
 
-		{"src": "gnome:ostree",
+		{"src": "git:/src/ostree",
+		 "branch": "wip/ostbuild-v2",
 		 "config-opts": ["--disable-documentation",
 		                 "--with-soup-gnome"]},
 
diff --git a/gnomeos/README-build.md b/gnomeos/README-build.md
index b432276..0414576 100644
--- a/gnomeos/README-build.md
+++ b/gnomeos/README-build.md
@@ -16,6 +16,9 @@ base "runtime", and one "devel" with all of the development tools like
 gcc.  We then import that into an OSTree branch
 e.g. "bases/yocto/gnomeos-3.4-i686-devel".
 
+At present, it's still (mostly) possible to put this data on an ext4
+filesystem and boot into it.
+
 We also have a Yocto recipe "ostree-native" which generates (as you
 might guess) a native binary of ostree.  That binary is used to import
 into an "archive mode" OSTree repository.  You can see it in
@@ -26,26 +29,59 @@ can use "ostbuild" which uses "linux-user-chroot" to chroot inside,
 run a build on a source tree, and outputs binaries, which we then add
 to the build tree for the next module, and so on.
 
+The final result of all of this is that the OSTree repository gains
+new commits (which can be downloaded by clients), while still
+retaining old build history.
+
+Yocto details
+-------------
+
+I have a branch of Yocto here:
+
+https://github.com/cgwalters/poky
+
+It has a collection of patches on top of the "Edison" release of
+Yocto, some of which are hacky, others upstreamable.  The most
+important part though are the modifications to commit the generated
+root filesystem into OSTree.
+
 ostbuild details
 ----------------
 
 The simple goal of ostbuild is that it only takes as input a
-"manifest" which is basically just a list of components to build.  A
-component is a pure metadata file which includes the git repository
+"manifest" which is basically just a list of components to build.  You
+can see this here:
+
+http://git.gnome.org/browse/ostree/tree/gnomeos/3.4/gnomeos-3.4-src.json
+
+A component is a pure metadata file which includes the git repository
 URL and branch name, as well as ./configure flags (--enable-foo).
 
 There is no support for building from "tarballs" - I want the ability
 to review all of the code that goes in, and to efficiently store
-source code updates.
+source code updates.  It's also just significantly easier from an
+implementation perspective, versus having to maintain a version
+control abstraction layer.
 
 The result of a build of a component is an OSTree branch like
 "artifacts/gnomeos-3.4-i686-devel/libxslt/master".  Then, a "compose"
 process merges together the individual filesystem trees into the final
 branches (e.g. gnomeos-3.4-i686-devel).
 
+Doing local builds
+------------------
+
+This is where you want to modify one (or a few) components on top of
+what comes from the ostree.gnome.org server, and test the result
+locally.  I'm working on this.
+
 Doing a full build on your system
 ---------------------------------
 
+Following this process is equivalent to what we have set up on the
+ostree.gnome.org build server.  It will generate a completely new
+repository.
+
 srcdir=/src
 builddir=/src/build
 
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index 772aaca..cac17b3 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -38,7 +38,7 @@ class BuildOptions(object):
 
 class OstbuildBuild(builtins.Builtin):
     name = "build"
-    short_description = "Rebuild all artifacts from the given manifest"
+    short_description = "Build artifacts and compose trees from snapshot"
 
     def __init__(self):
         builtins.Builtin.__init__(self)
@@ -49,7 +49,8 @@ class OstbuildBuild(builtins.Builtin):
             args = ['setarch', architecture]
         else:
             args = []
-        args.extend(['ostbuild', 'chroot-compile-one'])
+        args.extend(['ostbuild', 'chroot-compile-one',
+                     '--snapshot=' + self.snapshot_path])
         return args
 
     def _launch_debug_shell(self, architecture, buildroot, cwd=None):
@@ -96,15 +97,11 @@ class OstbuildBuild(builtins.Builtin):
 
         checkoutdir = os.path.join(self.workdir, 'src')
         component_src = os.path.join(checkoutdir, name)
-        run_sync(['ostbuild', 'checkout', '--clean', '--overwrite', name], cwd=checkoutdir)
+        run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
+                  '--clean', '--overwrite', name], cwd=checkoutdir)
 
         artifact_meta = dict(component)
 
-        metadata_path = os.path.join(component_src, '_ostbuild-meta.json')
-        f = open(metadata_path, 'w')
-        json.dump(artifact_meta, f, indent=4, sort_keys=True)
-        f.close()
-
         logdir = os.path.join(self.workdir, 'logs', name)
         fileutil.ensure_dir(logdir)
         log_path = os.path.join(logdir, 'compile.log')
@@ -181,18 +178,36 @@ class OstbuildBuild(builtins.Builtin):
         resolved_contents = list(target['contents'])
         for component in resolved_contents:
             component['ostree-revision'] = branch_to_rev[component['name']]
+
         metadata = {'source': 'ostbuild compose v0',
                     'base': resolved_base, 
                     'contents': resolved_contents}
+        for k,v in self.snapshot.iteritems():
+            if k in ['components', 'targets']:
+                continue
+            metadata[k] = v
 
+        compose_contents = []
         for branch in contents:
             branch_rev = branch_to_rev[branch]
             subtrees = branch_to_subtrees[branch]
-            for subtree in subtrees:
-                run_sync(['ostree', '--repo=' + self.repo,
-                          'checkout', '--user-mode',
-                          '--union', '--subpath=' + subtree,
-                          branch_rev, compose_rootdir])
+            for subpath in subtrees:
+                compose_contents.append((branch_rev, subpath))
+
+        (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
+        f = os.fdopen(fd, 'w')
+        for (branch, subpath) in compose_contents:
+            f.write(branch)
+            f.write('\0')
+            f.write(subpath)
+            f.write('\0')
+        f.close()
+
+        run_sync(['ostree', '--repo=' + self.repo,
+                  'checkout', '--user-mode', '--no-triggers',
+                  '--union', '--from-stdin', compose_rootdir],
+                 stdin=open(tmppath))
+        os.unlink(tmppath)
 
         contents_path = os.path.join(compose_rootdir, 'contents.json')
         f = open(contents_path, 'w')
@@ -207,6 +222,7 @@ class OstbuildBuild(builtins.Builtin):
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--skip-built', action='store_true')
+        parser.add_argument('--snapshot', required=True)
         parser.add_argument('--recompose', action='store_true')
         parser.add_argument('--skip-compose', action='store_true')
         parser.add_argument('--start-at')
@@ -218,7 +234,7 @@ class OstbuildBuild(builtins.Builtin):
         self.args = args
         
         self.parse_config()
-        self.parse_snapshot()
+        self.parse_snapshot(args.snapshot)
 
         self.buildopts = BuildOptions()
         self.buildopts.shell_on_failure = args.shell_on_failure
diff --git a/src/ostbuild/pyostbuild/builtin_checkout.py b/src/ostbuild/pyostbuild/builtin_checkout.py
index 553e505..ae94163 100755
--- a/src/ostbuild/pyostbuild/builtin_checkout.py
+++ b/src/ostbuild/pyostbuild/builtin_checkout.py
@@ -40,6 +40,7 @@ class OstbuildCheckout(builtins.Builtin):
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--overwrite', action='store_true')
+        parser.add_argument('--snapshot')
         parser.add_argument('--clean', action='store_true')
         parser.add_argument('components', nargs='*')
 
@@ -47,18 +48,20 @@ class OstbuildCheckout(builtins.Builtin):
         self.args = args
         
         self.parse_config()
-        self.parse_snapshot()
 
         if len(args.components) > 0:
             checkout_components = args.components
         else:
             checkout_components = [os.path.basename(os.getcwd())]
 
+        if args.snapshot:
+            self.parse_snapshot(args.snapshot)
+        else:
+            self.parse_active_branch()
+
         for component_name in checkout_components:
             found = False
-            component = self.snapshot['components'].get(component_name)
-            if component is None:
-                fatal("Unknown component %r" % (component_name, ))
+            component = self.get_component_meta(component_name)
             (keytype, uri) = buildutil.parse_src_key(component['src'])
             checkoutdir = os.path.join(os.getcwd(), component_name)
             fileutil.ensure_parent_dir(checkoutdir)
@@ -86,6 +89,11 @@ class OstbuildCheckout(builtins.Builtin):
                 for patch in patches['files']:
                     patch_path = os.path.join(patchdir, patch)
                     run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=checkoutdir)
+
+            metadata_path = os.path.join(checkoutdir, '_ostbuild-meta.json')
+            f = open(metadata_path, 'w')
+            json.dump(component, f, indent=4, sort_keys=True)
+            f.close()
         
             print "Checked out: %r" % (component_src, )
         
diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
index 0c6a96c..f3d09d8 100755
--- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
@@ -18,6 +18,7 @@
 import os,sys,re,subprocess,tempfile,shutil
 from StringIO import StringIO
 import argparse
+import time
 import json
 
 from . import builtins
@@ -32,13 +33,15 @@ class OstbuildChrootCompileOne(builtins.Builtin):
     short_description = "Build artifacts from the current source directory in a chroot"
 
     def _compose_buildroot(self, component_name, dirpath):
+        starttime = time.time()
+
         components = self.snapshot['components']
         dependencies = buildutil.build_depends(component_name, components)
         component = components.get(component_name)
 
-        base_devel_name = 'bases/%s-%s-%s' % (self.snapshot['base-prefix'],
-                                              component['architecture'],
-                                              'devel')
+        buildroots = self.snapshot['architecture-buildroots']
+        base_devel_name = 'bases/' + buildroots[component['architecture']]
+
         checkout_trees = [(base_devel_name, '/')]
         for dependency_name in dependencies:
             buildname = 'components/%s' % (dependency_name, )
@@ -48,22 +51,41 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         link_cache_dir = os.path.join(self.workdir, 'link-cache')
         fileutil.ensure_dir(link_cache_dir)
 
-        for (branch, rootpath) in checkout_trees:
-            run_sync(['ostree', '--repo=' + self.repo,
-                      'checkout', '--user-mode', '--link-cache=' + link_cache_dir,
-                      '--union', '--subpath=' + rootpath,
-                      branch, dirpath])
+        (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-buildroot-')
+        f = os.fdopen(fd, 'w')
+        for (branch, subpath) in checkout_trees:
+            f.write(branch)
+            f.write('\0')
+            f.write(subpath)
+            f.write('\0')
+        f.close()
+
+        log("roots: %r" % (checkout_trees, ))
+
+        link_cache_dir = os.path.join(self.workdir, 'link-cache')
+        fileutil.ensure_dir(link_cache_dir)
+
+        run_sync(['ostree', '--repo=' + self.repo,
+                  'checkout', '--link-cache=' + link_cache_dir,
+                  '--user-mode', '--union', '--from-stdin', dirpath],
+                 stdin=open(tmppath))
+
+        os.unlink(tmppath);
+
+        endtime = time.time()
+        log("Composed buildroot; %d seconds elapsed" % (int(endtime - starttime),))
 
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--pristine', action='store_true')
+        parser.add_argument('--snapshot', required=True)
         parser.add_argument('--name')
         parser.add_argument('--debug-shell', action='store_true')
         
         args = parser.parse_args(argv)
 
         self.parse_config()
-        self.parse_snapshot()
+        self.parse_snapshot(args.snapshot)
 
         if args.name:
             component_name = args.name
@@ -113,9 +135,6 @@ class OstbuildChrootCompileOne(builtins.Builtin):
             
         self._compose_buildroot(component_name, rootdir_tmp)
 
-        child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp]
-        run_sync(child_args)
-
         builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
         os.mkdir(builddir_tmp)
         os.mkdir(os.path.join(builddir_tmp, 'source'))
@@ -126,6 +145,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         sourcedir=os.path.join(builddir, 'source', component_name)
         fileutil.ensure_dir(sourcedir)
         
+        print "cwd: %r" % (os.getcwd(), )
         output_metadata = open('_ostbuild-meta.json', 'w')
         json.dump(self.metadata, output_metadata, indent=4, sort_keys=True)
         output_metadata.close()
@@ -148,7 +168,6 @@ class OstbuildChrootCompileOne(builtins.Builtin):
                                'compile-one',
                                '--ostbuild-resultdir=/ostbuild/results',
                                '--ostbuild-meta=_ostbuild-meta.json'])
-            child_args.extend(self.metadata.get('config-opts', []))
         env_copy = dict(buildutil.BUILD_ENV)
         env_copy['PWD'] = chroot_sourcedir
         run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell)
diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py
index f703d0d..eef08df 100755
--- a/src/ostbuild/pyostbuild/builtin_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_compile_one.py
@@ -57,6 +57,8 @@ class OstbuildCompileOne(builtins.Builtin):
 
     def execute(self, args):
         self.default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )]
+
+        starttime = time.time()
         
         uname=os.uname()
         kernel=uname[0].lower()
@@ -77,32 +79,28 @@ class OstbuildCompileOne(builtins.Builtin):
                       '--infodir=' + os.path.join(PREFIX, 'share', 'info')]
         self.makeargs = ['make']
 
-        self.ostbuild_resultdir=None
-        self.ostbuild_meta=None
+        self.ostbuild_resultdir='_ostbuild-results'
+        self.ostbuild_meta_path='_ostbuild-meta.json'
 
         chdir = None
+        opt_install = False
 
         for arg in args:
             if arg.startswith('--ostbuild-resultdir='):
                 self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):]
             elif arg.startswith('--ostbuild-meta='):
-                self.ostbuild_meta=arg[len('--ostbuild-meta='):]
+                self.ostbuild_meta_path=arg[len('--ostbuild-meta='):]
             elif arg.startswith('--chdir='):
                 os.chdir(arg[len('--chdir='):])
-            elif arg.startswith('--'):
-                self.configargs.append(arg)
             else:
                 self.makeargs.append(arg)
         
-        if self.ostbuild_resultdir is None:
-            fatal("Must specify --ostbuild-resultdir=")
-        if self.ostbuild_meta is None:
-            fatal("Must specify --ostbuild-meta=")
-
-        f = open(self.ostbuild_meta)
+        f = open(self.ostbuild_meta_path)
         self.metadata = json.load(f)
         f.close()
 
+        self.configargs.extend(self.metadata.get('config-opts', []))
+
         if self.metadata.get('rm-configure', False):
             configure_path = 'configure'
             if os.path.exists(configure_path):
@@ -235,6 +233,11 @@ class OstbuildCompileOne(builtins.Builtin):
                 except OSError, e:
                     pass
 
+        endtime = time.time()
+
+        log("Compliation succeeded; %d seconds elapsed" % (int(endtime - starttime),))
+        log("Results placed in %s" % (self.ostbuild_resultdir, ))
+
     def _install_and_unlink(self, src, dest):
         statsrc = os.lstat(src)
         dirname = os.path.dirname(dest)
diff --git a/src/ostbuild/pyostbuild/builtin_pull_components.py b/src/ostbuild/pyostbuild/builtin_pull_components.py
new file mode 100755
index 0000000..9b58e2e
--- /dev/null
+++ b/src/ostbuild/pyostbuild/builtin_pull_components.py
@@ -0,0 +1,70 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import copy
+import argparse
+import json
+import time
+import urlparse
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from . import ostbuildrc
+from . import buildutil
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import kvfile
+from . import odict
+
+class OstbuildPullComponents(builtins.Builtin):
+    name = "pull-components"
+    short_description = "Download the component data for active branch"
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('targets', nargs='*')
+
+        args = parser.parse_args(argv)
+
+        self.parse_active_branch()
+
+        if len(args.targets) == 0:
+            targets = [self.active_branch]
+        else:
+            targets = args.targets
+
+        tree_contents_list = []
+        for target in targets:
+            tree_contents_path = os.path.join(self.ostree_dir, target, 'contents.json')
+            tree_contents = json.load(open(tree_contents_path))
+            tree_contents_list.append(tree_contents)
+        revisions = set()
+        for tree_contents in tree_contents_list:
+            for content_item in tree_contents['contents']:
+                revisions.add(content_item['ostree-revision'])
+        args = ['ostree-pull', '--repo=' + self.repo]
+        # FIXME FIXME - don't hardcode origin here
+        args.append('origin')
+        for revision in revisions:
+            args.append(revision)
+        run_sync(args)
+        
+builtins.register(OstbuildPullComponents)
diff --git a/src/ostbuild/pyostbuild/builtin_query_content.py b/src/ostbuild/pyostbuild/builtin_query_content.py
index 89f5839..ab49912 100755
--- a/src/ostbuild/pyostbuild/builtin_query_content.py
+++ b/src/ostbuild/pyostbuild/builtin_query_content.py
@@ -35,33 +35,60 @@ class OstbuildQueryContent(builtins.Builtin):
     def __init__(self):
         builtins.Builtin.__init__(self)
 
+    def snapshot_from_contents(self, tree_contents_to_merge):
+        snapshot = {}
+        
+        # Arbitrarily merge in non-content metadata from
+        # the first snapshot.
+        first_content = tree_contents_to_merge[0]
+        for k,v in first_content.iteritems():
+            if k in ['base', 'contents']:
+                continue
+            snapshot[k] = v
+        components = {}
+        snapshot['components'] = components
+        targets = []
+        snapshot['targets'] = targets
+        for tree_contents in tree_contents_to_merge:
+            target = {}
+            targets.append(target)
+            target['base'] = tree_contents['base']
+            target['contents'] = tree_contents['contents']
+            for content_item in target['contents']:
+                name = content_item['name']
+                if name in components:
+                    continue
+                meta = self.get_component_meta_from_revision(content_item['ostree-revision'])
+                components[name] = meta
+        return snapshot
+
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
-        parser.add_argument('--branch', required=True)
+        parser.add_argument('targets', nargs='*')
         parser.add_argument('--component')
 
         args = parser.parse_args(argv)
         self.args = args
         self.parse_config()
 
-        contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                                  'cat', args.branch, 'contents.json'])
-        
-        if args.component is None:
-            sys.stdout.write(contents_json_text)
+        self.parse_active_branch()    
+
+        if args.component is not None:
+            meta = self.get_component_meta(args.component)
+            json.dump(meta, sys.stdout,
+                      indent=4, sort_keys=True)
         else:
-            contents = json.loads(contents_json_text)
-            contents_list = contents['contents']
-            found = False
-            for content in contents_list:
-                if content['name'] != args.component:
-                    found = True
-                    break
-            if not found:
-                fatal("Unknown component '%s'" % (args.component, ))
-            ostbuildmeta_json = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                                     'cat', content['ostree-revision'],
-                                                     '/_ostbuild-meta.json'])
-            sys.stdout.write(ostbuildmeta_json)
+            if len(self.args.targets) == 0:
+                targets = [self.active_branch]
+            else:
+                targets = self.args.targets
+
+            contents_to_merge = []
+            for target in targets:
+                contents_path = os.path.join(self.ostree_dir, target, 'contents.json')
+                contents = json.load(open(contents_path))
+                contents_to_merge.append(contents)
+            snapshot = self.snapshot_from_contents(contents_to_merge)
+            json.dump(snapshot, sys.stdout, indent=4, sort_keys=True)
 
 builtins.register(OstbuildQueryContent)
diff --git a/src/ostbuild/pyostbuild/builtin_replace_component.py b/src/ostbuild/pyostbuild/builtin_replace_component.py
new file mode 100755
index 0000000..d23c9dc
--- /dev/null
+++ b/src/ostbuild/pyostbuild/builtin_replace_component.py
@@ -0,0 +1,67 @@
+# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+from StringIO import StringIO
+import json
+import select,time
+import argparse
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+
+class OstbuildReplaceComponent(builtins.Builtin):
+    name = "replace-component"
+    short_description = "Replace contents of component in current filesystem"
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('component')
+        parser.add_argument('new-contents')
+
+        args = parser.parse_args(argv)
+        
+        self.parse_config()
+        self.parse_active_branch()
+
+        snapshot = self.get_component_snapshot(args.component)
+
+        replacing_trees = snapshot['trees']
+        previous_contents = set()
+        new_contents = set()
+        for tree in replacing_trees:
+            previous_contents_string = run_sync_get_output(['ostree', '--repo=' + self.repo,
+                                                            'ls', '--nul-filenames-only',
+                                                            '-R', snapshot['ostree-revision'],
+                                                            '/'])
+            for filename in previous_contents.split('\0'):
+                previous_contents.add(filename)
+        new_contents = run_sync_get_output(['find', '-print0'],
+                                           cwd=parser.new_contents)
+        for filename in new_contents.split('\0'):
+            new_contents.add(filename)
+
+        print "%r %r" % (previous_contents, new_contents)
+    
+builtins.register(OstbuildReplaceComponent)
diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py
index 3486396..ea70ea5 100755
--- a/src/ostbuild/pyostbuild/builtin_resolve.py
+++ b/src/ostbuild/pyostbuild/builtin_resolve.py
@@ -19,6 +19,7 @@ import os,sys,subprocess,tempfile,re,shutil
 import copy
 import argparse
 import json
+import time
 import urlparse
 from StringIO import StringIO
 
@@ -135,6 +136,7 @@ class OstbuildResolve(builtins.Builtin):
         self.args = args
         
         self.parse_config()
+        self.repo = ostbuildrc.get_key('repo')
 
         manifest_path = self.ostbuildrc.get_key('manifest')
         self.manifest = json.load(open(manifest_path))
@@ -193,9 +195,15 @@ class OstbuildResolve(builtins.Builtin):
                 component['patches']['files'] = patch_files
 
         name_prefix = snapshot['name-prefix']
+        del snapshot['name-prefix']
         base_prefix = snapshot['base-prefix']
+        del snapshot['base-prefix']
 
         manifest_architectures = snapshot['architectures']
+        
+        snapshot['architecture-buildroots'] = {}
+        for architecture in manifest_architectures:
+            snapshot['architecture-buildroots'][architecture] = '%s-%s-devel' % (base_prefix, architecture)
 
         components_by_name = {}
         component_ordering = []
@@ -278,7 +286,9 @@ class OstbuildResolve(builtins.Builtin):
             del component['name']
         snapshot['components'] = components_by_name
 
-        out_snapshot = os.path.join(self.workdir, '%s-snapshot.json' % (name_prefix, ))
+        current_time = time.time()
+
+        out_snapshot = os.path.join(self.workdir, '%s-src-snapshot-%d.json' % (name_prefix, int(current_time)))
         f = open(out_snapshot, 'w')
         json.dump(snapshot, f, indent=4, sort_keys=True)
         f.close()
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index 64f7be4..984c751 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -19,11 +19,13 @@
 
 import os
 import sys
+import stat
 import argparse
 import json
 
 from . import ostbuildrc
 from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
 
 _all_builtins = {}
 
@@ -31,9 +33,43 @@ class Builtin(object):
     name = None
     short_description = None
 
+    def __init__(self):
+        self._meta_cache = {}
+        self.manifest = None
+        self.snapshot = None
+        self.repo = None
+        self.ostree_dir = self._find_ostree_dir()
+        (self.active_branch, self.active_branch_checksum) = self._find_active_branch()
+
+    def _find_ostree_dir(self):
+        for path in ['/ostree', '/sysroot/ostree']:
+            if os.path.isdir(path):
+                return path
+        return None
+        
+    def _find_active_branch(self):
+        if self.ostree_dir is None:
+            return (None, None)
+        current_path = os.path.join(self.ostree_dir, 'current')
+        while True:
+            try:
+                target = os.path.join(self.ostree_dir, current_path)
+                stbuf = os.lstat(target)
+            except OSError, e:
+                current_path = None
+                break
+            if not stat.S_ISLNK(stbuf.st_mode):
+                break
+            current_path = os.readlink(target)
+        if current_path is not None:
+            basename = os.path.basename(current_path)
+            return basename.rsplit('-', 1)
+        else:
+            return (None, None)
+
     def parse_config(self):
         self.ostbuildrc = ostbuildrc
-        self.repo = ostbuildrc.get_key('repo')
+
         self.mirrordir = ostbuildrc.get_key('mirrordir')
         if not os.path.isdir(self.mirrordir):
             fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, ))
@@ -42,15 +78,52 @@ class Builtin(object):
             fatal("Specified workdir '%s' is not a directory" % (self.workdir, ))
         self.patchdir = os.path.join(self.workdir, 'patches')
 
-    def parse_manifest(self):
-        self.manifest_path = ostbuildrc.get_key('manifest')
-        self.manifest = json.load(open(self.manifest_path))
-        self.name_prefix = self.manifest['name-prefix']
+    def parse_active_branch(self):
+        if self.ostree_dir is None:
+            fatal("/ostree directory not found")
+        repo_path = os.path.join(self.ostree_dir, 'repo')
+        if not os.path.isdir(repo_path):
+            fatal("Repository '%s' doesn't exist" % (repo_path, ))
+        self.repo = repo_path
+        branch_path = os.path.join(self.ostree_dir, self.active_branch)
+        contents_path = os.path.join(branch_path, 'contents.json')
+        f = open(contents_path)
+        self.active_branch_contents = json.load(f)
+        f.close()
+
+    def get_component_snapshot(self, name):
+        found = False
+        for content in self.active_branch_contents['contents']:
+            if content['name'] == name:
+                found = True
+                break
+        if not found:
+            fatal("Unknown component '%s'" % (name, ))
+        return content
 
-    def parse_snapshot(self):
-        self.parse_manifest()
-        snapshot_path = os.path.join(self.workdir, '%s-snapshot.json' % (self.name_prefix, ))
-        self.snapshot = json.load(open(snapshot_path))
+    def get_component_meta_from_revision(self, revision):
+        text = run_sync_get_output(['ostree', '--repo=' + self.repo,
+                                    'cat', revision,
+                                    '/_ostbuild-meta.json'])
+        return json.loads(text)
+
+    def get_component_meta(self, name):
+        assert self.repo is not None
+
+        if self.snapshot is not None:
+            return self.snapshot['components'][name]
+
+        meta = self._meta_cache.get(name)
+        if meta is None:
+            content = self.get_component_snapshot(name)
+            meta = self.get_component_meta_from_revision(content['ostree-revision'])
+            self._meta_cache[name] = meta
+        return meta
+
+    def parse_snapshot(self, path):
+        self.repo = ostbuildrc.get_key('repo')
+        self.snapshot_path = path
+        self.snapshot = json.load(open(self.snapshot_path))
 
     def execute(self, args):
         raise NotImplementedError()
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 842cb5b..5b48d71 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -27,8 +27,10 @@ from . import builtin_checkout
 from . import builtin_chroot_compile_one
 from . import builtin_chroot_run_triggers
 from . import builtin_compile_one
+from . import builtin_pull_components
 from . import builtin_query_content
 from . import builtin_resolve
+from . import builtin_replace_component
 from . import builtin_status
 
 def usage(ecode):
diff --git a/src/ostbuild/pyostbuild/subprocess_helpers.py b/src/ostbuild/pyostbuild/subprocess_helpers.py
index 116d0fb..3754900 100755
--- a/src/ostbuild/pyostbuild/subprocess_helpers.py
+++ b/src/ostbuild/pyostbuild/subprocess_helpers.py
@@ -70,14 +70,16 @@ def run_sync_get_output(args, cwd=None, env=None, stdout=None, stderr=None, none
     return None
 
 def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False,
-             log_success=True, log_initiation=True, stdout=None,
+             log_success=True, log_initiation=True, stdin=None, stdout=None,
              stderr=None):
     if log_initiation:
         log("running: %s" % (subprocess.list2cmdline(args),))
 
     env_copy = _get_env_for_cwd(cwd, env)
 
-    if keep_stdin:
+    if stdin is not None:
+        stdin_target = stdin
+    elif keep_stdin:
         stdin_target = sys.stdin
     else:
         stdin_target = open('/dev/null', 'r')
[Date Prev][Date Next] [Thread Prev][Thread Next] [Thread Index] [Date Index] [Author Index]