[ostree/wip/ostbuild-v2] ostbuild: Drop architecture from component sources

[Colin Walters <walters src gnome org>]

commit 60e010a675fcd65c5811cabe9e137773c8d29f0b
Author: Colin Walters <walters verbum org>
Date:   Sat Apr 21 10:01:06 2012 -0400

    ostbuild: Drop architecture from component sources

 gnomeos/3.4/gnomeos-3.4-src.json                   |    6 +-
 .../pyostbuild/builtin_build_components.py         |   48 ++++++++----
 .../pyostbuild/builtin_chroot_compile_one.py       |   11 ++--
 src/ostbuild/pyostbuild/builtin_modify_snapshot.py |    3 -
 src/ostbuild/pyostbuild/builtin_resolve.py         |   76 ++++++++++----------
 5 files changed, 79 insertions(+), 65 deletions(-)
---
diff --git a/gnomeos/3.4/gnomeos-3.4-src.json b/gnomeos/3.4/gnomeos-3.4-src.json
index 6864e0c..257bc74 100644
--- a/gnomeos/3.4/gnomeos-3.4-src.json
+++ b/gnomeos/3.4/gnomeos-3.4-src.json
@@ -27,10 +27,12 @@
               "prefix": "gnomeos/3.4"},
 
   "components": [
-   		{"src": "cgwalters:ginitscripts"},
+   		{"src": "cgwalters:ginitscripts",
+		 "noarch": true},
 
    		{"src": "gnome:gtk-doc-stub",
-		 "component": "devel"},
+		 "component": "devel",
+		 "noarch": true},
 
 		{"src": "git:git://github.com/atgreen/libffi.git"},
 
diff --git a/src/ostbuild/pyostbuild/builtin_build_components.py b/src/ostbuild/pyostbuild/builtin_build_components.py
index 41bdaba..40f9b25 100755
--- a/src/ostbuild/pyostbuild/builtin_build_components.py
+++ b/src/ostbuild/pyostbuild/builtin_build_components.py
@@ -61,10 +61,10 @@ class OstbuildBuildComponents(builtins.Builtin):
         run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
         fatal("Exiting after debug shell")
 
-    def _build_one_component(self, name, component):
+    def _build_one_component(self, basename, component, architecture):
         branch = component['branch']
-        architecture = component['architecture']
 
+        name = '%s/%s' % (basename, architecture)
         buildname = 'components/%s' % (name, )
 
         current_vcs_version = component['revision']
@@ -74,7 +74,7 @@ class OstbuildBuildComponents(builtins.Builtin):
                                                      stderr=open('/dev/null', 'w'),
                                                      none_on_error=True)
         if previous_build_version is not None:
-            log("Previous build of '%s' is %s" % (buildname, previous_build_version))
+            log("Previous build of '%s' is %s" % (name, previous_build_version))
 
             previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
                                                           'cat', previous_build_version,
@@ -93,12 +93,12 @@ class OstbuildBuildComponents(builtins.Builtin):
             else:
                 log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
         else:
-            log("No previous build for '%s' found" % (buildname, ))
+            log("No previous build for '%s' found" % (name, ))
 
         checkoutdir = os.path.join(self.workdir, 'src')
-        component_src = os.path.join(checkoutdir, name)
+        component_src = os.path.join(checkoutdir, basename)
         run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
-                  '--clean', '--overwrite', name], cwd=checkoutdir)
+                  '--clean', '--overwrite', basename], cwd=checkoutdir)
 
         artifact_meta = dict(component)
 
@@ -113,7 +113,7 @@ class OstbuildBuildComponents(builtins.Builtin):
         log("Logging to %s" % (log_path, ))
         f = open(log_path, 'w')
         chroot_args = self._get_ostbuild_chroot_args(architecture)
-        chroot_args.extend(['--pristine', '--name=' + name])
+        chroot_args.extend(['--pristine', '--name=' + basename, '--arch=' + architecture])
         if self.buildopts.shell_on_failure:
             ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
             if ecode != 0:
@@ -149,7 +149,7 @@ class OstbuildBuildComponents(builtins.Builtin):
         output = run_sync_get_output(args)
         return output.split('\n')
 
-    def _save_bin_snapshot(self):
+    def _save_bin_snapshot(self, components, component_architectures):
         bin_snapshot = dict(self.snapshot)
 
         for target in bin_snapshot['targets']:
@@ -160,13 +160,16 @@ class OstbuildBuildComponents(builtins.Builtin):
             base['ostree-revision'] = base_revision
 
         component_refs = []
-        for name in bin_snapshot['components'].iterkeys():
-            component_refs.append('components/%s' % (name, ))
+        for name in components.iterkeys():
+            for architecture in component_architectures[name]:
+                component_refs.append('components/%s/%s' % (name, architecture))
 
         new_components = {}
         resolved_refs = self._resolve_refs(component_refs)
-        for name,rev in zip(bin_snapshot['components'].iterkeys(), resolved_refs):
-            new_components[name] = rev
+        for name,rev in zip(components.iterkeys(), resolved_refs):
+            for architecture in component_architectures[name]:
+                archname = '%s/%s' % (name, architecture)
+                new_components[archname] = rev
 
         bin_snapshot['components'] = new_components
 
@@ -193,9 +196,20 @@ class OstbuildBuildComponents(builtins.Builtin):
         self.buildopts.shell_on_failure = args.shell_on_failure
         self.buildopts.skip_built = args.skip_built
 
+        required_components = {}
+        component_architectures = {}
+        for target in self.snapshot['targets']:
+            for tree_content in target['contents']:
+                (name, arch) = tree_content['name'].rsplit('/', 1)
+                required_components[name] = self.snapshot['components'][name]
+                if name not in component_architectures:
+                    component_architectures[name] = [arch]
+                else:
+                    component_architectures[name].append(arch)
+
         build_component_order = []
         if len(args.components) == 0:
-            tsorted = buildutil.tsort_components(self.snapshot['components'], 'build-depends')
+            tsorted = buildutil.tsort_components(required_components, 'build-depends')
             tsorted.reverse()
             build_component_order = tsorted
         else:
@@ -220,9 +234,11 @@ class OstbuildBuildComponents(builtins.Builtin):
             start_at_index = 0
 
         for component_name in build_component_order[start_at_index:]:
-            component = self.snapshot['components'].get(component_name)
-            self._build_one_component(component_name, component)
+            component = required_components[component_name]
+            architectures = component_architectures[component_name]
+            for architecture in architectures:
+                self._build_one_component(component_name, component, architecture)
 
-        self._save_bin_snapshot()   
+        self._save_bin_snapshot(required_components, component_architectures)   
         
 builtins.register(OstbuildBuildComponents)
diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
index e3560b9..594936b 100755
--- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
@@ -32,7 +32,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
     name = "chroot-compile-one"
     short_description = "Build artifacts from the current source directory in a chroot"
 
-    def _compose_buildroot(self, component_name, dirpath):
+    def _compose_buildroot(self, component_name, architecture, dirpath):
         starttime = time.time()
 
         components = self.snapshot['components']
@@ -40,11 +40,11 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         component = components.get(component_name)
 
         buildroots = self.snapshot['architecture-buildroots']
-        base_devel_name = 'bases/' + buildroots[component['architecture']]
+        base_devel_name = 'bases/' + buildroots[architecture]
 
         checkout_trees = [(base_devel_name, '/')]
         for dependency_name in dependencies:
-            buildname = 'components/%s' % (dependency_name, )
+            buildname = 'components/%s/%s' % (dependency_name, architecture)
             checkout_trees.append((buildname, '/runtime'))
             checkout_trees.append((buildname, '/devel'))
 
@@ -80,6 +80,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         parser.add_argument('--pristine', action='store_true')
         parser.add_argument('--snapshot', required=True)
         parser.add_argument('--name')
+        parser.add_argument('--arch', required=True)
         parser.add_argument('--debug-shell', action='store_true')
         
         args = parser.parse_args(argv)
@@ -112,7 +113,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
             shutil.rmtree(child_tmpdir)
         fileutil.ensure_dir(child_tmpdir)
 
-        resultdir = os.path.join(self.workdir, 'results', component_name)
+        resultdir = os.path.join(self.workdir, 'results', component_name, args.arch)
         if os.path.isdir(resultdir):
             shutil.rmtree(resultdir)
         fileutil.ensure_dir(resultdir)
@@ -130,7 +131,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
             shutil.rmtree(rootdir_tmp)
         os.mkdir(rootdir_tmp)
             
-        self._compose_buildroot(component_name, rootdir_tmp)
+        self._compose_buildroot(component_name, args.arch, rootdir_tmp)
 
         builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
         os.mkdir(builddir_tmp)
diff --git a/src/ostbuild/pyostbuild/builtin_modify_snapshot.py b/src/ostbuild/pyostbuild/builtin_modify_snapshot.py
index eafe410..e1b2df5 100755
--- a/src/ostbuild/pyostbuild/builtin_modify_snapshot.py
+++ b/src/ostbuild/pyostbuild/builtin_modify_snapshot.py
@@ -15,9 +15,6 @@
 # Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 # Boston, MA 02111-1307, USA.
 
-# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
-# http://people.gnome.org/~walters/docs/build-api.txt
-
 import os,sys,stat,subprocess,tempfile,re,shutil
 from StringIO import StringIO
 import json
diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py
index 6d125d9..5cf94a8 100755
--- a/src/ostbuild/pyostbuild/builtin_resolve.py
+++ b/src/ostbuild/pyostbuild/builtin_resolve.py
@@ -40,6 +40,7 @@ class OstbuildResolve(builtins.Builtin):
         builtins.Builtin.__init__(self)
 
     def _ensure_vcs_mirror(self, name, keytype, uri, branch):
+        # FIXME - remove "name" from parameter list here - hash uri?
         mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri)
         tmp_mirror = mirror + '.tmp'
         if os.path.isdir(tmp_mirror):
@@ -206,45 +207,41 @@ class OstbuildResolve(builtins.Builtin):
 
         components_by_name = {}
         component_ordering = []
-        build_prev_component_by_arch = {}
-        runtime_prev_component_by_arch = {}
-        runtime_components_by_arch = {}
-        devel_components_by_arch = {}
-        for architecture in manifest_architectures:
-            runtime_components_by_arch[architecture] = []
-            devel_components_by_arch[architecture] = []
+        build_prev_component = None
+        runtime_prev_component = None
+        runtime_components = []
+        devel_components = []
 
-        for component in component_source_list:
-            component_architectures = component.get('architectures', manifest_architectures)
-            for architecture in component_architectures:
-                component_binary = copy.deepcopy(component)
-                source_name = component['name']
-                binary_name = '%s/%s/%s' % (name_prefix, architecture, source_name)
-                component_binary['name'] = binary_name
-                component_binary['architecture'] = architecture
+        builds = {}
 
-                components_by_name[binary_name] = component_binary
+        for component in component_source_list:
+            base_name = component['name']
+            name = '%s/%s' % (name_prefix, base_name)
+            component['name'] = name
 
-                prev_component = build_prev_component_by_arch.get(architecture)
-                if prev_component is not None:
-                    component_binary['build-depends'] = [prev_component['name']]
-                build_prev_component_by_arch[architecture] = component_binary
+            components_by_name[name] = component
 
-                is_runtime = component.get('component', 'runtime') == 'runtime'
+            if build_prev_component is not None:
+                component['build-depends'] = [build_prev_component['name']]
+            build_prev_component = component
 
-                prev_component = runtime_prev_component_by_arch.get(architecture)
-                if prev_component is not None:
-                    component_binary['runtime-depends'] = [prev_component['name']]
+            is_runtime = component.get('component', 'runtime') == 'runtime'
 
-                if is_runtime:
-                    runtime_prev_component_by_arch[architecture] = component_binary
+            if runtime_prev_component is not None:
+                component['runtime-depends'] = [runtime_prev_component['name']]
 
-                if is_runtime:
-                    runtime_components_by_arch[architecture].append(component_binary)
-                devel_components_by_arch[architecture].append(component_binary)
+            if is_runtime:
+                runtime_prev_component = component
+                runtime_components.append(component)
+            devel_components.append(component)
 
-                if 'architectures' in component_binary:
-                    del component_binary['architectures']
+            is_noarch = component.get('noarch', False)
+            if is_noarch:
+                # Just use the first specified architecture
+                component_arches = [manifest_architectures[0]]
+            else:
+                component_arches = component.get('architectures', manifest_architectures)
+            builds[name] = component_arches
 
         # We expanded these keys
         del snapshot['config-opts']
@@ -254,26 +251,27 @@ class OstbuildResolve(builtins.Builtin):
 
         targets_list = []
         snapshot['targets'] = targets_list
-        for architecture in manifest_architectures:
-            for target_component_type in ['runtime', 'devel']:
+        for target_component_type in ['runtime', 'devel']:
+            for architecture in manifest_architectures:
                 target = {}
                 targets_list.append(target)
                 target['name'] = '%s-%s-%s' % (name_prefix, architecture, target_component_type)
 
                 base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
-                base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                                     'rev-parse', 'bases/%s' % (base_ref, )])
                 target['base'] = {'name': base_ref}
 
                 if target_component_type == 'runtime':
-                    target_components = runtime_components_by_arch[architecture]
+                    target_components = runtime_components
                 else:
-                    target_components = devel_components_by_arch[architecture]
+                    target_components = devel_components
                     
                 contents = []
                 for component in target_components:
-                    name = component['name']
-                    component_ref = {'name': name}
+                    builds_for_component = builds[component['name']]
+                    if architecture not in builds_for_component:
+                        continue
+                    binary_name = '%s/%s' % (component['name'], architecture)
+                    component_ref = {'name': binary_name}
                     if target_component_type == 'runtime':
                         component_ref['trees'] = ['/runtime']
                     else: