GNOME.org
 

[ostree] gnomeos: Move everything into JSON, start plowing through X.org

commit fea117cd5e44472c198d3acf1d2d30b66c057a06
Author: Colin Walters <walters verbum org>
Date:   Thu Jan 5 19:10:09 2012 -0500

    gnomeos: Move everything into JSON, start plowing through X.org
    
    Also:
    
    * Stop generating tarballs for artifacts, use plain directories
    * Commit artifacts as uid/gid 0, with no xattrs
    
    Other misc stuff

 gnomeos/3.4/glib.txt                               |    1 -
 gnomeos/3.4/gobject-introspection.txt              |    1 -
 gnomeos/3.4/gtk-doc-stub.txt                       |    2 -
 gnomeos/3.4/libarchive.txt                         |    2 -
 gnomeos/3.4/libxslt.txt                            |    2 -
 gnomeos/3.4/manifest.json                          |  154 ++++++++++++++++-
 ...pixman-autogen.sh-Support-GNOME-Build-API.patch |   25 +++
 gnomeos/3.4/xorg-autogen.patch                     |   25 +++
 .../recipies-core/images/gnomeos-contents-devel.bb |    2 +-
 src/ostbuild/pyostbuild/buildutil.py               |    8 +-
 .../pyostbuild/builtin_autodiscover_meta.py        |    4 +-
 src/ostbuild/pyostbuild/builtin_build.py           |  182 ++++++++++++--------
 .../pyostbuild/builtin_chroot_compile_one.py       |   22 ++-
 src/ostbuild/pyostbuild/builtin_compile_one.py     |   50 +++---
 src/ostbuild/pyostbuild/builtins.py                |    6 +-
 src/ostbuild/pyostbuild/ostbuildlog.py             |    2 +-
 src/ostbuild/pyostbuild/subprocess_helpers.py      |   21 ++-
 17 files changed, 378 insertions(+), 131 deletions(-)
---
diff --git a/gnomeos/3.4/manifest.json b/gnomeos/3.4/manifest.json
index 9f23be1..e2a3a29 100644
--- a/gnomeos/3.4/manifest.json
+++ b/gnomeos/3.4/manifest.json
@@ -3,9 +3,157 @@
   "architectures": ["i686"],
   "base": "yocto/gnomeos-3.4",
 
+  "config-opts": ["--disable-static"],
+
+  "vcsconfig": {"gnome": "git:git://git.gnome.org/",
+                "fd": "git:git://anongit.freedesktop.org/git/"},
+
   "components": [
-   		"gtk-doc-stub",
-   		"gobject-introspection",
-   		"glib"
+   		{"src": "gnome:gtk-doc-stub",
+		 "component": "devel"},
+
+		{"src": "gnome:libxml2"},
+
+		{"src": "gnome:libxslt"},
+
+		{"src": "git:git://github.com/atgreen/libffi.git"},
+
+		{"src": "gnome:glib"},
+
+   		{"src": "fd:pixman"},
+
+		{"src": "fd:xorg/util/macros",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/bigreqsproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/compositeproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/damageproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/dmxproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/dri2proto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/evieproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/fixesproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/fontsproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/glproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/inputproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/kbproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/randrproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/recordproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/renderproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/resourceproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/scrnsaverproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/videoproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/xcmiscproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/xextproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/xf86bigfontproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/xf86dgaproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/xf86driproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/xf86miscproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/xf86vidmodeproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/xineramaproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/fixesproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/proto/xproto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/lib/libxtrans",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xcb/proto",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xcb/pthread-stubs",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xcb/libxcb",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:xorg/lib/libX11",
+		 "component": "devel",
+		 "patches": ["xorg-autogen.patch"]},
+
+		{"src": "fd:cairo"},
+
+		{"src": "gnome:gobject-introspection"}
 		]
 }
diff --git a/gnomeos/3.4/pixman-autogen.sh-Support-GNOME-Build-API.patch b/gnomeos/3.4/pixman-autogen.sh-Support-GNOME-Build-API.patch
new file mode 100644
index 0000000..bdc29e2
--- /dev/null
+++ b/gnomeos/3.4/pixman-autogen.sh-Support-GNOME-Build-API.patch
@@ -0,0 +1,25 @@
+From e4857495898aa2100f9dda77fbf7db68e61a0c48 Mon Sep 17 00:00:00 2001
+From: Colin Walters <walters verbum org>
+Date: Wed, 4 Jan 2012 08:06:05 -0500
+Subject: [PATCH] autogen.sh: Support GNOME Build API
+
+http://people.gnome.org/~walters/docs/build-api.txt
+---
+ autogen.sh |    4 +++-
+ 1 files changed, 3 insertions(+), 1 deletions(-)
+
+diff --git a/autogen.sh b/autogen.sh
+index 354f254..fc34bd5 100755
+--- a/autogen.sh
++++ b/autogen.sh
+@@ -9,4 +9,6 @@ cd $srcdir
+ autoreconf -v --install || exit 1
+ cd $ORIGDIR || exit $?
+ 
+-$srcdir/configure "$@"
++if test -z "$NOCONFIGURE"; then
++    $srcdir/configure "$@"
++fi
+-- 
+1.7.6.4
+
diff --git a/gnomeos/3.4/xorg-autogen.patch b/gnomeos/3.4/xorg-autogen.patch
new file mode 100644
index 0000000..5a3fd93
--- /dev/null
+++ b/gnomeos/3.4/xorg-autogen.patch
@@ -0,0 +1,25 @@
+From c3f75d8999b343131627a7f857e27f05a0f8d4aa Mon Sep 17 00:00:00 2001
+From: Colin Walters <walters verbum org>
+Date: Wed, 4 Jan 2012 17:37:06 -0500
+Subject: [PATCH] autogen.sh: Implement GNOME Build API
+
+http://people.gnome.org/~walters/docs/build-api.txt
+---
+ autogen.sh |    4 +++-
+ 1 files changed, 3 insertions(+), 1 deletions(-)
+
+diff --git a/autogen.sh b/autogen.sh
+index 904cd67..b47abdc 100755
+--- a/autogen.sh
++++ b/autogen.sh
+@@ -9,4 +9,6 @@ cd $srcdir
+ autoreconf -v --install || exit 1
+ cd $ORIGDIR || exit $?
+ 
+-$srcdir/configure --enable-maintainer-mode "$@"
++if test -z "$NOCONFIGURE"; then
++    $srcdir/configure --enable-maintainer-mode "$@"
++fi
+-- 
+1.7.6.4
+
diff --git a/gnomeos/yocto/recipies-core/images/gnomeos-contents-devel.bb b/gnomeos/yocto/recipies-core/images/gnomeos-contents-devel.bb
index 0e123b5..9606954 100644
--- a/gnomeos/yocto/recipies-core/images/gnomeos-contents-devel.bb
+++ b/gnomeos/yocto/recipies-core/images/gnomeos-contents-devel.bb
@@ -7,7 +7,7 @@ LIC_FILES_CHKSUM = "file://${COREBASE}/LICENSE;md5=3f40d7994397109285ec7b81fdeb3
 
 inherit gnomeos-contents
 
-PACKAGE_INSTALL += "task-core-sdk-dev \
+PACKAGE_INSTALL += "task-core-sdk \
 		python-dev \
 		bison flex \
 		git \
diff --git a/src/ostbuild/pyostbuild/buildutil.py b/src/ostbuild/pyostbuild/buildutil.py
index 26bea32..36e5954 100755
--- a/src/ostbuild/pyostbuild/buildutil.py
+++ b/src/ostbuild/pyostbuild/buildutil.py
@@ -17,7 +17,9 @@
 
 import re
 
-ARTIFACT_RE = re.compile(r'^artifact-([^,]+),([^,]+),([^,]+),([^,]+),(.+)-((?:runtime)|(?:devel))\.tar\.gz$')
+from .subprocess_helpers import run_sync_get_output
+
+ARTIFACT_RE = re.compile(r'^artifact-([^,]+),([^,]+),([^,]+),([^,]+),(.+)-((?:runtime)|(?:devel))\.tar$')
 
 def parse_artifact_name(artifact_basename):
     match = ARTIFACT_RE.match(artifact_basename)
@@ -36,3 +38,7 @@ def branch_name_for_artifact(a):
                                       a['branch'],
                                       a['type'])
 
+def get_git_version_describe(dirpath):
+    version = run_sync_get_output(['git', 'describe', '--long', '--abbrev=42', '--always'],
+                                  cwd=dirpath)
+    return version.strip()
diff --git a/src/ostbuild/pyostbuild/builtin_autodiscover_meta.py b/src/ostbuild/pyostbuild/builtin_autodiscover_meta.py
index 3d7ec8f..fe69be0 100755
--- a/src/ostbuild/pyostbuild/builtin_autodiscover_meta.py
+++ b/src/ostbuild/pyostbuild/builtin_autodiscover_meta.py
@@ -21,6 +21,7 @@ import os,sys,re,subprocess,tempfile,shutil
 import argparse
 
 from . import builtins
+from . import buildutil
 from .ostbuildlog import log, fatal
 
 class OstbuildAutodiscoverMeta(builtins.Builtin):
@@ -72,8 +73,7 @@ class OstbuildAutodiscoverMeta(builtins.Builtin):
         
     def _discover_version_from_git(self):
         if os.path.isdir('.git'):
-            version = subprocess.check_output(['git', 'describe', '--long', '--abbrev=42', '--always'])
-            return version.strip()
+            return buildutil.get_git_version_describe(os.getcwd())
         return None
         
     def _discover_branch_from_git(self):
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index fa13f83..e781b46 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -38,7 +38,7 @@ class OstbuildBuild(builtins.Builtin):
 
     def _ensure_vcs_mirror(self, name, keytype, uri, branch):
         assert keytype == 'git'
-        mirror = os.path.join(self.srcdir, name)
+        mirror = os.path.join(self.mirrordir, name)
         tmp_mirror = mirror + '.tmp'
         if os.path.isdir(tmp_mirror):
             shutil.rmtree(tmp_mirror)
@@ -48,7 +48,7 @@ class OstbuildBuild(builtins.Builtin):
         return mirror
 
     def _get_vcs_checkout(self, name, keytype, mirrordir, branch):
-        checkoutdir = os.path.join(self.srcdir, '_checkouts')
+        checkoutdir = os.path.join(self.workdir, 'src')
         if not os.path.isdir(checkoutdir):
             os.makedirs(checkoutdir)
         dest = os.path.join(checkoutdir, name)
@@ -57,16 +57,12 @@ class OstbuildBuild(builtins.Builtin):
             shutil.rmtree(dest)
         if os.path.isdir(tmp_dest):
             shutil.rmtree(tmp_dest)
-        subprocess.check_call(['git', 'clone', '--depth=1', '-q', mirrordir, tmp_dest])
+        subprocess.check_call(['git', 'clone', '-q', mirrordir, tmp_dest])
         subprocess.check_call(['git', 'checkout', '-q', branch], cwd=tmp_dest)
         subprocess.check_call(['git', 'submodule', 'update', '--init'], cwd=tmp_dest)
         os.rename(tmp_dest, dest)
         return dest
 
-    def _get_vcs_version_from_checkout(self, name):
-        vcsdir = os.path.join(self.srcdir, name)
-        return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=vcsdir)
-
     def _parse_src_key(self, srckey):
         idx = srckey.find(':')
         if idx < 0:
@@ -83,16 +79,6 @@ class OstbuildBuild(builtins.Builtin):
             uri = uri[0:idx]
         return (keytype, uri, branch)
 
-    def _parse_artifact_vcs_version(self, ver):
-        idx = ver.rfind('-')
-        if idx > 0:
-            vcs_ver = ver[idx+1:]
-        else:
-            vcs_ver = ver
-        if not vcs_ver.startswith('g'):
-            raise ValueError("Invalid artifact version '%s'" % (ver, ))
-        return vcs_ver[1:]
-
     def _get_ostbuild_chroot_args(self, architecture):
         current_machine = os.uname()[4]
         if current_machine != architecture:
@@ -111,46 +97,106 @@ class OstbuildBuild(builtins.Builtin):
         run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
         fatal("Exiting after debug shell")
 
-    def _build_one_component(self, name, architecture, meta):
-        (keytype, uri, branch) = self._parse_src_key(meta['SRC'])
-        component_vcs_mirror = self._ensure_vcs_mirror(name, keytype, uri, branch)
-        component_src = self._get_vcs_checkout(name, keytype, component_vcs_mirror, branch)
+    def _resolve_component_meta(self, component_meta):
+        result = dict(component_meta)
+        orig_src = component_meta['src']
+
+        did_expand = False
+        for (vcsprefix, expansion) in self.manifest['vcsconfig'].iteritems():
+            prefix = vcsprefix + ':'
+            if orig_src.startswith(prefix):
+                result['src'] = expansion + orig_src[len(prefix):]
+                did_expand = True
+                break
+
+        name = component_meta.get('name')
+        if name is None:
+            if did_expand:
+                src = orig_src
+                idx = src.rindex(':')
+                name = src[idx+1:]
+            else:
+                src = result['src']
+                idx = src.rindex('/')
+                name = src[idx+1:]
+            if name.endswith('.git'):
+                name = name[:-4]
+            name = name.replace('/', '-')
+            result['name'] = name
+        return result
+
+    def _build_one_component(self, meta, architecture):
+        name = meta['name']
+
+        (keytype, uri, branch) = self._parse_src_key(meta['src'])
+
         buildroot = '%s-%s-devel' % (self.manifest['name'], architecture)
-        branchname = 'artifacts/%s/%s/%s' % (buildroot, name, branch)
+        runtime_branchname = 'artifacts/%s/%s/%s/runtime' % (buildroot, name, branch)
         current_buildroot_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
                                                          'rev-parse', buildroot])
         current_buildroot_version = current_buildroot_version.strip()
+
+        artifact_base = {'buildroot': buildroot,
+                         'buildroot_version': current_buildroot_version,
+                         'name': name,
+                         'branch': branch,
+                         }
+
+        component_vcs_mirror = self._ensure_vcs_mirror(name, keytype, uri, branch)
+        component_src = self._get_vcs_checkout(name, keytype, component_vcs_mirror, branch)
+
+        current_vcs_version = buildutil.get_git_version_describe(component_src)
+        artifact_base['version'] = current_vcs_version
+
         previous_commit_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
-                                                       'rev-parse', branchname],
+                                                       'rev-parse', runtime_branchname],
                                                       stderr=open('/dev/null', 'w'),
                                                       none_on_error=True)
         if previous_commit_version is not None:
-            log("Previous build of '%s' is %s" % (branchname, previous_commit_version))
+            log("Previous build of '%s' is %s" % (runtime_branchname, previous_commit_version))
+
             previous_artifact_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
                                                              'show', '--print-metadata-key=ostbuild-artifact-version', previous_commit_version])
             previous_artifact_version = previous_artifact_version.strip()
             previous_buildroot_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
                                                               'show', '--print-metadata-key=ostbuild-buildroot-version', previous_commit_version])
             previous_buildroot_version = previous_buildroot_version.strip()
-            
-            previous_vcs_version = self._parse_artifact_vcs_version(previous_artifact_version)
-            current_vcs_version = self._get_vcs_version_from_checkout(name)
+
+            previous_artifact_base = dict(artifact_base)
+            previous_artifact_base['version'] = previous_artifact_version
+            previous_artifact_base['buildroot_version'] = previous_buildroot_version
+
+            previous_artifact_runtime = dict(previous_artifact_base)
+            previous_artifact_runtime['type'] = 'runtime'
+            previous_artifact_devel = dict(previous_artifact_base)
+            previous_artifact_devel['type'] = 'devel'
+            previous_artifacts = [previous_artifact_runtime,
+                                  previous_artifact_devel]
+                
             vcs_version_matches = False
-            if previous_vcs_version == current_vcs_version:
+            if previous_artifact_version == current_vcs_version:
                 vcs_version_matches = True
-                log("VCS version is unchanged from '%s'" % (previous_vcs_version, ))
+                log("VCS version is unchanged from '%s'" % (previous_artifact_version, ))
+                if self.buildopts.skip_built:
+                    return previous_artifacts
             else:
-                log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
+                log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_artifact_version))
             buildroot_version_matches = False
             if vcs_version_matches:    
                 buildroot_version_matches = (current_buildroot_version == previous_buildroot_version)
                 if buildroot_version_matches:
-                    log("Already have build '%s' of src commit '%s' for '%s' in buildroot '%s'" % (previous_commit_version, previous_vcs_version, branchname, buildroot))
-                    return
+                    log("Already have build '%s' of src commit '%s' for '%s' in buildroot '%s'" % (previous_commit_version, previous_artifact_version, runtime_branchname, buildroot))
+                    return previous_artifacts
                 else:
                     log("Buildroot is now '%s'" % (current_buildroot_version, ))
         else:
-            log("No previous build for '%s' found" % (branchname, ))
+            log("No previous build for '%s' found" % (runtime_branchname, ))
+
+        patches = meta.get('patches')
+        if patches is not None:
+            for patch in patches:
+                patch_path = os.path.join(self.manifestdir, patch)
+                run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=component_src)
         
         component_resultdir = os.path.join(self.workdir, name, 'results')
         if os.path.isdir(component_resultdir):
@@ -161,31 +207,33 @@ class OstbuildBuild(builtins.Builtin):
         chroot_args.extend(['--buildroot=' + buildroot,
                             '--workdir=' + self.workdir,
                             '--resultdir=' + component_resultdir])
+        global_config_opts = self.manifest.get('config-opts')
+        if global_config_opts is not None:
+            chroot_args.extend(global_config_opts)
         if self.buildopts.shell_on_failure:
             ecode = run_sync(chroot_args, cwd=component_src, fatal_on_error=False)
             if ecode != 0:
                 self._launch_debug_shell(architecture, buildroot, cwd=component_src)
         else:
             run_sync(chroot_args, cwd=component_src, fatal_on_error=True)
-        artifact_files = []
-        for name in os.listdir(component_resultdir):
-            if name.startswith('artifact-'):
-                log("Generated artifact file: %s" % (name, ))
-                artifact_files.append(os.path.join(component_resultdir, name))
-        assert len(artifact_files) >= 1 and len(artifact_files) <= 2
-        run_sync(['ostbuild', 'commit-artifacts',
-                  '--repo=' + self.repo] + artifact_files)
+
         artifacts = []
-        for filename in artifact_files:
-            parsed = buildutil.parse_artifact_name(os.path.basename(filename))
-            artifacts.append(parsed)
-        def _sort_artifact(a, b):
-            if a['type'] == b['type']:
-                return 0
-            elif a['type'] == 'runtime':
-                return -1
-            return 1
-        artifacts.sort(_sort_artifact)
+        for artifact_type in ['runtime', 'devel']:
+            artifact = dict(artifact_base)
+            artifacts.append(artifact)
+            artifact['type'] = artifact_type
+
+            artifact_branch = buildutil.branch_name_for_artifact(artifact)
+
+            artifact_resultdir = os.path.join(component_resultdir, artifact_branch)
+                                              
+            run_sync(['ostree', '--repo=' + self.repo,
+                      'commit', '-b', artifact_branch, '-s', 'Build ' + artifact_base['version'],
+                     '--add-metadata-string=ostbuild-buildroot-version=' + current_buildroot_version,
+                     '--add-metadata-string=ostbuild-artifact-version=' + artifact_base['version'],
+                     '--owner-uid=0', '--owner-gid=0', '--no-xattrs', 
+                     '--skip-if-unchanged'],
+                     cwd=artifact_resultdir)
         return artifacts
 
     def _compose(self, suffix, artifacts):
@@ -199,7 +247,7 @@ class OstbuildBuild(builtins.Builtin):
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--manifest', required=True)
-        parser.add_argument('--start-at')
+        parser.add_argument('--skip-built', action='store_true')
         parser.add_argument('--shell-on-failure', action='store_true')
         parser.add_argument('--debug-shell', action='store_true')
 
@@ -209,6 +257,7 @@ class OstbuildBuild(builtins.Builtin):
 
         self.buildopts = BuildOptions()
         self.buildopts.shell_on_failure = args.shell_on_failure
+        self.buildopts.skip_built = args.skip_built
 
         self.manifest = json.load(open(args.manifest))
 
@@ -217,43 +266,28 @@ class OstbuildBuild(builtins.Builtin):
             debug_shell_buildroot = '%s-%s-devel' % (self.manifest['name'], debug_shell_arch)
             self._launch_debug_shell(debug_shell_arch, debug_shell_buildroot)
 
-        dirname = os.path.dirname(args.manifest)
-        components = self.manifest['components']
+        self.manifestdir = os.path.dirname(args.manifest)
         runtime_components = []
         devel_components = []
         runtime_artifacts = []
         devel_artifacts = []
-        if args.start_at:
-            start_at_index = -1 
-            for i,component_name in enumerate(components):
-                if component_name == args.start_at:
-                    start_at_index = i
-                    break
-            if start_at_index == -1:
-                fatal("Unknown component '%s' for --start-at" % (args.start_at, ))
-        else:
-            start_at_index = 0
             
-        for component_name in components[start_at_index:]:
+        for component in self.manifest['components']:
             for architecture in self.manifest['architectures']:
-                path = os.path.join(dirname, component_name + '.txt')
-                f = open(path)
-                component_meta = kvfile.parse(f)
+                component_meta = self._resolve_component_meta(component)
     
-                artifact_branches = self._build_one_component(component_name, architecture, component_meta)
+                artifact_branches = self._build_one_component(component_meta, architecture)
     
-                target_component = component_meta.get('COMPONENT')
+                target_component = component_meta.get('component')
                 if target_component == 'devel':
-                    devel_components.append(component_name)
+                    devel_components.append(component_meta['name'])
                 else:
-                    runtime_components.append(component_name)
+                    runtime_components.append(component_meta['name'])
                     for branch in artifact_branches:
                         if branch['type'] == 'runtime':
                             runtime_artifacts.append(branch)
                 devel_artifacts.extend(artifact_branches)
 
-                f.close()
-
                 devel_branches = map(buildutil.branch_name_for_artifact, devel_artifacts)
                 self._compose(architecture + '-devel', devel_branches)
                 runtime_branches = map(buildutil.branch_name_for_artifact, runtime_artifacts)
diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
index 37b15ac..d78399c 100755
--- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
@@ -47,10 +47,10 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         parser.add_argument('--meta')
         parser.add_argument('--debug-shell', action='store_true')
         
-        args = parser.parse_args(argv)
+        (args, rest_args) = parser.parse_known_args(argv)
 
         if args.meta is None:
-            output = subprocess.check_output(['ostbuild-autodiscover-meta'])
+            output = subprocess.check_output(['ostbuild', 'autodiscover-meta'])
             ostbuild_meta_f = StringIO(output)
         else:
             ostbuild_meta_f = open(args.meta)
@@ -133,12 +133,18 @@ class OstbuildChrootCompileOne(builtins.Builtin):
                       '--mount-bind', os.getcwd(), chroot_sourcedir]
         if args.resultdir:
             child_args.extend(['--mount-bind', args.resultdir, '/ostbuild/results'])
-        child_args.extend([rootdir, '/bin/sh'])
-        if not args.debug_shell:
-            child_args.extend(['-c',
-                               'cd "%s" && ostbuild compile-one --ostbuild-resultdir=/ostbuild/results --ostbuild-meta=_ostbuild-meta' % (chroot_sourcedir, )
-                               ])
-        run_sync(child_args, env=BUILD_ENV)
+        if args.debug_shell:
+            child_args.extend([rootdir, '/bin/sh'])
+        else:
+            child_args.extend([rootdir, '/usr/bin/ostbuild',
+                               'compile-one',
+                               '--chdir=' + chroot_sourcedir,
+                               '--ostbuild-resultdir=/ostbuild/results',
+                               '--ostbuild-meta=_ostbuild-meta'])
+            child_args.extend(rest_args)
+        env_copy = dict(BUILD_ENV)
+        env_copy['PWD'] = chroot_sourcedir
+        run_sync(child_args, env=env_copy)
         
         if workdir_is_tmp:
             shutil.rmtree(workdir)
diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py
index 4306706..58d9be9 100755
--- a/src/ostbuild/pyostbuild/builtin_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_compile_one.py
@@ -85,18 +85,25 @@ class OstbuildCompileOne(builtins.Builtin):
                       '--infodir=' + os.path.join(PREFIX, 'share', 'info')]
         self.makeargs = ['make']
 
-        self.ostbuild_resultdir=os.getcwd()
+        self.ostbuild_resultdir=None
         self.ostbuild_meta=None
 
+        chdir = None
+
         for arg in args:
             if arg.startswith('--ostbuild-resultdir='):
                 self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):]
             elif arg.startswith('--ostbuild-meta='):
                 self.ostbuild_meta=arg[len('--ostbuild-meta='):]
+            elif arg.startswith('--chdir='):
+                os.chdir(arg[len('--chdir='):])
             elif arg.startswith('--'):
                 self.configargs.append(arg)
             else:
                 self.makeargs.append(arg)
+        
+        if self.ostbuild_resultdir is None:
+            fatal("Must specify --ostbuild-resultdir=")
 
         self.metadata = {}
 
@@ -107,6 +114,8 @@ class OstbuildCompileOne(builtins.Builtin):
             ostbuild_meta_f = open(self.ostbuild_meta)
 
         for line in ostbuild_meta_f:
+            if line == '':
+                continue
             (k,v) = line.split('=', 1)
             self.metadata[k.strip()] = v.strip()
 
@@ -198,7 +207,7 @@ class OstbuildCompileOne(builtins.Builtin):
         else:
             root_version = self.metadata.get('BUILDROOT_VERSION')
     
-        artifact_prefix='artifact-%s,%s,%s,%s,%s' % (root_name, root_version, name, branch, version)
+        artifact_prefix=os.path.join('artifacts', root_name, name, branch)
 
         tempdir = tempfile.mkdtemp(prefix='ostbuild-%s-' % (name,))
         self.tempfiles.append(tempdir)
@@ -206,6 +215,7 @@ class OstbuildCompileOne(builtins.Builtin):
         run_sync(args, cwd=builddir)
     
         devel_files = set()
+        dbg_files = set()
         runtime_files = set()
     
         oldpwd=os.getcwd()
@@ -234,9 +244,8 @@ class OstbuildCompileOne(builtins.Builtin):
                     runtime_files.add(path)
         os.chdir(oldpwd)
     
-        if devel_files:
-            self.make_artifact(artifact_prefix + '-devel', devel_files, tempdir=tempdir, resultdir=self.ostbuild_resultdir)
-        self.make_artifact(artifact_prefix + '-runtime', runtime_files, tempdir=tempdir, resultdir=self.ostbuild_resultdir)
+        self.make_artifact(artifact_prefix, 'devel', devel_files, tempdir=tempdir)
+        self.make_artifact(artifact_prefix, 'runtime', runtime_files, tempdir=tempdir)
 
         for tmpname in self.tempfiles:
             assert os.path.isabs(tmpname)
@@ -245,27 +254,22 @@ class OstbuildCompileOne(builtins.Builtin):
             else:
                 try:
                     os.unlink(tmpname)
-                    pass
                 except OSError, e:
                     pass
     
-    def make_artifact(self, name, from_files, tempdir=None, resultdir=None):
-        targz_name = name + '.tar.gz'
-        (fd,filelist_temp)=tempfile.mkstemp(prefix='ostbuild-filelist-%s' % (name, ))
-        os.close(fd)
-        self.tempfiles.append(filelist_temp)
-        f = open(filelist_temp, 'w')
+    def make_artifact(self, prefix, dirtype, from_files, tempdir):
+        resultdir = os.path.join(self.ostbuild_resultdir, prefix, dirtype)
+        if os.path.isdir(resultdir):
+            shutil.rmtree(resultdir)
+        os.makedirs(resultdir)
+                                 
         for filename in from_files:
-            assert ('\n' not in filename)
-            f.write(filename)
-            f.write('\n')
-        f.close()
-        if resultdir:
-            result_path = os.path.join(resultdir, targz_name)
-        else:
-            result_path = targz_name
-        args = ['tar', '-c', '-z', '-C', tempdir, '-f', result_path, '-T', filelist_temp]
-        run_sync(args)
-        log("created: %s" % (os.path.abspath (result_path), ))
+            src_path = os.path.join(tempdir, filename)
+            dest_path = os.path.join(resultdir, filename)
+            dest_dir = os.path.dirname(dest_path)
+            if not os.path.isdir(dest_dir):
+                os.makedirs(dest_dir)
+            shutil.move(src_path, dest_path)
+        log("created: %s" % (os.path.abspath (resultdir), ))
     
 builtins.register(OstbuildCompileOne)
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index b997c07..5fd4b33 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -32,9 +32,9 @@ class Builtin(object):
 
     def parse_config(self):
         self.repo = ostbuildrc.get_key('repo')
-        self.srcdir = ostbuildrc.get_key('srcdir')
-        if not os.path.isdir(self.srcdir):
-            fatal("Specified srcdir '%s' is not a directory" % (self.srcdir, ))
+        self.mirrordir = ostbuildrc.get_key('mirrordir')
+        if not os.path.isdir(self.mirrordir):
+            fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, ))
         self.workdir = ostbuildrc.get_key('workdir')
         if not os.path.isdir(self.workdir):
             fatal("Specified workdir '%s' is not a directory", (self.workdir, ))
diff --git a/src/ostbuild/pyostbuild/ostbuildlog.py b/src/ostbuild/pyostbuild/ostbuildlog.py
index 2174d71..a4af2a6 100755
--- a/src/ostbuild/pyostbuild/ostbuildlog.py
+++ b/src/ostbuild/pyostbuild/ostbuildlog.py
@@ -21,7 +21,7 @@ import os
 import sys
 
 def log(msg):
-    fullmsg = '%s: %s\n' % (sys.argv[0], msg)
+    fullmsg = '%s: %s\n' % (os.path.basename(sys.argv[0]), msg)
     sys.stdout.write(fullmsg)
     sys.stdout.flush()
 
diff --git a/src/ostbuild/pyostbuild/subprocess_helpers.py b/src/ostbuild/pyostbuild/subprocess_helpers.py
index 5b1557f..efa06c2 100755
--- a/src/ostbuild/pyostbuild/subprocess_helpers.py
+++ b/src/ostbuild/pyostbuild/subprocess_helpers.py
@@ -40,8 +40,8 @@ def _get_env_for_cwd(cwd=None, env=None):
         env_copy = env
     return env_copy
 
-def run_sync_get_output(args, cwd=None, env=None, stderr=None, none_on_error=False):
-    log("running: %s" % (subprocess.list2cmdline(args),))
+def run_sync_get_output(args, cwd=None, env=None, stderr=None, none_on_error=False,
+                        log_success=False):
     env_copy = _get_env_for_cwd(cwd, env)
     f = open('/dev/null', 'r')
     if stderr is None:
@@ -54,14 +54,18 @@ def run_sync_get_output(args, cwd=None, env=None, stderr=None, none_on_error=Fal
     output = proc.communicate()[0].strip()
     if proc.returncode != 0 and not none_on_error:
         logfn = fatal
-    else:
+    elif log_success:
         logfn = log
-    logfn("pid %d exited with code %d, %d bytes of output" % (proc.pid, proc.returncode, len(output)))
+    else:
+        logfn = None
+    if logfn is not None:
+        logfn("cmd '%s' exited with code %d, %d bytes of output" % (subprocess.list2cmdline(args), proc.returncode, len(output)))
     if proc.returncode == 0:
         return output
     return None
 
-def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False):
+def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False,
+             log_success=True):
     log("running: %s" % (subprocess.list2cmdline(args),))
     # This dance is necessary because we want to keep the PWD
     # environment variable up to date.  Not doing so is a recipie
@@ -88,7 +92,10 @@ def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False):
     returncode = proc.wait()
     if fatal_on_error and returncode != 0:
         logfn = fatal
-    else:
+    elif log_success:
         logfn = log
-    logfn("pid %d exited with code %d" % (proc.pid, returncode))
+    else:
+        logfn = None
+    if logfn is not None:
+        logfn("pid %d exited with code %d" % (proc.pid, returncode))
     return returncode
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]