GNOME.org
 

[ostree/wip/multibuild] more multibuild work

commit f12b7946b9cd08b9a2c9d47c886b4ea1a48a03ac
Author: Colin Walters <walters verbum org>
Date:   Mon Jan 2 15:11:38 2012 -0500

    more multibuild work

 gnomeos/3.4/bison.txt                              |    2 +
 gnomeos/3.4/flex.txt                               |    3 +
 gnomeos/3.4/gnulib.txt                             |    3 +
 gnomeos/3.4/gtk-doc-stub.txt                       |    2 +
 gnomeos/3.4/gtk-doc.txt                            |    2 -
 gnomeos/3.4/manifest.json                          |    6 +-
 src/ostbuild/pyostbuild/builtin_build.py           |   93 +++++++++++++++-----
 .../pyostbuild/builtin_chroot_compile_one.py       |    2 +-
 src/ostbuild/pyostbuild/builtin_compile_one.py     |    6 +-
 src/ostbuild/pyostbuild/subprocess_helpers.py      |   19 +++--
 10 files changed, 101 insertions(+), 37 deletions(-)
---
diff --git a/gnomeos/3.4/bison.txt b/gnomeos/3.4/bison.txt
new file mode 100644
index 0000000..b1b7822
--- /dev/null
+++ b/gnomeos/3.4/bison.txt
@@ -0,0 +1,2 @@
+SRC=git:git://git.savannah.gnu.org/bison.git
+COMPONENT=devel
diff --git a/gnomeos/3.4/flex.txt b/gnomeos/3.4/flex.txt
new file mode 100644
index 0000000..6af096b
--- /dev/null
+++ b/gnomeos/3.4/flex.txt
@@ -0,0 +1,3 @@
+UPSTREAM_SRC=git:git://github.com/cgwalters/flex
+SRC=git:file:///src/flex.git
+COMPONENT=devel
diff --git a/gnomeos/3.4/gnulib.txt b/gnomeos/3.4/gnulib.txt
new file mode 100644
index 0000000..6a704ab
--- /dev/null
+++ b/gnomeos/3.4/gnulib.txt
@@ -0,0 +1,3 @@
+UPSTREAM_SRC=git:git://git.savannah.gnu.org/gnulib.git
+SRC=git:file:///src/gnulib
+COMPONENT=devel
diff --git a/gnomeos/3.4/gtk-doc-stub.txt b/gnomeos/3.4/gtk-doc-stub.txt
new file mode 100644
index 0000000..75dd7a4
--- /dev/null
+++ b/gnomeos/3.4/gtk-doc-stub.txt
@@ -0,0 +1,2 @@
+SRC=git:git://git.gnome.org/gtk-doc-stub
+COMPONENT=devel
diff --git a/gnomeos/3.4/manifest.json b/gnomeos/3.4/manifest.json
index 0e38c44..ccd4a78 100644
--- a/gnomeos/3.4/manifest.json
+++ b/gnomeos/3.4/manifest.json
@@ -4,8 +4,10 @@
   "base": "gnomeos-3.4-yocto",
 
   "components": [
-   		"libxslt",
-   		"gtk-doc",
+   		"gtk-doc-stub",
+   		"gnulib",
+   		"flex",
+   		"bison",
    		"gobject-introspection"
 		]
 }
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index 6950108..48c53d4 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -26,6 +26,9 @@ from . import ostbuildrc
 from . import buildutil
 from . import kvfile
 
+class BuildOptions(object):
+    pass
+
 class OstbuildBuild(builtins.Builtin):
     name = "build"
     short_description = "Rebuild all artifacts from the given manifest"
@@ -33,17 +36,32 @@ class OstbuildBuild(builtins.Builtin):
     def __init__(self):
         builtins.Builtin.__init__(self)
 
-    def _ensure_vcs_checkout(self, name, keytype, uri, branch):
+    def _ensure_vcs_mirror(self, name, keytype, uri, branch):
         assert keytype == 'git'
-        destname = os.path.join(self.srcdir, name)
-        tmp_destname = destname + '.tmp'
-        if os.path.isdir(tmp_destname):
-            shutil.rmtree(tmp_destname)
-        if not os.path.isdir(destname):
-            run_sync(['git', 'clone', uri, tmp_destname])
-            os.rename(tmp_destname, destname)
-        subprocess.check_call(['git', 'checkout', '-q', branch], cwd=destname)
-        return destname
+        mirror = os.path.join(self.srcdir, name)
+        tmp_mirror = mirror + '.tmp'
+        if os.path.isdir(tmp_mirror):
+            shutil.rmtree(tmp_mirror)
+        if not os.path.isdir(mirror):
+            run_sync(['git', 'clone', '--mirror', uri, tmp_mirror])
+            os.rename(tmp_mirror, mirror)
+        return mirror
+
+    def _get_vcs_checkout(self, name, keytype, mirrordir, branch):
+        checkoutdir = os.path.join(self.srcdir, '_checkouts')
+        if not os.path.isdir(checkoutdir):
+            os.makedirs(checkoutdir)
+        dest = os.path.join(checkoutdir, name)
+        tmp_dest = dest + '.tmp'
+        if os.path.isdir(dest):
+            shutil.rmtree(dest)
+        if os.path.isdir(tmp_dest):
+            shutil.rmtree(tmp_dest)
+        subprocess.check_call(['git', 'clone', '--depth=1', '-q', mirrordir, tmp_dest])
+        subprocess.check_call(['git', 'checkout', '-q', branch], cwd=tmp_dest)
+        subprocess.check_call(['git', 'submodule', 'update', '--init'], cwd=tmp_dest)
+        os.rename(tmp_dest, dest)
+        return dest
 
     def _get_vcs_version_from_checkout(self, name):
         vcsdir = os.path.join(self.srcdir, name)
@@ -75,9 +93,20 @@ class OstbuildBuild(builtins.Builtin):
             raise ValueError("Invalid artifact version '%s'" % (ver, ))
         return vcs_ver[1:]
 
+    def _get_ostbuild_chroot_args(self, architecture):
+        current_machine = os.uname()[4]
+        if current_machine != architecture:
+            args = ['setarch', architecture]
+        else:
+            args = []
+        args.extend(['ostbuild', 'chroot-compile-one',
+                     '--repo=' + self.repo])
+        return args
+
     def _build_one_component(self, name, architecture, meta):
         (keytype, uri, branch) = self._parse_src_key(meta['SRC'])
-        component_src = self._ensure_vcs_checkout(name, keytype, uri, branch)
+        component_vcs_mirror = self._ensure_vcs_mirror(name, keytype, uri, branch)
+        component_src = self._get_vcs_checkout(name, keytype, component_vcs_mirror, branch)
         buildroot = '%s-%s-devel' % (self.manifest['name'], architecture)
         branchname = 'artifacts/%s/%s/%s' % (buildroot, name, branch)
         current_buildroot_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
@@ -119,18 +148,18 @@ class OstbuildBuild(builtins.Builtin):
         if os.path.isdir(component_resultdir):
             shutil.rmtree(component_resultdir)
         os.makedirs(component_resultdir)
-        current_machine = os.uname()[4]
-        if current_machine != architecture:
-            log("Current architecture '%s' differs from target '%s', using setarch" % (current_machine, architecture))
-            args = ['setarch', architecture]
+
+        chroot_args = self._get_ostbuild_chroot_args(architecture)
+        chroot_args.extend(['--buildroot=' + buildroot,
+                            '--workdir=' + self.workdir,
+                            '--resultdir=' + component_resultdir])
+        if self.buildopts.shell_on_failure:
+            ecode = run_sync(chroot_args, cwd=component_src, fatal_on_error=False)
+            if ecode != 0:
+                run_sync(chroot_args + ['--debug-shell'], cwd=component_src, keep_stdin=True, fatal_on_error=False)
+                fatal("Exiting after debug shell")
         else:
-            args = []
-        args.extend(['ostbuild', 'chroot-compile-one',
-                     '--repo=' + self.repo,
-                     '--buildroot=' + buildroot,
-                     '--workdir=' + self.workdir,
-                     '--resultdir=' + component_resultdir])
-        run_sync(args, cwd=component_src)
+            run_sync(chroot_args, cwd=component_src, fatal_on_error=True)
         artifact_files = []
         for name in os.listdir(component_resultdir):
             if name.startswith('artifact-'):
@@ -163,11 +192,16 @@ class OstbuildBuild(builtins.Builtin):
     def execute(self, argv):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--manifest', required=True)
+        parser.add_argument('--start-at')
+        parser.add_argument('--shell-on-failure', action='store_true')
 
         args = parser.parse_args(argv)
-
+        
         self.parse_config()
 
+        self.buildopts = BuildOptions()
+        self.buildopts.shell_on_failure = args.shell_on_failure
+
         self.manifest = json.load(open(args.manifest))
         dirname = os.path.dirname(args.manifest)
         components = self.manifest['components']
@@ -175,7 +209,18 @@ class OstbuildBuild(builtins.Builtin):
         devel_components = []
         runtime_artifacts = []
         devel_artifacts = []
-        for component_name in components:
+        if args.start_at:
+            start_at_index = -1 
+            for i,component_name in enumerate(components):
+                if component_name == args.start_at:
+                    start_at_index = i
+                    break
+            if start_at_index == -1:
+                fatal("Unknown component '%s' for --start-at" % (args.start_at, ))
+        else:
+            start_at_index = 0
+            
+        for component_name in components[start_at_index:]:
             for architecture in self.manifest['architectures']:
                 path = os.path.join(dirname, component_name + '.txt')
                 f = open(path)
diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
index 8c18965..71fb6bc 100755
--- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
@@ -45,7 +45,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
         parser.add_argument('--resultdir')
         parser.add_argument('--buildroot', required=True)
         parser.add_argument('--meta')
-        parser.add_argument('--debug-shell', type=bool)
+        parser.add_argument('--debug-shell', action='store_true')
         
         args = parser.parse_args(argv)
 
diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py
index ff87464..4306706 100755
--- a/src/ostbuild/pyostbuild/builtin_compile_one.py
+++ b/src/ostbuild/pyostbuild/builtin_compile_one.py
@@ -118,8 +118,10 @@ class OstbuildCompileOne(builtins.Builtin):
 
         autogen_script = None
         if not os.path.exists('configure'):
-            for name in ['autogen', 'autogen.sh']:
+            log("No 'configure' script found, looking for autogen/bootstrap")
+            for name in ['autogen', 'autogen.sh', 'bootstrap']:
                 if os.path.exists(name):
+                    log("Using bootstrap script '%s'" % (name, ))
                     autogen_script = name
             if autogen_script is None:
                 fatal("No configure or autogen script detected; unknown buildsystem")
@@ -128,6 +130,8 @@ class OstbuildCompileOne(builtins.Builtin):
             env = dict(os.environ)
             env['NOCONFIGURE'] = '1'
             run_sync(['./' + autogen_script], env=env)
+        else:
+            log("Using existing 'configure' script")
     
         use_builddir = True
         doesnot_support_builddir = self._has_buildapi_configure_variable('no-builddir')
diff --git a/src/ostbuild/pyostbuild/subprocess_helpers.py b/src/ostbuild/pyostbuild/subprocess_helpers.py
index a05679e..5b1557f 100755
--- a/src/ostbuild/pyostbuild/subprocess_helpers.py
+++ b/src/ostbuild/pyostbuild/subprocess_helpers.py
@@ -41,7 +41,7 @@ def _get_env_for_cwd(cwd=None, env=None):
     return env_copy
 
 def run_sync_get_output(args, cwd=None, env=None, stderr=None, none_on_error=False):
-    log("running: %r" % (args,))
+    log("running: %s" % (subprocess.list2cmdline(args),))
     env_copy = _get_env_for_cwd(cwd, env)
     f = open('/dev/null', 'r')
     if stderr is None:
@@ -61,9 +61,8 @@ def run_sync_get_output(args, cwd=None, env=None, stderr=None, none_on_error=Fal
         return output
     return None
 
-def run_sync(args, cwd=None, env=None):
-    log("running: %r" % (args,))
-    f = open('/dev/null', 'r')
+def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False):
+    log("running: %s" % (subprocess.list2cmdline(args),))
     # This dance is necessary because we want to keep the PWD
     # environment variable up to date.  Not doing so is a recipie
     # for triggering edge conditions in pwd lookup.
@@ -78,12 +77,18 @@ def run_sync(args, cwd=None, env=None):
             env_copy['PWD'] = cwd
     else:
         env_copy = env
-    proc = subprocess.Popen(args, stdin=f, stdout=sys.stdout, stderr=sys.stderr,
+    if keep_stdin:
+        target_stdin = sys.stdin
+    else:
+        target_stdin = open('/dev/null', 'r')
+    proc = subprocess.Popen(args, stdin=target_stdin, stdout=sys.stdout, stderr=sys.stderr,
                             close_fds=True, cwd=cwd, env=env_copy)
-    f.close()
+    if not keep_stdin:
+        target_stdin.close()
     returncode = proc.wait()
-    if returncode != 0:
+    if fatal_on_error and returncode != 0:
         logfn = fatal
     else:
         logfn = log
     logfn("pid %d exited with code %d" % (proc.pid, returncode))
+    return returncode
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]