[ostree] ostbuild: Break out git-mirror from resolve

[Colin Walters <walters src gnome org>]

commit 29b230107ea0db881255e7cc551de5215eb11d59
Author: Colin Walters <walters verbum org>
Date:   Tue May 1 20:51:29 2012 -0400

    ostbuild: Break out git-mirror from resolve
    
    Other bits

 Makefile-ostbuild.am                               |    1 +
 .../pyostbuild/builtin_build_components.py         |    1 +
 src/ostbuild/pyostbuild/builtin_git_mirror.py      |   63 +++++++++++++++++
 src/ostbuild/pyostbuild/builtin_resolve.py         |   71 ++------------------
 src/ostbuild/pyostbuild/main.py                    |    1 +
 src/ostbuild/pyostbuild/vcs.py                     |   63 +++++++++++++++++-
 6 files changed, 134 insertions(+), 66 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 15b53c9..4069558 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -30,6 +30,7 @@ pyostbuild_PYTHON =					\
 	src/ostbuild/pyostbuild/builtin_chroot_compile_one.py	\
 	src/ostbuild/pyostbuild/builtin_compile_one.py	\
 	src/ostbuild/pyostbuild/builtin_pull_components.py	\
+	src/ostbuild/pyostbuild/builtin_git_mirror.py	\
 	src/ostbuild/pyostbuild/builtin_prefix.py	\
 	src/ostbuild/pyostbuild/builtin_resolve.py	\
 	src/ostbuild/pyostbuild/builtin_modify_snapshot.py	\
diff --git a/src/ostbuild/pyostbuild/builtin_build_components.py b/src/ostbuild/pyostbuild/builtin_build_components.py
index af1ba7c..6adcbc7 100755
--- a/src/ostbuild/pyostbuild/builtin_build_components.py
+++ b/src/ostbuild/pyostbuild/builtin_build_components.py
@@ -96,6 +96,7 @@ class OstbuildBuildComponents(builtins.Builtin):
             log("No previous build for '%s' found" % (name, ))
 
         checkoutdir = os.path.join(self.workdir, 'checkouts')
+        fileutil.ensure_dir(checkoutdir)
         component_src = os.path.join(checkoutdir, basename)
         run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
                   '--clean', '--overwrite', basename], cwd=checkoutdir)
diff --git a/src/ostbuild/pyostbuild/builtin_git_mirror.py b/src/ostbuild/pyostbuild/builtin_git_mirror.py
new file mode 100755
index 0000000..5d827de
--- /dev/null
+++ b/src/ostbuild/pyostbuild/builtin_git_mirror.py
@@ -0,0 +1,63 @@
+# Copyright (C) 2011,2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
+# http://people.gnome.org/~walters/docs/build-api.txt
+
+import os,sys,stat,subprocess,tempfile,re,shutil
+import argparse
+from StringIO import StringIO
+import json
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from . import vcs
+from .subprocess_helpers import run_sync, run_sync_get_output
+from . import buildutil
+
+class OstbuildGitMirror(builtins.Builtin):
+    name = "git-mirror"
+    short_description = "Update internal git mirror for one or more components"
+
+    def __init__(self):
+        builtins.Builtin.__init__(self)
+
+    def execute(self, argv):
+        parser = argparse.ArgumentParser(description=self.short_description)
+        parser.add_argument('--prefix')
+        parser.add_argument('--src-snapshot')
+        parser.add_argument('--fetch', action='store_true')
+        parser.add_argument('components', nargs='*')
+
+        args = parser.parse_args(argv)
+        self.parse_config()
+        self.parse_snapshot(args.prefix, args.src_snapshot)
+
+        if len(args.components) == 0:
+            components = self.snapshot['components'].keys()
+        else:
+            components = args.components
+        for name in components:
+            component = self.snapshot['components'][name]
+            src = component['src']
+            (keytype, uri) = vcs.parse_src_key(src)
+            mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, component['branch'])
+
+            if args.fetch:
+                run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False)
+
+builtins.register(OstbuildGitMirror)
diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py
index 2c39b05..96116be 100755
--- a/src/ostbuild/pyostbuild/builtin_resolve.py
+++ b/src/ostbuild/pyostbuild/builtin_resolve.py
@@ -27,6 +27,7 @@ from . import builtins
 from .ostbuildlog import log, fatal
 from .subprocess_helpers import run_sync, run_sync_get_output
 from . import ostbuildrc
+from . import vcs
 from . import jsondb
 from . import buildutil
 from . import kvfile
@@ -39,64 +40,6 @@ class OstbuildResolve(builtins.Builtin):
     def __init__(self):
         builtins.Builtin.__init__(self)
 
-    def _ensure_vcs_mirror(self, name, keytype, uri, branch):
-        # FIXME - remove "name" from parameter list here - hash uri?
-        mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri)
-        tmp_mirror = mirror + '.tmp'
-        if os.path.isdir(tmp_mirror):
-            shutil.rmtree(tmp_mirror)
-        if not os.path.isdir(mirror):
-            run_sync(['git', 'clone', '--mirror', uri, tmp_mirror])
-            run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror)
-            os.rename(tmp_mirror, mirror)
-        if branch is None:
-            return mirror
-        last_fetch_path = mirror + '.%s-lastfetch' % (name, )
-        if os.path.exists(last_fetch_path):
-            f = open(last_fetch_path)
-            last_fetch_contents = f.read()
-            f.close()
-            last_fetch_contents = last_fetch_contents.strip()
-        else:
-            last_fetch_contents = None
-        current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
-        current_vcs_version = current_vcs_version.strip()
-        if current_vcs_version != last_fetch_contents:
-            log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version))
-            tmp_checkout = buildutil.get_mirrordir(self.mirrordir, keytype, uri, prefix='_tmp-checkouts')
-            if os.path.isdir(tmp_checkout):
-                shutil.rmtree(tmp_checkout)
-            parent = os.path.dirname(tmp_checkout)
-            if not os.path.isdir(parent):
-                os.makedirs(parent)
-            run_sync(['git', 'clone', '-q', '--no-checkout', mirror, tmp_checkout])
-            run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout)
-            submodules = []
-            submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout)
-            submodule_status_lines = submodules_status_text.split('\n')
-            for line in submodule_status_lines:
-                if line == '': continue
-                line = line[1:]
-                (sub_checksum, sub_name) = line.split(' ', 1)
-                sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
-                                               'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout)
-                self._ensure_vcs_mirror(name + '-' + sub_name, keytype, sub_url, sub_checksum)
-            shutil.rmtree(tmp_checkout)
-            f = open(last_fetch_path, 'w')
-            f.write(current_vcs_version + '\n')
-            f.close()
-        return mirror
-
-    def _parse_src_key(self, srckey):
-        idx = srckey.find(':')
-        if idx < 0:
-            raise ValueError("Invalid SRC uri=%s" % (srckey, ))
-        keytype = srckey[:idx]
-        if keytype not in ['git']:
-            raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
-        uri = srckey[idx+1:]
-        return (keytype, uri)
-
     def _resolve_component_meta(self, component_meta):
         result = dict(component_meta)
         orig_src = component_meta['src']
@@ -160,29 +103,29 @@ class OstbuildResolve(builtins.Builtin):
                         break
                 if not found:
                     fatal("Unknown component %r" % (component_name, ))
-                (keytype, uri) = self._parse_src_key(component['src'])
-                mirrordir = self._ensure_vcs_mirror(component_name, keytype, uri, None)
+                (keytype, uri) = vcs.parse_src_key(component['src'])
+                mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, None)
                 log("Running git fetch for %s" % (component['name'], ))
                 run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False)
         else:
             fetch_components = []
 
         global_patches_meta = self._resolve_component_meta(self.manifest['patches'])
-        (keytype, uri) = self._parse_src_key(global_patches_meta['src'])
-        mirrordir = self._ensure_vcs_mirror(global_patches_meta['name'], keytype, uri, global_patches_meta['branch'])
+        (keytype, uri) = vcs.parse_src_key(global_patches_meta['src'])
+        mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, global_patches_meta['branch'])
         revision = buildutil.get_git_version_describe(mirrordir, global_patches_meta['branch'])
         global_patches_meta['revision'] = revision
 
         unique_component_names = set()
         for component in component_source_list:
-            (keytype, uri) = self._parse_src_key(component['src'])
+            (keytype, uri) = vcs.parse_src_key(component['src'])
             name = component['name']
 
             if name in unique_component_names:
                 fatal("Duplicate component name '%s'" % (name, ))
             unique_component_names.add(name)
 
-            mirrordir = self._ensure_vcs_mirror(name, keytype, uri, component['branch'])
+            mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, component['branch'])
             revision = buildutil.get_git_version_describe(mirrordir,
                                                           component['branch'])
             component['revision'] = revision
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 507cb1e..8437769 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -29,6 +29,7 @@ from . import builtin_checkout
 from . import builtin_chroot_compile_one
 from . import builtin_compose
 from . import builtin_compile_one
+from . import builtin_git_mirror
 from . import builtin_pull_components
 from . import builtin_prefix
 from . import builtin_resolve
diff --git a/src/ostbuild/pyostbuild/vcs.py b/src/ostbuild/pyostbuild/vcs.py
index 7811f5e..43adbbd 100755
--- a/src/ostbuild/pyostbuild/vcs.py
+++ b/src/ostbuild/pyostbuild/vcs.py
@@ -21,6 +21,8 @@ import urlparse
 import shutil
 
 from .subprocess_helpers import run_sync_get_output, run_sync
+from . import buildutil
+from .ostbuildlog import log, fatal
 
 def get_mirrordir(mirrordir, keytype, uri, prefix=''):
     assert keytype == 'git'
@@ -61,10 +63,10 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
     if not os.path.isdir(tmp_dest):
         run_sync(['git', 'clone', '-q', '--origin', 'localmirror',
                   '--no-checkout', module_mirror, tmp_dest])
+        run_sync(['git', 'remote', 'add', 'upstream', uri], cwd=tmp_dest)
     else:
-        run_sync(['git', 'fetch'], cwd=tmp_dest)
+        run_sync(['git', 'fetch', 'localmirror'], cwd=tmp_dest)
     run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest)
-    run_sync(['git', 'remote', 'add', 'upstream', uri], cwd=tmp_dest)
     run_sync(['git', 'submodule', 'init'], cwd=tmp_dest)
     have_submodules = _fixup_submodule_references(mirrordir, tmp_dest)
     if have_submodules:
@@ -76,3 +78,60 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
 def clean(keytype, checkoutdir):
     assert keytype in ('git', 'dirty-git')
     run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)
+
+def parse_src_key(srckey):
+    idx = srckey.find(':')
+    if idx < 0:
+        raise ValueError("Invalid SRC uri=%s" % (srckey, ))
+    keytype = srckey[:idx]
+    if keytype not in ['git']:
+        raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
+    uri = srckey[idx+1:]
+    return (keytype, uri)
+
+def ensure_vcs_mirror(mirrordir, keytype, uri, branch):
+    mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
+    tmp_mirror = mirror + '.tmp'
+    if os.path.isdir(tmp_mirror):
+        shutil.rmtree(tmp_mirror)
+    if not os.path.isdir(mirror):
+        run_sync(['git', 'clone', '--mirror', uri, tmp_mirror])
+        run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror)
+        os.rename(tmp_mirror, mirror)
+    if branch is None:
+        return mirror
+    last_fetch_path = mirror + '.lastfetch'
+    if os.path.exists(last_fetch_path):
+        f = open(last_fetch_path)
+        last_fetch_contents = f.read()
+        f.close()
+        last_fetch_contents = last_fetch_contents.strip()
+    else:
+        last_fetch_contents = None
+    current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
+    current_vcs_version = current_vcs_version.strip()
+    if current_vcs_version != last_fetch_contents:
+        log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version))
+        tmp_checkout = buildutil.get_mirrordir(mirrordir, keytype, uri, prefix='_tmp-checkouts')
+        if os.path.isdir(tmp_checkout):
+            shutil.rmtree(tmp_checkout)
+        parent = os.path.dirname(tmp_checkout)
+        if not os.path.isdir(parent):
+            os.makedirs(parent)
+        run_sync(['git', 'clone', '-q', '--no-checkout', mirror, tmp_checkout])
+        run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout)
+        submodules = []
+        submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout)
+        submodule_status_lines = submodules_status_text.split('\n')
+        for line in submodule_status_lines:
+            if line == '': continue
+            line = line[1:]
+            (sub_checksum, sub_name) = line.split(' ', 1)
+            sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
+                                           'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout)
+            ensure_vcs_mirror(mirrordir, keytype, sub_url, sub_checksum)
+        shutil.rmtree(tmp_checkout)
+        f = open(last_fetch_path, 'w')
+        f.write(current_vcs_version + '\n')
+        f.close()
+    return mirror