[ostree] ostbuild: Ensure we don't retrieve submodules from the network

[Colin Walters <walters src gnome org>]

commit 55f6998d37fad490d314d2b8e70a2b80e1d725d8
Author: Colin Walters <walters verbum org>
Date:   Wed Jan 18 13:35:53 2012 -0500

    ostbuild: Ensure we don't retrieve submodules from the network
    
    This is a bit gross...we have to manually frob the submodule
    URLs to point to our mirror.

 src/ostbuild/pyostbuild/builtin_build.py   |   35 +++++++++++--
 src/ostbuild/pyostbuild/builtin_resolve.py |   77 +++++++++++++++++++++++++---
 2 files changed, 100 insertions(+), 12 deletions(-)
---
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index 670b4e8..900a170 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -18,6 +18,7 @@
 import os,sys,subprocess,tempfile,re,shutil
 import argparse
 import time
+import urlparse
 import json
 from StringIO import StringIO
 
@@ -40,6 +41,22 @@ class OstbuildBuild(builtins.Builtin):
     def __init__(self):
         builtins.Builtin.__init__(self)
 
+    def _mirror_for_url(self, url):
+        parsed = urlparse.urlsplit(url)
+        return os.path.join(self.mirrordir, 'git', parsed.scheme, parsed.netloc, parsed.path[1:])
+
+    def _fixup_submodule_references(self, cwd):
+        submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=cwd)
+        submodule_status_lines = submodules_status_text.split('\n')
+        for line in submodule_status_lines:
+            if line == '': continue
+            line = line[1:]
+            (sub_checksum, sub_name) = line.split(' ', 1)
+            sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
+                                           'submodule.%s.url' % (sub_name, )], cwd=cwd)
+            mirrordir = self._mirror_for_url(sub_url)
+            run_sync(['git', 'config', 'submodule.%s.url' % (sub_name, ), 'file://' + mirrordir], cwd=cwd)
+
     def _get_vcs_checkout(self, name, keytype, mirrordir, branch):
         checkoutdir = os.path.join(self.workdir, 'src')
         if not os.path.isdir(checkoutdir):
@@ -50,9 +67,18 @@ class OstbuildBuild(builtins.Builtin):
             shutil.rmtree(dest)
         if os.path.isdir(tmp_dest):
             shutil.rmtree(tmp_dest)
-        subprocess.check_call(['git', 'clone', '-q', '--recursive', mirrordir, tmp_dest])
-        subprocess.check_call(['git', 'checkout', '-q', branch], cwd=tmp_dest)
-        subprocess.check_call(['git', 'submodule', 'update', '--init'], cwd=tmp_dest)
+        git_mirrors_path = os.path.join(self.mirrordir, 'gitconfig')
+        f = open(git_mirrors_path)
+        git_mirrors = f.read()
+        f.close()
+        run_sync(['git', 'clone', '-q',
+                  '--no-checkout', mirrordir, tmp_dest])
+        run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest)
+        run_sync(['git', 'submodule', 'init'], cwd=tmp_dest)
+        self._fixup_submodule_references(tmp_dest)
+        run_sync(['linux-user-chroot',
+                  '--unshare-net', '--chdir', tmp_dest, '/',
+                  '/usr/bin/git', 'submodule', 'update'])
         os.rename(tmp_dest, dest)
         return dest
 
@@ -178,7 +204,8 @@ class OstbuildBuild(builtins.Builtin):
         else:
             log("No previous build for '%s' found" % (buildname, ))
 
-        mirror = os.path.join(self.mirrordir, name)
+            
+        mirror = self._mirror_for_url(uri)
         component_src = self._get_vcs_checkout(name, keytype, mirror, branch)
 
         if meta.get('rm-configure', False):
diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py
index 516d144..c1f20c0 100755
--- a/src/ostbuild/pyostbuild/builtin_resolve.py
+++ b/src/ostbuild/pyostbuild/builtin_resolve.py
@@ -18,6 +18,7 @@
 import os,sys,subprocess,tempfile,re,shutil
 import argparse
 import json
+import urlparse
 from StringIO import StringIO
 
 from . import builtins
@@ -35,15 +36,51 @@ class OstbuildResolve(builtins.Builtin):
     def __init__(self):
         builtins.Builtin.__init__(self)
 
-    def _ensure_vcs_mirror(self, name, keytype, uri, branch):
+    def _ensure_vcs_mirror(self, keytype, uri, branch):
         assert keytype == 'git'
-        mirror = os.path.join(self.mirrordir, name)
+        parsed = urlparse.urlsplit(uri)
+        mirror = os.path.join(self.mirrordir, keytype, parsed.scheme, parsed.netloc, parsed.path[1:])
         tmp_mirror = mirror + '.tmp'
         if os.path.isdir(tmp_mirror):
             shutil.rmtree(tmp_mirror)
         if not os.path.isdir(mirror):
             run_sync(['git', 'clone', '--mirror', uri, tmp_mirror])
+            run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror)
             os.rename(tmp_mirror, mirror)
+        last_fetch_path = mirror + '.lastfetch'
+        if os.path.exists(last_fetch_path):
+            f = open(last_fetch_path)
+            last_fetch_contents = f.read()
+            f.close()
+            last_fetch_contents = last_fetch_contents.strip()
+        else:
+            last_fetch_contents = None
+        current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
+        current_vcs_version = current_vcs_version.strip()
+        if current_vcs_version != last_fetch_contents:
+            log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version))
+            tmp_checkout = os.path.join(self.mirrordir, '_tmp-checkouts', keytype, parsed.netloc, parsed.path[1:])
+            if os.path.isdir(tmp_checkout):
+                shutil.rmtree(tmp_checkout)
+            parent = os.path.dirname(tmp_checkout)
+            if not os.path.isdir(parent):
+                os.makedirs(parent)
+            run_sync(['git', 'clone', '-b', branch, mirror, tmp_checkout])
+            run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout)
+            submodules = []
+            submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout)
+            submodule_status_lines = submodules_status_text.split('\n')
+            for line in submodule_status_lines:
+                if line == '': continue
+                line = line[1:]
+                (sub_checksum, sub_name) = line.split(' ', 1)
+                sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
+                                               'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout)
+                self._ensure_vcs_mirror(keytype, sub_url, sub_checksum)
+            shutil.rmtree(tmp_checkout)
+            f = open(last_fetch_path, 'w')
+            f.write(current_vcs_version + '\n')
+            f.close()
         return mirror
 
     def _parse_src_key(self, srckey):
@@ -117,21 +154,45 @@ class OstbuildResolve(builtins.Builtin):
                 if not found:
                     fatal("Unknown component %r" % (component_name, ))
                 (keytype, uri) = self._parse_src_key(component['src'])
-                mirrordir = self._ensure_vcs_mirror(component['name'],
-                                                    keytype, uri,
-                                                    component['branch'])
+                mirrordir = self._ensure_vcs_mirror(keytype, uri, component['branch'])
                 log("Running git fetch for %s" % (component['name'], ))
                 run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False)
+        else:
+            fetch_components = []
 
         for component in self.resolved_components:
             (keytype, uri) = self._parse_src_key(component['src'])
-            mirrordir = self._ensure_vcs_mirror(component['name'],
-                                                keytype, uri,
-                                                component['branch'])
+            try:
+                fetch_components.index(component['name'])
+                continue
+            except ValueError, e:
+                pass
+            mirrordir = self._ensure_vcs_mirror(keytype, uri, component['branch'])
             revision = buildutil.get_git_version_describe(mirrordir,
                                                           component['branch'])
             component['revision'] = revision
 
+        mirror_gitconfig_path = os.path.join(self.mirrordir, 'gitconfig')
+        git_mirrordir = os.path.join(self.mirrordir, 'git')
+        f = open(mirror_gitconfig_path, 'w')
+        find_proc = subprocess.Popen(['find', '-type', 'f', '-name', 'HEAD'],
+                                     cwd=git_mirrordir, stdout=subprocess.PIPE)
+        path_to_url_re = re.compile(r'^([^/]+)/([^/]+)/(.+)$')
+        for line in find_proc.stdout:
+            assert line.startswith('./')
+            path = line[2:-6]
+            f.write('[url "')
+            f.write('file://' + os.path.join(git_mirrordir, path) + '/')
+            f.write('"]\n')
+            f.write('   insteadOf = ')
+            match = path_to_url_re.match(path)
+            assert match is not None
+            url = urlparse.urlunparse([match.group(1), match.group(2), match.group(3),
+                                       None, None, None])
+            f.write(url)
+            f.write('/\n')
+        print "Generated git mirror config: %s" % (mirror_gitconfig_path, )
+
         self.manifest['components'] = self.resolved_components
 
         out_manifest = os.path.join(self.workdir, 'manifest.json')