[gnome-ostree] git-mirror: Run "git fetch" for submodules too

[Colin Walters <walters src gnome org>]

commit 63b8f3714c31e96218195919a5c6be56b3de1c4c
Author: Colin Walters <walters verbum org>
Date:   Sun Jul 29 09:12:26 2012 -0400

    git-mirror: Run "git fetch" for submodules too
    
    The build kept occasionally falling over when gstreamer updated their
    common/ revision.

 src/ostbuild/pyostbuild/builtin_git_mirror.py |    8 +--
 src/ostbuild/pyostbuild/vcs.py                |   67 +++++++++++++-----------
 2 files changed, 38 insertions(+), 37 deletions(-)
---
diff --git a/src/ostbuild/pyostbuild/builtin_git_mirror.py b/src/ostbuild/pyostbuild/builtin_git_mirror.py
index 6ecdafa..5895c91 100755
--- a/src/ostbuild/pyostbuild/builtin_git_mirror.py
+++ b/src/ostbuild/pyostbuild/builtin_git_mirror.py
@@ -80,13 +80,9 @@ class OstbuildGitMirror(builtins.Builtin):
             branch = component.get('branch')
             tag = component.get('tag')
             branch_or_tag = branch or tag
-            mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag)
 
-            if not args.fetch:
-                continue
-
-            if tag is not None:
-                log("Skipping fetch for %s at tag %s" % (name, tag))
+            if (not args.fetch) or (tag is not None):
+                vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag)
                 continue
 
             curtime = time.time()
diff --git a/src/ostbuild/pyostbuild/vcs.py b/src/ostbuild/pyostbuild/vcs.py
index 85f2f85..b8b9ff4 100755
--- a/src/ostbuild/pyostbuild/vcs.py
+++ b/src/ostbuild/pyostbuild/vcs.py
@@ -94,18 +94,49 @@ def get_lastfetch_path(mirrordir, keytype, uri, branch):
     branch_safename = branch.replace('/','_').replace('.', '_')
     return mirror + '.lastfetch-%s' % (branch_safename, )
 
-def ensure_vcs_mirror(mirrordir, keytype, uri, branch):
+def _list_submodules(mirrordir, mirror, keytype, uri, branch):
+    current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
+    tmp_checkout = buildutil.get_mirrordir(mirrordir, keytype, uri, prefix='_tmp-checkouts')
+    if os.path.isdir(tmp_checkout):
+        shutil.rmtree(tmp_checkout)
+    parent = os.path.dirname(tmp_checkout)
+    if not os.path.isdir(parent):
+        os.makedirs(parent)
+    run_sync(['git', 'clone', '-q', '--no-checkout', mirror, tmp_checkout])
+    run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout)
+    submodules = []
+    submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout)
+    submodule_status_lines = submodules_status_text.split('\n')
+    for line in submodule_status_lines:
+        if line == '': continue
+        line = line[1:]
+        (sub_checksum, sub_name) = line.split(' ', 1)
+        sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
+                                       'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout)
+        submodules.append((sub_checksum, sub_name, sub_url))
+    shutil.rmtree(tmp_checkout)
+    return submodules
+
+def ensure_vcs_mirror(mirrordir, keytype, uri, branch, fetch=False):
     mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
     tmp_mirror = mirror + '.tmp'
+    last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch)
     if os.path.isdir(tmp_mirror):
         shutil.rmtree(tmp_mirror)
     if not os.path.isdir(mirror):
         run_sync(['git', 'clone', '--mirror', uri, tmp_mirror])
         run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror)
         os.rename(tmp_mirror, mirror)
+    elif fetch:
+        run_sync(['git', 'fetch'], cwd=mirror, log_initiation=False) 
+        current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
+        if current_vcs_version is not None:
+            current_vcs_version = current_vcs_version.strip()
+            f = open(last_fetch_path, 'w')
+            f.write(current_vcs_version + '\n')
+            f.close()
     if branch is None:
         return mirror
-    last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch)
     if os.path.exists(last_fetch_path):
         f = open(last_fetch_path)
         last_fetch_contents = f.read()
@@ -117,39 +148,13 @@ def ensure_vcs_mirror(mirrordir, keytype, uri, branch):
     current_vcs_version = current_vcs_version.strip()
     if current_vcs_version != last_fetch_contents:
         log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version))
-        tmp_checkout = buildutil.get_mirrordir(mirrordir, keytype, uri, prefix='_tmp-checkouts')
-        if os.path.isdir(tmp_checkout):
-            shutil.rmtree(tmp_checkout)
-        parent = os.path.dirname(tmp_checkout)
-        if not os.path.isdir(parent):
-            os.makedirs(parent)
-        run_sync(['git', 'clone', '-q', '--no-checkout', mirror, tmp_checkout])
-        run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout)
-        submodules = []
-        submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout)
-        submodule_status_lines = submodules_status_text.split('\n')
-        for line in submodule_status_lines:
-            if line == '': continue
-            line = line[1:]
-            (sub_checksum, sub_name) = line.split(' ', 1)
-            sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
-                                           'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout)
-            ensure_vcs_mirror(mirrordir, keytype, sub_url, sub_checksum)
-        shutil.rmtree(tmp_checkout)
+        for (sub_checksum, sub_name, sub_url) in _list_submodules(mirrordir, mirror, keytype, uri, branch):
+            ensure_vcs_mirror(mirrordir, keytype, sub_url, sub_checksum, fetch=fetch)
         f = open(last_fetch_path, 'w')
         f.write(current_vcs_version + '\n')
         f.close()
     return mirror
 
 def fetch(mirrordir, keytype, uri, branch, keep_going=False):
-    mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
-    last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch)
-    run_sync(['git', 'fetch'], cwd=mirror, log_initiation=False,
-             fatal_on_error=not keep_going) 
-    current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
-    if current_vcs_version is not None:
-        current_vcs_version = current_vcs_version.strip()
-        f = open(last_fetch_path, 'w')
-        f.write(current_vcs_version + '\n')
-        f.close()
+    ensure_vcs_mirror(mirrordir, keytype, uri, branch, fetch=True)