[ostree] pull: Optimize file:/// URIs to skip libsoup and hardlink if possible

[Colin Walters <walters src gnome org>]

commit ab3bf493dd9fa5cd6abba299d9eea182d9d17d96
Author: Colin Walters <walters verbum org>
Date:   Fri Feb 6 10:02:57 2015 -0500

    pull: Optimize file:/// URIs to skip libsoup and hardlink if possible
    
    It's always been suboptimal to have both pull and pull-local; as we go
    beyond the raw object data into things like deltas and summary files,
    the logic to perform e.g. mirroring should only be in one place.
    
    This will be used by Pulp's OSTree content plugin at least to perform
    promotions.

 src/libostree/ostree-repo-pull.c |   62 +++++++++++++++++++++++++++----------
 tests/pull-test.sh               |    9 +++++
 2 files changed, 54 insertions(+), 17 deletions(-)
---
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index d3e228c..f79c46c 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -42,6 +42,7 @@ typedef struct {
   OstreeRepoMode remote_mode;
   OstreeFetcher *fetcher;
   SoupURI      *base_uri;
+  OstreeRepo   *remote_repo_local;
 
   GMainContext    *main_context;
   GMainLoop    *loop;
@@ -400,8 +401,15 @@ scan_dirtree_object (OtPullData   *pull_data,
       if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_FILE, file_checksum,
                                    &file_is_stored, cancellable, error))
         goto out;
-      
-      if (!file_is_stored && !g_hash_table_lookup (pull_data->requested_content, file_checksum))
+
+      if (!file_is_stored && pull_data->remote_repo_local)
+        {
+          if (!ostree_repo_import_object_from (pull_data->repo, pull_data->remote_repo_local,
+                                               OSTREE_OBJECT_TYPE_FILE, file_checksum,
+                                               cancellable, error))
+            goto out;
+        }
+      else if (!file_is_stored && !g_hash_table_lookup (pull_data->requested_content, file_checksum))
         {
           g_hash_table_insert (pull_data->requested_content, file_checksum, file_checksum);
           enqueue_one_object_request (pull_data, file_checksum, OSTREE_OBJECT_TYPE_FILE, FALSE);
@@ -409,7 +417,6 @@ scan_dirtree_object (OtPullData   *pull_data,
         }
     }
 
-
     if (pull_data->dir)
       {
         const char *subpath = NULL;  
@@ -1072,6 +1079,16 @@ scan_one_metadata_object_c (OtPullData         *pull_data,
                                cancellable, error))
     goto out;
 
+  if (pull_data->remote_repo_local)
+    {
+      if (!ostree_repo_import_object_from (pull_data->repo, pull_data->remote_repo_local,
+                                           objtype, tmp_checksum,
+                                           cancellable, error))
+        goto out;
+      is_stored = TRUE;
+      is_requested = TRUE;
+    }
+
   if (!is_stored && !is_requested)
     {
       char *duped_checksum = g_strdup (tmp_checksum);
@@ -1828,22 +1845,32 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
                                             &configured_branches, error))
     goto out;
 
-  if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
-    goto out;
-
-  if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare",
-                                          &remote_mode_str, error))
-    goto out;
+  if (strcmp (soup_uri_get_scheme (pull_data->base_uri), "file") == 0)
+    {
+      gs_unref_object GFile *remote_repo_path = g_file_new_for_path (soup_uri_get_path 
(pull_data->base_uri));
+      pull_data->remote_repo_local = ostree_repo_new (remote_repo_path);
+      if (!ostree_repo_open (pull_data->remote_repo_local, cancellable, error))
+        goto out;
+    }
+  else
+    {
+      if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
+        goto out;
 
-  if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error))
-    goto out;
+      if (!ot_keyfile_get_value_with_default (remote_config, "core", "mode", "bare",
+                                              &remote_mode_str, error))
+        goto out;
 
-  if (pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE_Z2)
-    {
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                   "Can't pull from archives with mode \"%s\"",
-                   remote_mode_str);
-      goto out;
+      if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error))
+        goto out;
+    
+      if (pull_data->remote_mode != OSTREE_REPO_MODE_ARCHIVE_Z2)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Can't pull from archives with mode \"%s\"",
+                       remote_mode_str);
+          goto out;
+        }
     }
 
   pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
@@ -2135,6 +2162,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
     g_main_loop_unref (pull_data->loop);
   g_strfreev (configured_branches);
   g_clear_object (&pull_data->fetcher);
+  g_clear_object (&pull_data->remote_repo_local);
   g_free (pull_data->remote_name);
   if (pull_data->base_uri)
     soup_uri_free (pull_data->base_uri);
diff --git a/tests/pull-test.sh b/tests/pull-test.sh
index 5aee090..62dccf5 100755
--- a/tests/pull-test.sh
+++ b/tests/pull-test.sh
@@ -56,6 +56,15 @@ $OSTREE show main >/dev/null
 echo "ok pull mirror"
 
 cd ${test_tmpdir}
+mkdir mirrorrepo-local
+ostree --repo=mirrorrepo-local init --mode=archive-z2
+${CMD_PREFIX} ostree --repo=mirrorrepo-local remote add --set=gpg-verify=false origin 
file://$(pwd)/ostree-srv/gnomerepo
+${CMD_PREFIX} ostree --repo=mirrorrepo-local pull --mirror origin main
+${CMD_PREFIX} ostree --repo=mirrorrepo-local fsck
+$OSTREE show main >/dev/null
+echo "ok pull local mirror"
+
+cd ${test_tmpdir}
 ostree --repo=ostree-srv/gnomerepo commit -b main -s "Metadata string" 
--add-detached-metadata-string=SIGNATURE=HANCOCK --tree=ref=main
 ${CMD_PREFIX} ostree --repo=repo pull origin main
 ${CMD_PREFIX} ostree --repo=repo fsck