[ostree/wip/soup-hard-dep] Rebase pull-local reusing 'pull'

[Colin Walters <walters src gnome org>]

commit 7e31d0f6fc05cab5335f82086fcd658e843a9cd0
Author: Colin Walters <walters verbum org>
Date:   Thu Nov 1 13:24:49 2012 -0400

    Rebase pull-local reusing 'pull'
    
    This gets us async scanning and commit, progress bar, etc.

 src/ostree/ostree-fetcher.c        |   12 ++-
 src/ostree/ostree-pull.c           |  139 +++++++++++++++++++------
 src/ostree/ot-builtin-pull-local.c |  198 ++++--------------------------------
 3 files changed, 135 insertions(+), 214 deletions(-)
---
diff --git a/src/ostree/ostree-fetcher.c b/src/ostree/ostree-fetcher.c
index 6d4b5db..96c2dae 100644
--- a/src/ostree/ostree-fetcher.c
+++ b/src/ostree/ostree-fetcher.c
@@ -193,7 +193,8 @@ on_request_sent (GObject        *object,
 
   pending->request_body = soup_request_send_finish ((SoupRequest*) object,
                                                    result, &local_error);
-  msg = soup_request_http_get_message ((SoupRequestHTTP*) object);
+  if (SOUP_IS_REQUEST_HTTP (object))
+    msg = soup_request_http_get_message ((SoupRequestHTTP*) object);
 
   if (!pending->request_body)
     {
@@ -201,7 +202,7 @@ on_request_sent (GObject        *object,
       g_simple_async_result_take_error (pending->result, local_error);
       g_simple_async_result_complete (pending->result);
     }
-  else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
+  else if (msg && !SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
     {
       g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED,
                    "Server returned status %u: %s",
@@ -255,9 +256,10 @@ ostree_fetcher_request_uri_async (OstreeFetcher         *self,
   g_assert_no_error (local_error);
 
   pending->refcount++;
-  g_hash_table_insert (self->message_to_request,
-                       soup_request_http_get_message ((SoupRequestHTTP*)pending->request),
-                       pending);
+  if (SOUP_IS_REQUEST_HTTP (pending->request))
+    g_hash_table_insert (self->message_to_request,
+                         soup_request_http_get_message ((SoupRequestHTTP*)pending->request),
+                         pending);
 
   pending->result = g_simple_async_result_new ((GObject*) self,
                                                callback, user_data,
diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c
index 4c0e2dd..0f01d85 100644
--- a/src/ostree/ostree-pull.c
+++ b/src/ostree/ostree-pull.c
@@ -86,6 +86,8 @@ static GOptionEntry options[] = {
 typedef struct {
   OstreeRepo   *repo;
   char         *remote_name;
+  OstreeRepo   *remote_repo_on_local_fs;
+  gboolean      remote_on_local_fs;
   OstreeRepoMode remote_mode;
   OstreeFetcher *fetcher;
   SoupURI      *base_uri;
@@ -560,18 +562,51 @@ process_one_file_request (OtFetchOneContentItemData *data)
   gboolean compressed = pull_data->remote_mode == OSTREE_REPO_MODE_ARCHIVE_Z2;
   ot_lfree char *objpath = NULL;
   SoupURI *obj_uri = NULL;
+  GError *local_error = NULL;
+  GError **error = &local_error;
+
+  if (pull_data->remote_repo_on_local_fs)
+    {
+      guint64 length;
+      ot_lobj GInputStream *object_input = NULL;
+      ot_lobj GInputStream *input = NULL;
+      ot_lobj GFileInfo *file_info = NULL;
+      ot_lvariant GVariant *xattrs = NULL;
 
-  objpath = ostree_get_relative_object_path (checksum, OSTREE_OBJECT_TYPE_FILE, compressed);
-  obj_uri = suburi_new (pull_data->base_uri, objpath, NULL);
+      if (!ostree_repo_load_file (pull_data->remote_repo_on_local_fs,
+                                  checksum, &input, &file_info, &xattrs,
+                                  pull_data->cancellable, error))
+        goto out;
+
+      if (!ostree_raw_file_to_content_stream (input, file_info, xattrs,
+                                              &object_input, &length,
+                                              pull_data->cancellable, error))
+        goto out;
+
+      data->pull_data->outstanding_content_stage_requests++;
+      ostree_repo_stage_content_async (data->pull_data->repo, checksum,
+                                       object_input, length,
+                                       pull_data->cancellable,
+                                       content_fetch_on_stage_complete, data);
+    }
+  else
+    {
+      objpath = ostree_get_relative_object_path (checksum, OSTREE_OBJECT_TYPE_FILE, compressed);
+      obj_uri = suburi_new (pull_data->base_uri, objpath, NULL);
       
-  ostree_fetcher_request_uri_async (pull_data->fetcher, obj_uri, pull_data->cancellable,
-                                    content_fetch_on_complete, data);
-  soup_uri_free (obj_uri);
+      ostree_fetcher_request_uri_async (pull_data->fetcher, obj_uri, pull_data->cancellable,
+                                        content_fetch_on_complete, data);
+      soup_uri_free (obj_uri);
   
-  if (compressed)
-    pull_data->outstanding_filecontent_requests++;
-  else
-    pull_data->outstanding_filemeta_requests++;
+      if (compressed)
+        pull_data->outstanding_filecontent_requests++;
+      else
+        pull_data->outstanding_filemeta_requests++;
+
+    }
+
+ out:
+  check_outstanding_requests_handle_error (data->pull_data, local_error);
 }
 
 static void
@@ -719,6 +754,9 @@ idle_queue_content_request (gpointer user_data)
     {
       g_queue_push_tail (&pull_data->queued_filemeta, data);
     }
+  else if (pull_data->remote_on_local_fs)
+    {
+    }
   else
     {
       process_one_file_request (data);
@@ -762,9 +800,9 @@ meta_fetch_on_complete (GObject           *object,
 {
   IdleFetchMetadataObjectData *fetch_data = user_data;
   OtPullData *pull_data = fetch_data->pull_data;
-  ot_lvariant GVariant *metadata = NULL;
   const char *checksum;
   OstreeObjectType objtype;
+  ot_lvariant GVariant *metadata = NULL;
   GError *local_error = NULL;
   GError **error = &local_error;
 
@@ -800,18 +838,36 @@ idle_fetch_metadata_object (gpointer data)
   const char *checksum;
   OstreeObjectType objtype;
   SoupURI *obj_uri = NULL;
-  gboolean compressed;
-
-  compressed = pull_data->remote_mode == OSTREE_REPO_MODE_ARCHIVE_Z2;
+  GError *local_error = NULL;
+  GError **error = &local_error;
 
   ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
 
-  objpath = ostree_get_relative_object_path (checksum, objtype, compressed);
-  obj_uri = suburi_new (pull_data->base_uri, objpath, NULL);
+  if (pull_data->remote_on_local_fs)
+    {
+      ot_lvariant GVariant *metadata = NULL;
+      if (!ostree_repo_load_variant (pull_data->remote_repo_on_local_fs, objtype,
+                                     checksum, &metadata, error))
+        goto out;
+      ostree_repo_stage_metadata_async (pull_data->repo, objtype, checksum, metadata,
+                                        pull_data->cancellable,
+                                        on_metadata_staged, fetch_data);
+    }
+  else
+    {
+      gboolean compressed;
+
+      compressed = pull_data->remote_mode == OSTREE_REPO_MODE_ARCHIVE_Z2;
+      objpath = ostree_get_relative_object_path (checksum, objtype, compressed);
 
-  ostree_fetcher_request_uri_async (pull_data->fetcher, obj_uri, pull_data->cancellable,
-                                    meta_fetch_on_complete, fetch_data);
-  soup_uri_free (obj_uri);
+      obj_uri = suburi_new (pull_data->base_uri, objpath, NULL);
+      ostree_fetcher_request_uri_async (pull_data->fetcher, obj_uri, pull_data->cancellable,
+                                        meta_fetch_on_complete, fetch_data);
+      soup_uri_free (obj_uri);
+    }
+
+ out:
+  check_outstanding_requests_handle_error (pull_data, local_error);
 
   return FALSE;
 }
@@ -1229,10 +1285,23 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
   pull_data->fetcher = ostree_fetcher_new (ostree_repo_get_tmpdir (pull_data->repo));
   config = ostree_repo_get_config (repo);
 
-  remote_key = g_strdup_printf ("remote \"%s\"", pull_data->remote_name);
-  if (!repo_get_string_key_inherit (repo, remote_key, "url", &baseurl, error))
-    goto out;
-  pull_data->base_uri = soup_uri_new (baseurl);
+  if (g_str_has_prefix (pull_data->remote_name, "file://"))
+    {
+      ot_lobj GFile *raw_path = g_file_new_for_path (pull_data->remote_name + strlen ("file://"));
+      pull_data->remote_repo_on_local_fs = ostree_repo_new (raw_path);
+      if (!ostree_repo_check (pull_data->remote_repo_on_local_fs, error))
+        goto out;
+      pull_data->base_uri = soup_uri_new (pull_data->remote_name);
+      pull_data->remote_on_local_fs = TRUE;
+    }
+  else
+    {
+      remote_key = g_strdup_printf ("remote \"%s\"", pull_data->remote_name);
+      if (!repo_get_string_key_inherit (repo, remote_key, "url", &baseurl, error))
+        goto out;
+      pull_data->base_uri = soup_uri_new (baseurl);
+      pull_data->remote_on_local_fs = FALSE;
+    }
 
   if (!pull_data->base_uri)
     {
@@ -1251,15 +1320,18 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
   if (!ostree_repo_mode_from_string (remote_mode_str, &pull_data->remote_mode, error))
     goto out;
 
-  switch (pull_data->remote_mode)
+  if (!pull_data->remote_on_local_fs)
     {
-    case OSTREE_REPO_MODE_ARCHIVE:
-    case OSTREE_REPO_MODE_ARCHIVE_Z2:
-      break;
-    default:
-      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                   "Can't pull from archives with mode \"%s\"",
-                   remote_mode_str);
+      switch (pull_data->remote_mode)
+        {
+        case OSTREE_REPO_MODE_ARCHIVE:
+        case OSTREE_REPO_MODE_ARCHIVE_Z2:
+          break;
+        default:
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Can't pull from archives with mode \"%s\"",
+                       remote_mode_str);
+        }
     }
 
   requested_refs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
@@ -1293,7 +1365,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
       GError *temp_error = NULL;
       gboolean fetch_all_refs;
 
-      configured_branches = g_key_file_get_string_list (config, remote_key, "branches", NULL, &temp_error);
+      if (remote_key)
+        configured_branches = g_key_file_get_string_list (config, remote_key, "branches", NULL, &temp_error);
       if (configured_branches == NULL && temp_error != NULL)
         {
           if (g_error_matches (temp_error, G_KEY_FILE_ERROR, G_KEY_FILE_ERROR_KEY_NOT_FOUND))
@@ -1443,7 +1516,9 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
   g_free (pull_data->remote_name);
   if (pull_data->base_uri)
     soup_uri_free (pull_data->base_uri);
-  g_clear_pointer (&pull_data->metadata_objects_to_scan, (GDestroyNotify) ot_worker_queue_unref);
+  if (pull_data->metadata_objects_to_scan)
+    ot_worker_queue_unref (pull_data->metadata_objects_to_scan);
+  g_clear_object (&pull_data->remote_repo_on_local_fs);
   g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref);
diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c
index 94f0b88..6a7de53 100644
--- a/src/ostree/ot-builtin-pull-local.c
+++ b/src/ostree/ot-builtin-pull-local.c
@@ -32,207 +32,51 @@ static GOptionEntry options[] = {
   { NULL }
 };
 
-typedef struct {
-  OstreeRepo *src_repo;
-  OstreeRepo *dest_repo;
-} OtLocalCloneData;
-
-static gboolean
-import_one_object (OtLocalCloneData *data,
-                   const char   *checksum,
-                   OstreeObjectType objtype,
-                   GCancellable  *cancellable,
-                   GError        **error)
-{
-  gboolean ret = FALSE;
-  ot_lobj GFile *content_path = NULL;
-  ot_lobj GFileInfo *archive_info = NULL;
-
-  if (objtype == OSTREE_OBJECT_TYPE_FILE)
-    {
-      guint64 length;
-      ot_lobj GInputStream *file_object = NULL;
-      ot_lobj GInputStream *input = NULL;
-      ot_lobj GFileInfo *file_info = NULL;
-      ot_lvariant GVariant *xattrs = NULL;
-
-      if (!ostree_repo_load_file (data->src_repo, checksum,
-                                  &input, &file_info, &xattrs,
-                                  cancellable, error))
-        goto out;
-
-      if (!ostree_raw_file_to_content_stream (input, file_info, xattrs,
-                                              &file_object, &length,
-                                              cancellable, error))
-        goto out;
-
-      if (!ostree_repo_stage_content_trusted (data->dest_repo, checksum,
-                                              file_object, length,
-                                              cancellable, error))
-        goto out;
-    }
-  else
-    {
-      ot_lvariant GVariant *metadata = NULL;
-
-      if (!ostree_repo_load_variant (data->src_repo, objtype, checksum, &metadata,
-                                     error))
-        goto out;
-
-      if (!ostree_repo_stage_metadata_trusted (data->dest_repo, objtype, checksum, metadata,
-                                               cancellable, error))
-        goto out;
-    }
-
-  ret = TRUE;
- out:
-  return ret;
-}
-
 gboolean
 ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **error)
 {
-  gboolean ret = FALSE;
-  GCancellable *cancellable = NULL;
   GOptionContext *context;
-  const char *src_repo_path;
+  gboolean ret = FALSE;
+  GPtrArray *args = NULL;
   int i;
-  GHashTableIter hash_iter;
-  gpointer key, value;
-  ot_lhash GHashTable *objects = NULL;
-  ot_lobj GFile *src_f = NULL;
-  ot_lobj GFile *src_repo_dir = NULL;
-  ot_lobj GFile *dest_repo_dir = NULL;
-  ot_lobj GFile *src_dir = NULL;
-  ot_lobj GFile *dest_dir = NULL;
-  ot_lhash GHashTable *refs_to_clone = NULL;
-  ot_lhash GHashTable *source_objects = NULL;
-  ot_lhash GHashTable *objects_to_copy = NULL;
-  OtLocalCloneData data;
+  gs_lfree char *file_uri = NULL;
+  gs_lfree char *repo_arg = NULL;
+  gs_lfree char *abspath = NULL;
 
   context = g_option_context_new ("SRC_REPO [REFS...] -  Copy data from SRC_REPO");
   g_option_context_add_main_entries (context, options, NULL);
 
-  memset (&data, 0, sizeof (data));
-
   if (!g_option_context_parse (context, &argc, &argv, error))
     goto out;
 
-  data.dest_repo = ostree_repo_new (repo_path);
-  if (!ostree_repo_check (data.dest_repo, error))
-    goto out;
-
-  if (argc < 2)
-    {
-      gchar *help = g_option_context_get_help (context, TRUE, NULL);
-      g_printerr ("%s\n", help);
-      g_free (help);
-      g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                               "DESTINATION must be specified");
-      goto out;
-    }
-
-  src_repo_path = argv[1];
-  src_f = g_file_new_for_path (src_repo_path);
-
-  data.src_repo = ostree_repo_new (src_f);
-  if (!ostree_repo_check (data.src_repo, error))
-    goto out;
-
-  src_repo_dir = g_object_ref (ostree_repo_get_path (data.src_repo));
-  dest_repo_dir = g_object_ref (ostree_repo_get_path (data.dest_repo));
-
-  if (argc == 2)
+  if (!g_path_is_absolute (argv[1]))
     {
-      if (!ostree_repo_list_all_refs (data.src_repo, &refs_to_clone, cancellable, error))
-        goto out;
+      gs_lfree char *cwd = g_get_current_dir ();;
+      abspath = g_build_filename (cwd, argv[1], NULL);
     }
   else
-    {
-      refs_to_clone = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
-      for (i = 2; i < argc; i++)
-        {
-          const char *ref = argv[i];
-          char *rev;
-          
-          if (!ostree_repo_resolve_rev (data.src_repo, ref, FALSE, &rev, error))
-            goto out;
-          
-          /* Transfer ownership of rev */
-          g_hash_table_insert (refs_to_clone, g_strdup (ref), rev);
-        }
-    }
+    abspath = g_strdup (argv[1]);
 
-  g_print ("Enumerating objects...\n");
-
-  source_objects = ostree_traverse_new_reachable ();
-
-  g_hash_table_iter_init (&hash_iter, refs_to_clone);
-  while (g_hash_table_iter_next (&hash_iter, &key, &value))
-    {
-      const char *checksum = value;
-
-      if (!ostree_traverse_commit (data.src_repo, checksum, 0, source_objects, cancellable, error))
-        goto out;
-    }
-
-  objects_to_copy = ostree_traverse_new_reachable ();
-  g_hash_table_iter_init (&hash_iter, source_objects);
-  while (g_hash_table_iter_next (&hash_iter, &key, &value))
-    {
-      GVariant *serialized_key = key;
-      gboolean has_object;
-      const char *checksum;
-      OstreeObjectType objtype;
+  file_uri = g_strconcat ("file://", abspath, NULL);
+  repo_arg = g_strconcat ("--repo=", ot_gfile_get_path_cached (repo_path), NULL);
 
-      ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
-
-      if (!ostree_repo_has_object (data.dest_repo, objtype, checksum, &has_object,
-                                   cancellable, error))
-        goto out;
-      if (!has_object)
-        g_hash_table_insert (objects_to_copy, g_variant_ref (serialized_key), serialized_key);
-    }
-
-  g_print ("%u objects to copy\n", g_hash_table_size (objects_to_copy));
-
-  if (!ostree_repo_prepare_transaction (data.dest_repo, FALSE, cancellable, error))
-    goto out;
+  args = g_ptr_array_new ();
+  ot_ptrarray_add_many (args, "ostree", "pull", repo_arg, file_uri, NULL);
   
-  g_hash_table_iter_init (&hash_iter, objects_to_copy);
-  while (g_hash_table_iter_next (&hash_iter, &key, &value))
-    {
-      GVariant *serialized_key = key;
-      const char *checksum;
-      OstreeObjectType objtype;
+  for (i = 2; i < argc; i++)
+    g_ptr_array_add (args, argv[i]);
 
-      ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+  g_ptr_array_add (args, NULL);
 
-      if (!import_one_object (&data, checksum, objtype, cancellable, error))
-        goto out;
-    }
+  execvp ("ostree", (char**)args->pdata);
+  /* Shouldn't return */
 
-  if (!ostree_repo_commit_transaction (data.dest_repo, NULL, error))
-    goto out;
-
-  g_print ("Writing %u refs\n", g_hash_table_size (refs_to_clone));
-
-  g_hash_table_iter_init (&hash_iter, refs_to_clone);
-  while (g_hash_table_iter_next (&hash_iter, &key, &value))
-    {
-      const char *name = key;
-      const char *checksum = value;
-
-      if (!ostree_repo_write_ref (data.dest_repo, NULL, name, checksum, error))
-        goto out;
-    }
+  ot_util_set_error_from_errno (error, errno);
+  goto out;
 
   ret = TRUE;
  out:
-  if (data.src_repo)
-    g_object_unref (data.src_repo);
-  if (data.dest_repo)
-    g_object_unref (data.dest_repo);
+  g_clear_pointer (&args, (GDestroyNotify)g_ptr_array_unref);
   if (context)
     g_option_context_free (context);
   return ret;