[ostree] libostree: Change synchronous fetching API to return a stream

[Colin Walters <walters src gnome org>]

commit 597da6ca6bdddd0c50aa31f1faf8749ebcae144c
Author: Colin Walters <walters verbum org>
Date:   Sat Aug 31 11:22:55 2013 -0400

    libostree: Change synchronous fetching API to return a stream
    
    There's not a good reason to write small things such as repo/config to
    the filesystem, only to read them back in again.  Change the
    non-partial API to just return a stream, then read it into a memory
    buffer.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=707157

 src/libostree/ostree-fetcher.c   |   81 ++++++++++++++++++++------------------
 src/libostree/ostree-fetcher.h   |    8 ++--
 src/libostree/ostree-repo-pull.c |   31 ++++++++++----
 3 files changed, 69 insertions(+), 51 deletions(-)
---
diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c
index 697ea8e..519345f 100644
--- a/src/libostree/ostree-fetcher.c
+++ b/src/libostree/ostree-fetcher.c
@@ -42,9 +42,9 @@ typedef struct {
 
   SoupRequest *request;
 
-  gboolean is_partial;
-  GFile *tmpfile;
+  gboolean is_stream;
   GInputStream *request_body;
+  GFile *out_tmpfile;
   GOutputStream *out_stream;
 
   guint64 content_length;
@@ -63,7 +63,7 @@ pending_uri_free (OstreeFetcherPendingURI *pending)
 
   soup_uri_free (pending->uri);
   g_clear_object (&pending->self);
-  g_clear_object (&pending->tmpfile);
+  g_clear_object (&pending->out_tmpfile);
   g_clear_object (&pending->request);
   g_clear_object (&pending->request_body);
   g_clear_object (&pending->out_stream);
@@ -179,7 +179,7 @@ on_splice_complete (GObject        *object,
   GError *local_error = NULL;
 
   pending->state = OSTREE_FETCHER_STATE_COMPLETE;
-  file_info = g_file_query_info (pending->tmpfile, OSTREE_GIO_FAST_QUERYINFO,
+  file_info = g_file_query_info (pending->out_tmpfile, OSTREE_GIO_FAST_QUERYINFO,
                                  G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
                                  pending->cancellable, &local_error);
   if (!file_info)
@@ -256,18 +256,20 @@ on_request_sent (GObject        *object,
   
   pending->content_length = soup_request_get_content_length (pending->request);
 
-  if (pending->is_partial)
-    pending->out_stream = G_OUTPUT_STREAM (g_file_append_to (pending->tmpfile, G_FILE_CREATE_NONE,
-                                                             pending->cancellable, &local_error));
+  if (!pending->is_stream)
+    {
+      pending->out_stream = G_OUTPUT_STREAM (g_file_append_to (pending->out_tmpfile, G_FILE_CREATE_NONE,
+                                                               pending->cancellable, &local_error));
+      if (!pending->out_stream)
+        goto out;
+      g_output_stream_splice_async (pending->out_stream, pending->request_body, flags, G_PRIORITY_DEFAULT,
+                                    pending->cancellable, on_splice_complete, pending);
+    }
   else
-    pending->out_stream = G_OUTPUT_STREAM (g_file_replace (pending->tmpfile, NULL, FALSE,
-                                                           G_FILE_CREATE_REPLACE_DESTINATION,
-                                                           pending->cancellable, &local_error));
-  if (!pending->out_stream)
-    goto out;
+    {
+      g_simple_async_result_complete (pending->result);
+    }
   
-  g_output_stream_splice_async (pending->out_stream, pending->request_body, flags, G_PRIORITY_DEFAULT,
-                                pending->cancellable, on_splice_complete, pending);
  out:
   if (local_error)
     {
@@ -279,7 +281,7 @@ on_request_sent (GObject        *object,
 static OstreeFetcherPendingURI *
 ostree_fetcher_request_uri_internal (OstreeFetcher         *self,
                                      SoupURI               *uri,
-                                     gboolean               is_partial,
+                                     gboolean               is_stream,
                                      GCancellable          *cancellable,
                                      GAsyncReadyCallback    callback,
                                      gpointer               user_data,
@@ -287,15 +289,18 @@ ostree_fetcher_request_uri_internal (OstreeFetcher         *self,
 {
   OstreeFetcherPendingURI *pending;
   GError *local_error = NULL;
-  gs_free char *uristring = soup_uri_to_string (uri, FALSE);
-  gs_free char *hash = g_compute_checksum_for_string (G_CHECKSUM_SHA256, uristring, strlen (uristring));
 
   pending = g_new0 (OstreeFetcherPendingURI, 1);
   pending->refcount = 1;
   pending->self = g_object_ref (self);
   pending->uri = soup_uri_copy (uri);
-  pending->is_partial = is_partial;
-  pending->tmpfile = g_file_get_child (self->tmpdir, hash);
+  pending->is_stream = is_stream;
+  if (!is_stream)
+    {
+      gs_free char *uristring = soup_uri_to_string (uri, FALSE);
+      gs_free char *hash = g_compute_checksum_for_string (G_CHECKSUM_SHA256, uristring, strlen (uristring));
+      pending->out_tmpfile = g_file_get_child (self->tmpdir, hash);
+    }
   pending->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
   pending->request = soup_requester_request_uri (self->requester, uri, &local_error);
   pending->result = g_simple_async_result_new ((GObject*) self,
@@ -326,11 +331,11 @@ ostree_fetcher_request_uri_with_partial_async (OstreeFetcher         *self,
 
   self->total_requests++;
 
-  pending = ostree_fetcher_request_uri_internal (self, uri, TRUE, cancellable,
+  pending = ostree_fetcher_request_uri_internal (self, uri, FALSE, cancellable,
                                                  callback, user_data,
                                                  ostree_fetcher_request_uri_with_partial_async);
 
-  if (!ot_gfile_query_info_allow_noent (pending->tmpfile, OSTREE_GIO_FAST_QUERYINFO,
+  if (!ot_gfile_query_info_allow_noent (pending->out_tmpfile, OSTREE_GIO_FAST_QUERYINFO,
                                         G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
                                         &file_info, cancellable, &local_error))
     goto out;
@@ -373,23 +378,23 @@ ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher         *self,
     return NULL;
   pending = g_simple_async_result_get_op_res_gpointer (simple);
 
-  return g_object_ref (pending->tmpfile);
+  return g_object_ref (pending->out_tmpfile);
 }
 
 void
-ostree_fetcher_request_uri_async (OstreeFetcher         *self,
-                                  SoupURI               *uri,
-                                  GCancellable          *cancellable,
-                                  GAsyncReadyCallback    callback,
-                                  gpointer               user_data)
+ostree_fetcher_stream_uri_async (OstreeFetcher         *self,
+                                 SoupURI               *uri,
+                                 GCancellable          *cancellable,
+                                 GAsyncReadyCallback    callback,
+                                 gpointer               user_data)
 {
   OstreeFetcherPendingURI *pending;
 
   self->total_requests++;
 
-  pending = ostree_fetcher_request_uri_internal (self, uri, FALSE, cancellable,
+  pending = ostree_fetcher_request_uri_internal (self, uri, TRUE, cancellable,
                                                  callback, user_data,
-                                                 ostree_fetcher_request_uri_async);
+                                                 ostree_fetcher_stream_uri_async);
 
   if (SOUP_IS_REQUEST_HTTP (pending->request))
     {
@@ -402,22 +407,22 @@ ostree_fetcher_request_uri_async (OstreeFetcher         *self,
                            on_request_sent, pending);
 }
 
-GFile *
-ostree_fetcher_request_uri_finish (OstreeFetcher         *self,
-                                   GAsyncResult          *result,
-                                   GError               **error)
+GInputStream *
+ostree_fetcher_stream_uri_finish (OstreeFetcher         *self,
+                                  GAsyncResult          *result,
+                                  GError               **error)
 {
   GSimpleAsyncResult *simple;
   OstreeFetcherPendingURI *pending;
 
-  g_return_val_if_fail (g_simple_async_result_is_valid (result, (GObject*)self, 
ostree_fetcher_request_uri_async), FALSE);
+  g_return_val_if_fail (g_simple_async_result_is_valid (result, (GObject*)self, 
ostree_fetcher_stream_uri_async), FALSE);
 
   simple = G_SIMPLE_ASYNC_RESULT (result);
   if (g_simple_async_result_propagate_error (simple, error))
     return NULL;
   pending = g_simple_async_result_get_op_res_gpointer (simple);
 
-  return g_object_ref (pending->tmpfile);
+  return g_object_ref (pending->request_body);
 }
 
 static char *
@@ -457,17 +462,17 @@ ostree_fetcher_query_state_text (OstreeFetcher              *self)
           active = g_hash_table_lookup (self->message_to_request, key);
           g_assert (active != NULL);
 
-          if (active->tmpfile)
+          if (active->out_tmpfile)
             {
               gs_unref_object GFileInfo *file_info = NULL;
 
-              file_info = g_file_query_info (active->tmpfile, OSTREE_GIO_FAST_QUERYINFO,
+              file_info = g_file_query_info (active->out_tmpfile, OSTREE_GIO_FAST_QUERYINFO,
                                              G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
                                              NULL, NULL);
               if (file_info)
                 {
                   gs_free char *size = format_size_pair (g_file_info_get_size (file_info),
-                                                          active->content_length);
+                                                         active->content_length);
                   g_string_append_printf (buf, " [%s]", size);
                 }
             }
diff --git a/src/libostree/ostree-fetcher.h b/src/libostree/ostree-fetcher.h
index c0b85b6..220bd93 100644
--- a/src/libostree/ostree-fetcher.h
+++ b/src/libostree/ostree-fetcher.h
@@ -70,15 +70,15 @@ GFile *ostree_fetcher_request_uri_with_partial_finish (OstreeFetcher *self,
                                                        GAsyncResult  *result,
                                                        GError       **error);
 
-void ostree_fetcher_request_uri_async (OstreeFetcher         *self,
+void ostree_fetcher_stream_uri_async (OstreeFetcher         *self,
                                        SoupURI               *uri,
                                        GCancellable          *cancellable,
                                        GAsyncReadyCallback    callback,
                                        gpointer               user_data);
 
-GFile *ostree_fetcher_request_uri_finish (OstreeFetcher         *self,
-                                          GAsyncResult          *result,
-                                          GError               **error);
+GInputStream *ostree_fetcher_stream_uri_finish (OstreeFetcher         *self,
+                                                GAsyncResult          *result,
+                                                GError               **error);
 
 G_END_DECLS
 
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index da6a5a1..65ddca2 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -328,7 +328,7 @@ run_mainloop_monitor_fetcher (OtPullData   *pull_data)
 
 typedef struct {
   OtPullData     *pull_data;
-  GFile          *result_file;
+  GInputStream   *result_stream;
 } OstreeFetchUriSyncData;
 
 static void
@@ -338,8 +338,8 @@ fetch_uri_sync_on_complete (GObject        *object,
 {
   OstreeFetchUriSyncData *data = user_data;
 
-  data->result_file = ostree_fetcher_request_uri_finish ((OstreeFetcher*)object,
-                                                         result, data->pull_data->async_error);
+  data->result_stream = ostree_fetcher_stream_uri_finish ((OstreeFetcher*)object,
+                                                          result, data->pull_data->async_error);
   data->pull_data->fetching_sync_uri = NULL;
   g_main_loop_quit (data->pull_data->loop);
 }
@@ -352,8 +352,9 @@ fetch_uri_contents_utf8_sync (OtPullData  *pull_data,
                               GError     **error)
 {
   gboolean ret = FALSE;
-  gsize len;
+  const guint8 nulchar = 0;
   gs_free char *ret_contents = NULL;
+  gs_unref_object GMemoryOutputStream *buf = NULL;
   OstreeFetchUriSyncData fetch_data = { 0, };
 
   if (g_cancellable_set_error_if_cancelled (cancellable, error))
@@ -362,16 +363,28 @@ fetch_uri_contents_utf8_sync (OtPullData  *pull_data,
   fetch_data.pull_data = pull_data;
 
   pull_data->fetching_sync_uri = uri;
-  ostree_fetcher_request_uri_async (pull_data->fetcher, uri, cancellable,
-                                    fetch_uri_sync_on_complete, &fetch_data);
+  ostree_fetcher_stream_uri_async (pull_data->fetcher, uri, cancellable,
+                                   fetch_uri_sync_on_complete, &fetch_data);
 
   run_mainloop_monitor_fetcher (pull_data);
-  if (!fetch_data.result_file)
+  if (!fetch_data.result_stream)
     goto out;
 
-  if (!g_file_load_contents (fetch_data.result_file, cancellable, &ret_contents, &len, NULL, error))
+  buf = (GMemoryOutputStream*)g_memory_output_stream_new_resizable ();
+  if (g_output_stream_splice ((GOutputStream*)buf, fetch_data.result_stream,
+                              G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
+                              cancellable, error) < 0)
     goto out;
 
+  /* Add trailing NUL */
+  if (!g_output_stream_write ((GOutputStream*)buf, &nulchar, 1, cancellable, error))
+    goto out;
+
+  if (!g_output_stream_close ((GOutputStream*)buf, cancellable, error))
+    goto out;
+
+  ret_contents = g_memory_output_stream_steal_data (buf);
+
   if (!g_utf8_validate (ret_contents, -1, NULL))
     {
       g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
@@ -382,7 +395,7 @@ fetch_uri_contents_utf8_sync (OtPullData  *pull_data,
   ret = TRUE;
   ot_transfer_out_value (out_contents, &ret_contents);
  out:
-  g_clear_object (&(fetch_data.result_file));
+  g_clear_object (&(fetch_data.result_stream));
   return ret;
 }