[ostree] pull: cache summary and summary.sig

[Giuseppe Scrivano <gscrivano src gnome org>]

commit 58b48424bcf854e6c4cbb4b529ae2266f17b5883
Author: Giuseppe Scrivano <gscrivan redhat com>
Date:   Fri Mar 11 12:39:32 2016 +0100

    pull: cache summary and summary.sig
    
    It allows an optimization to skip the download of the summary file
    if its .sig file is unchanged.
    
    Downloading the .sig file is much cheaper than downloading the summary
    file from repositories with many branches.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=762973
    
    Signed-off-by: Giuseppe Scrivano <gscrivan redhat com>

 src/libostree/ostree-repo-private.h |   22 +++++-
 src/libostree/ostree-repo-pull.c    |  145 ++++++++++++++++++++++++++++++++---
 src/libostree/ostree-repo.c         |   11 +++
 3 files changed, 165 insertions(+), 13 deletions(-)
---
diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h
index 463b3dd..484a6ec 100644
--- a/src/libostree/ostree-repo-private.h
+++ b/src/libostree/ostree-repo-private.h
@@ -33,6 +33,8 @@ G_BEGIN_DECLS
 
 #define _OSTREE_OBJECT_SIZES_ENTRY_SIGNATURE "ay"
 
+#define _OSTREE_SUMMARY_CACHE_PATH "tmp/cache/summaries"
+
 /**
  * OstreeRepo:
  *
@@ -316,5 +318,23 @@ _ostree_repo_read_bare_fd (OstreeRepo           *self,
 gboolean
 _ostree_repo_update_mtime (OstreeRepo        *self,
                            GError           **error);
-                           
+
+/* Load the summary from the cache if the provided .sig file is the same as the
+   cached version.  */
+gboolean
+_ostree_repo_load_cache_summary_if_same_sig (OstreeRepo        *self,
+                                             const char        *remote,
+                                             GBytes            *summary_sig,
+                                             GBytes            **summary,
+                                             GCancellable      *cancellable,
+                                             GError           **error);
+
+gboolean
+_ostree_repo_cache_summary (OstreeRepo        *self,
+                            const char        *remote,
+                            GBytes            *summary,
+                            GBytes            *summary_sig,
+                            GCancellable      *cancellable,
+                            GError           **error);
+
 G_END_DECLS
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 0be1b38..9087848 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -28,6 +28,7 @@
 #include "ostree-repo-static-delta-private.h"
 #include "ostree-metalink.h"
 #include "otutil.h"
+#include "ot-fs-utils.h"
 
 #include <gio/gunixinputstream.h>
 
@@ -1767,6 +1768,102 @@ ostree_repo_pull_one_dir (OstreeRepo               *self,
                                         progress, cancellable, error);
 }
 
+gboolean
+_ostree_repo_load_cache_summary_if_same_sig (OstreeRepo        *self,
+                                             const char        *remote,
+                                             GBytes            *summary_sig,
+                                             GBytes            **summary,
+                                             GCancellable      *cancellable,
+                                             GError           **error)
+{
+  gboolean ret = FALSE;
+  const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote, ".sig");
+
+  glnx_fd_close int prev_fd = -1;
+  g_autoptr(GBytes) old_sig_contents = NULL;
+
+  if (!ot_openat_ignore_enoent (self->repo_dir_fd, summary_cache_sig_file, &prev_fd, error))
+    goto out;
+
+  if (prev_fd < 0)
+    {
+      ret = TRUE;
+      goto out;
+    }
+
+  old_sig_contents = glnx_fd_readall_bytes (prev_fd, cancellable, error);
+  if (!old_sig_contents)
+    goto out;
+
+  if (g_bytes_compare (old_sig_contents, summary_sig) == 0)
+    {
+      const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote);
+      glnx_fd_close int summary_fd = -1;
+      GBytes *summary_data;
+
+
+      summary_fd = openat (self->repo_dir_fd, summary_cache_file, O_CLOEXEC | O_RDONLY);
+      if (summary_fd < 0)
+        {
+          if (errno == ENOENT)
+            {
+              (void) unlinkat (self->repo_dir_fd, summary_cache_sig_file, 0);
+              ret = TRUE;
+              goto out;
+            }
+
+          glnx_set_error_from_errno (error);
+          goto out;
+        }
+
+      summary_data = glnx_fd_readall_bytes (summary_fd, cancellable, error);
+      if (!summary_data)
+        goto out;
+      *summary = summary_data;
+    }
+  ret = TRUE;
+
+ out:
+  return ret;
+}
+
+gboolean
+_ostree_repo_cache_summary (OstreeRepo        *self,
+                            const char        *remote,
+                            GBytes            *summary,
+                            GBytes            *summary_sig,
+                            GCancellable      *cancellable,
+                            GError           **error)
+{
+  gboolean ret = FALSE;
+  const char *summary_cache_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote);
+  const char *summary_cache_sig_file = glnx_strjoina (_OSTREE_SUMMARY_CACHE_PATH, "/", remote, ".sig");
+
+  if (!glnx_shutil_mkdir_p_at (self->repo_dir_fd, _OSTREE_SUMMARY_CACHE_PATH, 0775, cancellable, error))
+    goto out;
+
+  if (!glnx_file_replace_contents_at (self->repo_dir_fd,
+                                      summary_cache_file,
+                                      g_bytes_get_data (summary, NULL),
+                                      g_bytes_get_size (summary),
+                                      self->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : 
GLNX_FILE_REPLACE_DATASYNC_NEW,
+                                      cancellable, error))
+    goto out;
+
+  if (!glnx_file_replace_contents_at (self->repo_dir_fd,
+                                      summary_cache_sig_file,
+                                      g_bytes_get_data (summary_sig, NULL),
+                                      g_bytes_get_size (summary_sig),
+                                      self->disable_fsync ? GLNX_FILE_REPLACE_NODATASYNC : 
GLNX_FILE_REPLACE_DATASYNC_NEW,
+                                      cancellable, error))
+    goto out;
+
+  ret = TRUE;
+ out:
+  return ret;
+
+}
+
 /* Documented in ostree-repo.c */
 gboolean
 ostree_repo_pull_with_options (OstreeRepo             *self,
@@ -1995,15 +2092,36 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
     g_autoptr(GVariant) refs = NULL;
     g_autoptr(GVariant) deltas = NULL;
     g_autoptr(GVariant) additional_metadata = NULL;
-      
-    if (!pull_data->summary)
+    gboolean summary_from_cache = FALSE;
+
+    if (!pull_data->summary_data_sig)
+      {
+        uri = suburi_new (pull_data->base_uri, "summary.sig", NULL);
+        if (!fetch_uri_contents_membuf_sync (pull_data, uri, FALSE, TRUE,
+                                             &bytes_sig, cancellable, error))
+          goto out;
+        soup_uri_free (uri);
+      }
+
+    if (bytes_sig && !_ostree_repo_load_cache_summary_if_same_sig (self,
+                                                                   remote_name_or_baseurl,
+                                                                   bytes_sig,
+                                                                   &bytes_summary,
+                                                                   cancellable,
+                                                                   error))
+      goto out;
+
+    if (bytes_summary)
+      summary_from_cache = TRUE;
+
+    if (!pull_data->summary && !bytes_summary)
       {
         uri = suburi_new (pull_data->base_uri, "summary", NULL);
         if (!fetch_uri_contents_membuf_sync (pull_data, uri, FALSE, TRUE,
                                              &bytes_summary, cancellable, error))
           goto out;
         soup_uri_free (uri);
-     }
+      }
 
     if (!bytes_summary && pull_data->gpg_verify_summary)
       {
@@ -2019,15 +2137,6 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
         goto out;
       }
 
-    if (bytes_summary)
-      {
-        uri = suburi_new (pull_data->base_uri, "summary.sig", NULL);
-        if (!fetch_uri_contents_membuf_sync (pull_data, uri, FALSE, TRUE,
-                                             &bytes_sig, cancellable, error))
-          goto out;
-        soup_uri_free (uri);
-      }
-
     if (!bytes_sig && pull_data->gpg_verify_summary)
       {
         g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
@@ -2044,6 +2153,18 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
           pull_data->summary_data_sig = g_bytes_ref (bytes_sig);
       }
 
+
+    if (!summary_from_cache && bytes_summary && bytes_sig)
+      {
+        if (!_ostree_repo_cache_summary (self,
+                                         remote_name_or_baseurl,
+                                         bytes_summary,
+                                         bytes_sig,
+                                         cancellable,
+                                         error))
+          goto out;
+      }
+
     if (pull_data->gpg_verify_summary && bytes_summary && bytes_sig)
       {
         g_autoptr(GVariant) sig_variant = NULL;
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 2398c46..25cfcb8 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -1795,6 +1795,17 @@ repo_remote_fetch_summary (OstreeRepo    *self,
         goto out;
     }
 
+  if (*out_summary && *out_signatures)
+    {
+      if (!_ostree_repo_cache_summary (self,
+                                       name,
+                                       *out_summary,
+                                       *out_signatures,
+                                       cancellable,
+                                       error))
+        goto out;
+    }
+
   ret = TRUE;
 
  out: