[ostree] static deltas: Add support for inline-parts

[Alexander Larsson <alexl src gnome org>]

commit 11a79220e2b5789ed6d835cd3d1374be306f2398
Author: Alexander Larsson <alexl redhat com>
Date:   Thu Sep 10 20:52:54 2015 +0200

    static deltas: Add support for inline-parts
    
    In this mode the parts are stored in the metadata of the main delta
    superblock file.  This can be useful if you want a single-file delta
    for easy transport, or for http in the case the delta is very small.

 src/libostree/ostree-repo-pull.c                   |   58 ++++++++++++++++----
 .../ostree-repo-static-delta-compilation.c         |   48 ++++++++++++-----
 src/libostree/ostree-repo-static-delta-core.c      |   45 +++++++++++++---
 3 files changed, 119 insertions(+), 32 deletions(-)
---
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 7ba3ef6..b1fdc68 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -1489,11 +1489,13 @@ process_one_static_delta (OtPullData   *pull_data,
                           GError      **error)
 {
   gboolean ret = FALSE;
+  g_autoptr(GVariant) metadata = NULL;
   g_autoptr(GVariant) headers = NULL;
   g_autoptr(GVariant) fallback_objects = NULL;
   guint i, n;
 
   /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
+  metadata = g_variant_get_child_value (delta_superblock, 0);
   headers = g_variant_get_child_value (delta_superblock, 6);
   fallback_objects = g_variant_get_child_value (delta_superblock, 7);
 
@@ -1557,6 +1559,8 @@ process_one_static_delta (OtPullData   *pull_data,
       FetchStaticDeltaData *fetch_data;
       g_autoptr(GVariant) csum_v = NULL;
       g_autoptr(GVariant) objects = NULL;
+      g_autoptr(GVariant) part_data = NULL;
+      g_autoptr(GBytes) delta_data = NULL;
       guint64 size, usize;
       guint32 version;
 
@@ -1574,6 +1578,29 @@ process_one_static_delta (OtPullData   *pull_data,
       if (!csum)
         goto out;
 
+      deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i);
+
+      part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE ("(yay)"));
+      if (part_data)
+        {
+          g_autofree char *actual_checksum = NULL;
+          g_autofree char *expected_checksum = ostree_checksum_from_bytes_v (csum_v);
+
+          delta_data = g_variant_get_data_as_bytes (part_data);
+
+          /* For inline parts we are relying on per-commit GPG, so this isn't strictly necessary for 
security.
+           * See https://github.com/GNOME/ostree/pull/139
+           */
+          actual_checksum = g_compute_checksum_for_bytes (G_CHECKSUM_SHA256, delta_data);
+          if (strcmp (actual_checksum, expected_checksum) != 0)
+            {
+              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                           "Corrupted static delta part; checksum expected='%s' actual='%s'",
+                           expected_checksum, actual_checksum);
+              goto out;
+            }
+        }
+
       pull_data->total_deltapart_size += size;
 
       if (!_ostree_repo_static_delta_part_have_all_objects (pull_data->repo,
@@ -1596,16 +1623,27 @@ process_one_static_delta (OtPullData   *pull_data,
       fetch_data->objects = g_variant_ref (objects);
       fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v);
 
-      deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i);
-
-      target_uri = suburi_new (pull_data->base_uri, deltapart_path, NULL);
-      _ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, target_uri, size,
-                                                      OSTREE_FETCHER_DEFAULT_PRIORITY,
-                                                      pull_data->cancellable,
-                                                      static_deltapart_fetch_on_complete,
-                                                      fetch_data);
-      pull_data->n_outstanding_deltapart_fetches++;
-      soup_uri_free (target_uri);
+      if (delta_data != NULL)
+        {
+          _ostree_static_delta_part_execute_async (pull_data->repo,
+                                                   fetch_data->objects,
+                                                   delta_data,
+                                                   pull_data->cancellable,
+                                                   on_static_delta_written,
+                                                   fetch_data);
+          pull_data->n_outstanding_deltapart_write_requests++;
+        }
+      else
+        {
+          target_uri = suburi_new (pull_data->base_uri, deltapart_path, NULL);
+          _ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, target_uri, size,
+                                                          OSTREE_FETCHER_DEFAULT_PRIORITY,
+                                                          pull_data->cancellable,
+                                                          static_deltapart_fetch_on_complete,
+                                                          fetch_data);
+          pull_data->n_outstanding_deltapart_fetches++;
+          soup_uri_free (target_uri);
+        }
     }
 
   ret = TRUE;
diff --git a/src/libostree/ostree-repo-static-delta-compilation.c 
b/src/libostree/ostree-repo-static-delta-compilation.c
index 1208e65..3fe856b 100644
--- a/src/libostree/ostree-repo-static-delta-compilation.c
+++ b/src/libostree/ostree-repo-static-delta-compilation.c
@@ -1226,6 +1226,7 @@ get_fallback_headers (OstreeRepo               *self,
  *   for input files
  *   - compression: y: Compression type: 0=none, x=lzma, g=gzip
  *   - bsdiff-enabled: b: Enable bsdiff compression.  Default TRUE.
+ *   - inline-parts: b: Put part data in header, to get a single file delta.  Default FALSE.
  *   - verbose: b: Print diagnostic messages.  Default FALSE.
  */
 gboolean
@@ -1244,7 +1245,7 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
   guint min_fallback_size;
   guint max_bsdiff_size;
   guint max_chunk_size;
-  GVariant *metadata_source;
+  GVariantBuilder metadata_builder;
   DeltaOpts delta_opts = DELTAOPT_FLAG_NONE;
   guint64 total_compressed_size = 0;
   guint64 total_uncompressed_size = 0;
@@ -1257,6 +1258,7 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
   g_autoptr(GFile) descriptor_dir = NULL;
   g_autoptr(GVariant) tmp_metadata = NULL;
   g_autoptr(GVariant) fallback_headers = NULL;
+  gboolean inline_parts;
 
   builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref);
   builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
@@ -1286,6 +1288,9 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
       delta_opts |= DELTAOPT_FLAG_VERBOSE;
   }
 
+  if (!g_variant_lookup (params, "inline-parts", "b", &inline_parts))
+    inline_parts = FALSE;
+
   if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, to,
                                  &to_commit, error))
     goto out;
@@ -1295,6 +1300,20 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
                                   cancellable, error))
     goto out;
 
+  g_variant_builder_init (&metadata_builder, G_VARIANT_TYPE ("a{sv}"));
+  if (metadata != NULL)
+    {
+      GVariantIter iter;
+      GVariant *item;
+
+      g_variant_iter_init (&iter, metadata);
+      while ((item = g_variant_iter_next_value (&iter)))
+        {
+          g_variant_builder_add (&metadata_builder, "@{sv}", item);
+          g_variant_unref (item);
+        }
+    }
+
   part_headers = g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT));
   part_tempfiles = g_ptr_array_new_with_free_func (g_object_unref);
   for (i = 0; i < builder.parts->len; i++)
@@ -1359,10 +1378,16 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
       delta_part = g_variant_new ("(y ay)",
                                   compression_type_char,
                                   ot_gvariant_new_ay_bytes (g_memory_output_stream_steal_as_bytes 
(part_payload_out)));
+      g_variant_ref_sink (delta_part);
 
-      if (!gs_file_open_in_tmpdir (self->tmp_dir, 0644,
-                                   &part_tempfile, &part_temp_outstream,
-                                   cancellable, error))
+      if (inline_parts)
+        {
+          g_autofree char *part_relpath = _ostree_get_relative_static_delta_part_path (from, to, i);
+          g_variant_builder_add (&metadata_builder, "{sv}", part_relpath, delta_part);
+        }
+      else if (!gs_file_open_in_tmpdir (self->tmp_dir, 0644,
+                                        &part_tempfile, &part_temp_outstream,
+                                        cancellable, error))
         goto out;
       part_in = ot_variant_read (delta_part);
       if (!ot_gio_splice_get_checksum (part_temp_outstream, part_in,
@@ -1378,8 +1403,10 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
                                          (guint64) g_variant_get_size (delta_part),
                                          part_builder->uncompressed_size,
                                          ot_gvariant_new_ay_bytes (objtype_checksum_array));
+
       g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header));
-      g_ptr_array_add (part_tempfiles, g_object_ref (part_tempfile));
+      if (part_tempfile)
+        g_ptr_array_add (part_tempfiles, g_object_ref (part_tempfile));
       
       total_compressed_size += g_variant_get_size (delta_part);
       total_uncompressed_size += part_builder->uncompressed_size;
@@ -1400,7 +1427,7 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
   if (!gs_file_ensure_directory (descriptor_dir, TRUE, cancellable, error))
     goto out;
 
-  for (i = 0; i < builder.parts->len; i++)
+  for (i = 0; i < part_tempfiles->len; i++)
     {
       GFile *tempfile = part_tempfiles->pdata[i];
       g_autofree char *part_relpath = _ostree_get_relative_static_delta_part_path (from, to, i);
@@ -1410,13 +1437,6 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
         goto out;
     }
 
-  if (metadata != NULL)
-    metadata_source = metadata;
-  else
-    {
-      metadata_source = ot_gvariant_new_empty_string_dict ();
-    }
-
   if (!get_fallback_headers (self, &builder, &fallback_headers,
                              cancellable, error))
     goto out;
@@ -1432,7 +1452,7 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
     delta_descriptor = g_variant_new ("(@a{sv}t ay@ay@" OSTREE_COMMIT_GVARIANT_STRING "ay"
                                       "a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT
                                       "@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")",
-                                      metadata_source,
+                                      g_variant_builder_end (&metadata_builder),
                                       GUINT64_TO_BE (g_date_time_to_unix (now)),
                                       from_csum_v,
                                       to_csum_v,
diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c
index 688be0a..1a96166 100644
--- a/src/libostree/ostree-repo-static-delta-core.c
+++ b/src/libostree/ostree-repo-static-delta-core.c
@@ -20,6 +20,7 @@
 
 #include "config.h"
 
+#include "ostree-core-private.h"
 #include "ostree-repo-private.h"
 #include "ostree-repo-static-delta-private.h"
 #include "otutil.h"
@@ -227,7 +228,10 @@ ostree_repo_static_delta_execute_offline (OstreeRepo                    *self,
   g_autoptr(GFile) meta_file = g_file_get_child (dir, "superblock");
   g_autoptr(GVariant) meta = NULL;
   g_autoptr(GVariant) headers = NULL;
+  g_autoptr(GVariant) metadata = NULL;
   g_autoptr(GVariant) fallback = NULL;
+  g_autofree char *to_checksum = NULL;
+  g_autofree char *from_checksum = NULL;
 
   if (!ot_util_variant_map (meta_file, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT),
                             FALSE, &meta, error))
@@ -238,7 +242,7 @@ ostree_repo_static_delta_execute_offline (OstreeRepo                    *self,
   /* Write the to-commit object */
   {
     g_autoptr(GVariant) to_csum_v = NULL;
-    g_autofree char *to_checksum = NULL;
+    g_autoptr(GVariant) from_csum_v = NULL;
     g_autoptr(GVariant) to_commit = NULL;
     gboolean have_to_commit;
 
@@ -247,6 +251,14 @@ ostree_repo_static_delta_execute_offline (OstreeRepo                    *self,
       goto out;
     to_checksum = ostree_checksum_from_bytes_v (to_csum_v);
 
+    from_csum_v = g_variant_get_child_value (meta, 2);
+    if (g_variant_n_children (from_csum_v) > 0)
+      {
+        if (!ostree_validate_structureof_csum_v (from_csum_v, error))
+          goto out;
+        from_checksum = ostree_checksum_from_bytes_v (from_csum_v);
+      }
+
     if (!ostree_repo_has_object (self, OSTREE_OBJECT_TYPE_COMMIT, to_checksum,
                                  &have_to_commit, cancellable, error))
       goto out;
@@ -270,6 +282,7 @@ ostree_repo_static_delta_execute_offline (OstreeRepo                    *self,
     }
 
   headers = g_variant_get_child_value (meta, 6);
+  metadata = g_variant_get_child_value (meta, 0);
   n = g_variant_n_children (headers);
   for (i = 0; i < n; i++)
     {
@@ -278,12 +291,15 @@ ostree_repo_static_delta_execute_offline (OstreeRepo                    *self,
       guint64 usize;
       const guchar *csum;
       gboolean have_all;
+      g_autoptr(GBytes) delta_data = NULL;
+      g_autoptr(GVariant) part_data = NULL;
       g_autoptr(GVariant) header = NULL;
       g_autoptr(GVariant) csum_v = NULL;
       g_autoptr(GVariant) objects = NULL;
       g_autoptr(GFile) part_path = NULL;
       g_autoptr(GInputStream) raw_in = NULL;
       g_autoptr(GInputStream) in = NULL;
+      g_autofree char *deltapart_path = NULL;
 
       header = g_variant_get_child_value (headers, i);
       g_variant_get (header, "(u aytt@ay)", &version, &csum_v, &size, &usize, &objects);
@@ -309,9 +325,17 @@ ostree_repo_static_delta_execute_offline (OstreeRepo                    *self,
       if (!csum)
         goto out;
 
-      part_path = ot_gfile_resolve_path_printf (dir, "%u", i);
+      deltapart_path =
+        _ostree_get_relative_static_delta_part_path (from_checksum, to_checksum, i);
 
-      in = (GInputStream*)g_file_read (part_path, cancellable, error);
+      part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE("(yay)"));
+      if (part_data)
+        in = ot_variant_read (part_data);
+      else
+        {
+          part_path = ot_gfile_resolve_path_printf (dir, "%u", i);
+          in = (GInputStream*)g_file_read (part_path, cancellable, error);
+        }
       if (!in)
         goto out;
 
@@ -325,14 +349,19 @@ ostree_repo_static_delta_execute_offline (OstreeRepo                    *self,
         }
 
       {
-        GMappedFile *mfile = gs_file_map_noatime (part_path, cancellable, error);
         g_autoptr(GBytes) bytes = NULL;
 
-        if (!mfile)
-          goto out;
+        if (part_data)
+          bytes = g_variant_get_data_as_bytes (part_data);
+        else
+          {
+            GMappedFile *mfile = gs_file_map_noatime (part_path, cancellable, error);
+            if (!mfile)
+              goto out;
 
-        bytes = g_mapped_file_get_bytes (mfile);
-        g_mapped_file_unref (mfile);
+            bytes = g_mapped_file_get_bytes (mfile);
+            g_mapped_file_unref (mfile);
+          }
         
         if (!_ostree_static_delta_part_execute (self, objects, bytes,
                                                 cancellable, error))