[ostree/wip/delta3] delta: start splitting meta/content

[Colin Walters <walters src gnome org>]

commit d7d6d6908061068d0a96c4a1216d3153ac6266ff
Author: Colin Walters <walters verbum org>
Date:   Mon Apr 28 00:09:29 2014 -0400

    delta: start splitting meta/content
    
    We need to embed the size of each content object in order
    to sanely stream it.
    
    To make this further better, split metadata/content delta types.
    (Although we could stick with the same code for both at a relatively
    minor efficiency hit)

 .../ostree-repo-static-delta-compilation.c         |    3 +-
 src/libostree/ostree-repo-static-delta-private.h   |   42 +++++++++++++++++---
 .../ostree-repo-static-delta-processing.c          |   21 ++++++++--
 3 files changed, 55 insertions(+), 11 deletions(-)
---
diff --git a/src/libostree/ostree-repo-static-delta-compilation.c 
b/src/libostree/ostree-repo-static-delta-compilation.c
index 7f98e64..8f66800 100644
--- a/src/libostree/ostree-repo-static-delta-compilation.c
+++ b/src/libostree/ostree-repo-static-delta-compilation.c
@@ -38,7 +38,8 @@ typedef struct {
 } OstreeStaticDeltaPartBuilder;
 
 typedef struct {
-  GPtrArray *parts;
+  GPtrArray *meta_parts;
+  GPtrArray *content_parts;
   GPtrArray *fallback_objects;
   guint64 loose_compressed_size;
   guint64 max_usize_bytes;
diff --git a/src/libostree/ostree-repo-static-delta-private.h 
b/src/libostree/ostree-repo-static-delta-private.h
index a6f8583..26e84bc 100644
--- a/src/libostree/ostree-repo-static-delta-private.h
+++ b/src/libostree/ostree-repo-static-delta-private.h
@@ -30,20 +30,29 @@ G_BEGIN_DECLS
 #define OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN 33
 
 /**
- * OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT:
+ * OSTREE_STATIC_DELTA_PART_META_PAYLOAD_FORMAT:
+ *
+ *   y  compression type (0: none, 'z': zlib)
+ *   ---
+ *   aay objects
+ */
+#define OSTREE_STATIC_DELTA_PART_META_PAYLOAD_FORMAT "(aay)"
+
+/**
+ * OSTREE_STATIC_DELTA_PART_CONTENT_PAYLOAD_FORMAT:
  *
  *   y  compression type (0: none, 'z': zlib)
  *   ---
  *   ay data source
- *   ay operations
+ *   ay (varint size, operations*)
  */
-#define OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT "(ayay)"
+#define OSTREE_STATIC_DELTA_PART_CONTENT_PAYLOAD_FORMAT "(ayay)"
 
 /**
  * OSTREE_STATIC_DELTA_META_ENTRY_FORMAT:
  *
  *   ay checksum
- *   guint64 size:   Total size of delta (sum of parts)
+ *   guint64 size:   Compressed delta size
  *   guint64 usize:   Uncompressed size of resulting objects on disk
  *   ARRAY[(guint8 objtype, csum object)]
  *
@@ -53,6 +62,19 @@ G_BEGIN_DECLS
 
 #define OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "(ayttay)"
 
+/**
+ * OSTREE_STATIC_DELTA_CONTENT_ENTRY_FORMAT:
+ *
+ *   ay checksum
+ *   guint64 size:   Compressed delta size
+ *   guint64 usize:   Uncompressed size of resulting objects on disk
+ *   ARRAY[csum]
+ *
+ * Like the meta entry, except without an object type.
+ */
+
+#define OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "(ayttay)"
+
 
 /**
  * OSTREE_STATIC_DELTA_FALLBACK_FORMAT:
@@ -80,6 +102,7 @@ G_BEGIN_DECLS
  *   commit: new commit object
  *   ARRAY[(csum from, csum to)]: ay
  *   ARRAY[delta-meta-entry]
+ *   ARRAY[delta-content-entry]
  *   array[fallback]
  *
  * The metadata would include things like a version number, as well as
@@ -90,13 +113,20 @@ G_BEGIN_DECLS
  * recursion mechanism that would potentially allow saving significant
  * storage space on the server.
  *
- * The heart of the static delta: the array of delta parts.
+ * The heart of the static delta: two arrays of delta parts, one for
+ * metadata, one for content.
  *
  * Finally, we have the fallback array, which is the set of objects to
  * fetch individually - the compiler determined it wasn't worth
  * duplicating the space.
  */ 
-#define OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT "(a{sv}tayay" OSTREE_COMMIT_GVARIANT_STRING "aya" 
OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")"
+#define OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT "(a{sv}tayay" \
+  OSTREE_COMMIT_GVARIANT_STRING \
+  "ay"                                      \
+  "a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT \
+  "a" OSTREE_STATIC_DELTA_CONTENT_ENTRY_FORMAT \
+  "a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT \
+  ")"
 
 gboolean _ostree_static_delta_part_validate (OstreeRepo     *repo,
                                              GFile          *part_path,
diff --git a/src/libostree/ostree-repo-static-delta-processing.c 
b/src/libostree/ostree-repo-static-delta-processing.c
index 271c39f..89636e2 100644
--- a/src/libostree/ostree-repo-static-delta-processing.c
+++ b/src/libostree/ostree-repo-static-delta-processing.c
@@ -43,6 +43,7 @@ typedef struct {
   const guint8   *output_target;
   GFile          *output_tmp_path;
   GOutputStream  *output_tmp_stream;
+  GInputStream   *content_in_stream;
   const guint8   *input_target_csum;
 
   const guint8   *payload_data;
@@ -99,10 +100,22 @@ open_output_target_csum (OstreeRepo                  *repo,
 
   state->output_objtype = (OstreeObjectType) *objcsum;
   state->output_target = objcsum + 1;
-  if (!gs_file_open_in_tmpdir (repo->tmp_dir, 0644,
-                               &state->output_tmp_path, &state->output_tmp_stream,
-                               cancellable, error))
-    goto out;
+  if (OSTREE_OBJECT_TYPE_IS_META (state->output_objtype))
+    {
+      if (!gs_file_open_in_tmpdir (repo->tmp_dir, 0644,
+                                   &state->output_tmp_path, &state->output_tmp_stream,
+                                   cancellable, error))
+        goto out;
+    }
+  else
+    {
+      int pipefds[2];
+      
+      if (!g_unix_open_pipe (pipefds, FD_CLOEXEC, error))
+        goto out;
+
+      state->output_tmp_stream = g_unix_output_stream_new (pipefds[1], TRUE);
+    }
 
   ret = TRUE;
  out: