[ostree/wip/delta2: 14/21] deltas: Add support for LZMA compression

[Colin Walters <walters src gnome org>]

commit b90a7201b70a72d8b9f811affdd85df6adf752ab
Author: Colin Walters <walters verbum org>
Date:   Sun Apr 27 03:07:01 2014 -0400

    deltas: Add support for LZMA compression
    
    I got 30% better compression than gzip for my main static delta test
    case, and that will likely improve more once we do better clustering.

 .../ostree-repo-static-delta-compilation.c         |   15 ++++++----
 .../ostree-repo-static-delta-processing.c          |   31 ++++++++++++++------
 2 files changed, 31 insertions(+), 15 deletions(-)
---
diff --git a/src/libostree/ostree-repo-static-delta-compilation.c 
b/src/libostree/ostree-repo-static-delta-compilation.c
index 0ebaa34..f309bbd 100644
--- a/src/libostree/ostree-repo-static-delta-compilation.c
+++ b/src/libostree/ostree-repo-static-delta-compilation.c
@@ -24,6 +24,7 @@
 
 #include "ostree-core-private.h"
 #include "ostree-repo-private.h"
+#include "ostree-lzma-compressor.h"
 #include "ostree-repo-static-delta-private.h"
 #include "ostree-diff.h"
 #include "otutil.h"
@@ -151,7 +152,9 @@ generate_delta_lowlatency (OstreeRepo                       *repo,
       g_hash_table_insert (new_reachable_objects, g_variant_ref (serialized_key), serialized_key);
     }
   
-  g_printerr ("%u new reachable objects\n", g_hash_table_size (new_reachable_objects));
+  g_printerr ("modified: %u removed: %u added: %u; total %u new reachable objects\n",
+              modified->len, removed->len, added->len, 
+              g_hash_table_size (new_reachable_objects));
 
   current_part = allocate_part (builder);
 
@@ -374,7 +377,7 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
       gs_unref_object GInputStream *part_payload_in = NULL;
       gs_unref_object GMemoryOutputStream *part_payload_out = NULL;
       gs_unref_object GConverterOutputStream *part_payload_compressor = NULL;
-      gs_unref_object GConverter *zlib_compressor = NULL;
+      gs_unref_object GConverter *compressor = NULL;
       gs_unref_variant GVariant *delta_part_content = NULL;
       gs_unref_variant GVariant *delta_part = NULL;
       gs_unref_variant GVariant *delta_part_header = NULL;
@@ -390,11 +393,11 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
                                           ot_gvariant_new_ay_bytes (operations_b));
       g_variant_ref_sink (delta_part_content);
 
-      /* Hardcode gzip for now */
-      zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9);
+      /* Hardcode xz for now */
+      compressor = (GConverter*)_ostree_lzma_compressor_new (NULL);
       part_payload_in = ot_variant_read (delta_part_content);
       part_payload_out = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
-      part_payload_compressor = (GConverterOutputStream*)g_converter_output_stream_new 
((GOutputStream*)part_payload_out, zlib_compressor);
+      part_payload_compressor = (GConverterOutputStream*)g_converter_output_stream_new 
((GOutputStream*)part_payload_out, compressor);
 
       if (0 > g_output_stream_splice ((GOutputStream*)part_payload_compressor, part_payload_in,
                                       G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET | 
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
@@ -403,7 +406,7 @@ ostree_repo_static_delta_generate (OstreeRepo                   *self,
 
       /* FIXME - avoid duplicating memory here */
       delta_part = g_variant_new ("(y ay)",
-                                  (guint8)'g',
+                                  (guint8)'x',
                                   ot_gvariant_new_ay_bytes (g_memory_output_stream_steal_as_bytes 
(part_payload_out)));
 
       if (!gs_file_open_in_tmpdir (self->tmp_dir, 0644,
diff --git a/src/libostree/ostree-repo-static-delta-processing.c 
b/src/libostree/ostree-repo-static-delta-processing.c
index 01dbd1b..271c39f 100644
--- a/src/libostree/ostree-repo-static-delta-processing.c
+++ b/src/libostree/ostree-repo-static-delta-processing.c
@@ -24,6 +24,7 @@
 
 #include "ostree-repo-private.h"
 #include "ostree-repo-static-delta-private.h"
+#include "ostree-lzma-decompressor.h"
 #include "otutil.h"
 #include "ostree-varint.h"
 
@@ -204,17 +205,16 @@ _ostree_static_delta_part_execute_raw (OstreeRepo      *repo,
 }
 
 static gboolean
-zlib_uncompress_data (GBytes       *data,
-                      GBytes      **out_uncompressed,
-                      GCancellable *cancellable,
-                      GError      **error)
+decompress_all (GConverter   *converter,
+                GBytes       *data,
+                GBytes      **out_uncompressed,
+                GCancellable *cancellable,
+                GError      **error)
 {
   gboolean ret = FALSE;
   gs_unref_object GMemoryInputStream *memin = (GMemoryInputStream*)g_memory_input_stream_new_from_bytes 
(data);
   gs_unref_object GMemoryOutputStream *memout = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, 
g_realloc, g_free);
-  gs_unref_object GConverter *zlib_decomp =
-    (GConverter*) g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW);
-  gs_unref_object GInputStream *convin = g_converter_input_stream_new ((GInputStream*)memin, zlib_decomp);
+  gs_unref_object GInputStream *convin = g_converter_input_stream_new ((GInputStream*)memin, converter);
 
   if (0 > g_output_stream_splice ((GOutputStream*)memout, convin,
                                   G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE |
@@ -264,8 +264,21 @@ _ostree_static_delta_part_execute (OstreeRepo      *repo,
       break;
     case 'g':
       {
-        if (!zlib_uncompress_data (part_payload_bytes, &payload_data,
-                                   cancellable, error))
+        gs_unref_object GConverter *decomp =
+          (GConverter*) g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW);
+
+        if (!decompress_all (decomp, part_payload_bytes, &payload_data,
+                             cancellable, error))
+          goto out;
+      }
+      break;
+    case 'x':
+      {
+        gs_unref_object GConverter *decomp =
+          (GConverter*) _ostree_lzma_decompressor_new ();
+
+        if (!decompress_all (decomp, part_payload_bytes, &payload_data,
+                             cancellable, error))
           goto out;
       }
       break;