[ostree] deltas: Heuristically detect endianness for older deltas

[Colin Walters <walters src gnome org>]

commit 7fdf07271031cd4a88af9280b41ece5dc7c1d580
Author: Colin Walters <walters verbum org>
Date:   Wed Feb 24 17:04:04 2016 -0500

    deltas: Heuristically detect endianness for older deltas
    
    If the average object size is greater than 4GiB, let's assume we're
    dealing with opposite endianness.  I'm fairly confident no one is
    going to be shipping peta- or exa- byte size ostree deltas, period.
    Past the gigabyte scale you really want bittorrent or something.

 Makefile-tests.am                                |    2 +
 src/libostree/ostree-repo-static-delta-core.c    |   74 +++++++++++++++++++---
 src/libostree/ostree-repo-static-delta-private.h |    3 +-
 tests/pre-endian-deltas-repo-big.tar.xz          |  Bin 0 -> 5760 bytes
 tests/pre-endian-deltas-repo-little.tar.xz       |  Bin 0 -> 5752 bytes
 tests/test-delta.sh                              |   21 ++++++
 6 files changed, 89 insertions(+), 11 deletions(-)
---
diff --git a/Makefile-tests.am b/Makefile-tests.am
index 475519d..6ec5835 100644
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -85,6 +85,8 @@ insttest_DATA = tests/archive-test.sh \
        tests/test-basic-user.sh \
        tests/test-local-pull.sh \
        tests/corrupt-repo-ref.js \
+       tests/pre-endian-deltas-repo-big.tar.xz \
+       tests/pre-endian-deltas-repo-little.tar.xz \
        $(NULL)
 
 insttest_SCRIPTS += \
diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c
index 97cdb8c..0669f69 100644
--- a/src/libostree/ostree-repo-static-delta-core.c
+++ b/src/libostree/ostree-repo-static-delta-core.c
@@ -670,15 +670,22 @@ show_one_part (OstreeRepo                    *self,
 }
 
 OstreeDeltaEndianness
-_ostree_delta_get_endianness (GVariant *superblock)
+_ostree_delta_get_endianness (GVariant *superblock,
+                              gboolean *out_was_heuristic)
 {
   guint8 endianness_char;
   g_autoptr(GVariant) delta_meta = NULL;
   g_autoptr(GVariantDict) delta_metadict = NULL;
+  guint64 total_size = 0;
+  guint64 total_usize = 0;
+  guint total_objects = 0;
 
   delta_meta = g_variant_get_child_value (superblock, 0);
   delta_metadict = g_variant_dict_new (delta_meta);
 
+  if (out_was_heuristic)
+    *out_was_heuristic = FALSE;
+
   if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char))
     {
       switch (endianness_char)
@@ -691,13 +698,58 @@ _ostree_delta_get_endianness (GVariant *superblock)
           return OSTREE_DELTA_ENDIAN_INVALID;
         }
     }
-  return OSTREE_DELTA_ENDIAN_UNKNOWN;
+
+  if (out_was_heuristic)
+    *out_was_heuristic = TRUE;
+
+  { g_autoptr(GVariant) meta_entries = NULL;
+    guint n_parts;
+    guint i;
+
+    g_variant_get_child (superblock, 6, "@a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT, &meta_entries);
+    n_parts = g_variant_n_children (meta_entries);
+
+    for (i = 0; i < n_parts; i++)
+      {
+        g_autoptr(GVariant) objects = NULL;
+        guint64 size, usize;
+        guint n_objects;
+
+        g_variant_get_child (meta_entries, i, "(u aytt@ay)", NULL, NULL, &size, &usize, &objects);
+        n_objects = (guint)(g_variant_get_size (objects) / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN);
+
+        total_objects += n_objects;
+        total_size += size;
+        total_usize += usize;
+      }
+
+    /* If the average object size is greater than 4GiB, let's assume
+     * we're dealing with opposite endianness.  I'm fairly confident
+     * no one is going to be shipping peta- or exa- byte size ostree
+     * deltas, period.  Past the gigabyte scale you really want
+     * bittorrent or something.
+     */
+    if ((total_size / total_objects) > G_MAXUINT32)
+      {
+        switch (G_BYTE_ORDER)
+          {
+          case G_BIG_ENDIAN:
+            return OSTREE_DELTA_ENDIAN_LITTLE;
+          case G_LITTLE_ENDIAN:
+            return OSTREE_DELTA_ENDIAN_BIG;
+          default:
+            g_assert_not_reached ();
+          }
+      }
+
+    return G_BYTE_ORDER;
+  }
 }
 
 gboolean
 _ostree_delta_needs_byteswap (GVariant *superblock)
 {
-  switch (_ostree_delta_get_endianness (superblock))
+  switch (_ostree_delta_get_endianness (superblock, NULL))
     {
     case OSTREE_DELTA_ENDIAN_BIG:
       return G_BYTE_ORDER == G_LITTLE_ENDIAN;
@@ -738,24 +790,28 @@ _ostree_repo_static_delta_dump (OstreeRepo                    *self,
 
   g_print ("Delta: %s\n", delta_id);
   { const char *endianness_description;
+    gboolean was_heuristic;
 
-    endianness = _ostree_delta_get_endianness (delta_superblock);
+    endianness = _ostree_delta_get_endianness (delta_superblock, &was_heuristic);
 
     switch (endianness)
       {
       case OSTREE_DELTA_ENDIAN_BIG:
-        endianness_description = "big";
+        if (was_heuristic)
+          endianness_description = "big (heuristic)";
+        else
+          endianness_description = "big";
         if (G_BYTE_ORDER == G_LITTLE_ENDIAN)
           swap_endian = TRUE;
         break;
       case OSTREE_DELTA_ENDIAN_LITTLE:
-        endianness_description = "little";
+        if (was_heuristic)
+          endianness_description = "little (heuristic)";
+        else
+          endianness_description = "little";
         if (G_BYTE_ORDER == G_BIG_ENDIAN)
           swap_endian = TRUE;
         break;
-      case OSTREE_DELTA_ENDIAN_UNKNOWN:
-        endianness_description = "unknown";
-        break;
       case OSTREE_DELTA_ENDIAN_INVALID:
         endianness_description = "invalid";
         break;
diff --git a/src/libostree/ostree-repo-static-delta-private.h 
b/src/libostree/ostree-repo-static-delta-private.h
index d9e5c45..41ddad4 100644
--- a/src/libostree/ostree-repo-static-delta-private.h
+++ b/src/libostree/ostree-repo-static-delta-private.h
@@ -222,11 +222,10 @@ maybe_swap_endian_u64 (gboolean swap,
 typedef enum {
   OSTREE_DELTA_ENDIAN_BIG,
   OSTREE_DELTA_ENDIAN_LITTLE,
-  OSTREE_DELTA_ENDIAN_UNKNOWN,
   OSTREE_DELTA_ENDIAN_INVALID
 } OstreeDeltaEndianness;
 
-OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock);
+OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock, gboolean *out_was_heuristic);
 
 gboolean _ostree_delta_needs_byteswap (GVariant *superblock);
 
diff --git a/tests/pre-endian-deltas-repo-big.tar.xz b/tests/pre-endian-deltas-repo-big.tar.xz
new file mode 100644
index 0000000..05e51d6
Binary files /dev/null and b/tests/pre-endian-deltas-repo-big.tar.xz differ
diff --git a/tests/pre-endian-deltas-repo-little.tar.xz b/tests/pre-endian-deltas-repo-little.tar.xz
new file mode 100644
index 0000000..ef05045
Binary files /dev/null and b/tests/pre-endian-deltas-repo-little.tar.xz differ
diff --git a/tests/test-delta.sh b/tests/test-delta.sh
index 4679ab8..ebe3557 100755
--- a/tests/test-delta.sh
+++ b/tests/test-delta.sh
@@ -128,6 +128,27 @@ assert_streq "${totalsize_orig}" "${totalsize_swapped}"
 
 echo 'ok generate + show endian swapped'
 
+tar xf ${SRCDIR}/pre-endian-deltas-repo-big.tar.xz
+mv pre-endian-deltas-repo{,-big}
+tar xf ${SRCDIR}/pre-endian-deltas-repo-little.tar.xz
+mv pre-endian-deltas-repo{,-little}
+legacy_origrev=$(${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big rev-parse main^)
+legacy_newrev=$(${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big rev-parse main)
+${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big static-delta show ${legacy_origrev}-${legacy_newrev} 
> show-legacy-big.txt
+totalsize_legacy_big=$(grep 'Total Size:' show-legacy-big.txt)
+${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big static-delta show ${legacy_origrev}-${legacy_newrev} 
> show-legacy-little.txt
+totalsize_legacy_little=$(grep 'Total Size:' show-legacy-little.txt)
+for f in show-legacy-{big,little}.txt; do
+    if grep 'Endianness:.*heuristic' $f; then
+       found_heuristic=yes
+       break
+    fi
+done
+assert_streq "${found_heuristic}" "yes"
+assert_streq "${totalsize_legacy_big}" "${totalsize_legacy_little}"
+
+echo 'ok heuristic endian detection'
+
 mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=archive-z2
 ${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev}
 ${CMD_PREFIX} ostree --repo=repo2 fsck