[ostree] deltas: Use endianness marker when parsing
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree] deltas: Use endianness marker when parsing
- Date: Fri, 26 Feb 2016 13:19:31 +0000 (UTC)
commit 04d77da0056c89f2f18ae9e47ef121c960f2307b
Author: Colin Walters <walters verbum org>
Date: Wed Feb 24 14:29:56 2016 -0500
deltas: Use endianness marker when parsing
Extend the `static-delta show` and `pull` commands to use the
endianness information (if available).
src/libostree/ostree-repo-pull.c | 13 +++-
.../ostree-repo-static-delta-compilation.c | 20 +++-
src/libostree/ostree-repo-static-delta-core.c | 92 +++++++++++++++-----
src/libostree/ostree-repo-static-delta-private.h | 46 +++++++++--
src/ostree/ot-builtin-static-delta.c | 38 ++++++++
tests/pull-test.sh | 19 ++++-
tests/test-delta.sh | 9 ++
7 files changed, 201 insertions(+), 36 deletions(-)
---
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index efa424f..0be1b38 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -1443,6 +1443,7 @@ request_static_delta_superblock_sync (OtPullData *pull_data,
static gboolean
process_one_static_delta_fallback (OtPullData *pull_data,
+ gboolean delta_byteswap,
GVariant *fallback_object,
GCancellable *cancellable,
GError **error)
@@ -1462,6 +1463,9 @@ process_one_static_delta_fallback (OtPullData *pull_data,
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
+ compressed_size = maybe_swap_endian_u64 (delta_byteswap, compressed_size);
+ uncompressed_size = maybe_swap_endian_u64 (delta_byteswap, uncompressed_size);
+
pull_data->total_deltapart_size += compressed_size;
pull_data->total_deltapart_usize += uncompressed_size;
@@ -1518,11 +1522,14 @@ process_one_static_delta (OtPullData *pull_data,
GError **error)
{
gboolean ret = FALSE;
+ gboolean delta_byteswap;
g_autoptr(GVariant) metadata = NULL;
g_autoptr(GVariant) headers = NULL;
g_autoptr(GVariant) fallback_objects = NULL;
guint i, n;
+ delta_byteswap = _ostree_delta_needs_byteswap (delta_superblock);
+
/* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
metadata = g_variant_get_child_value (delta_superblock, 0);
headers = g_variant_get_child_value (delta_superblock, 6);
@@ -1535,7 +1542,7 @@ process_one_static_delta (OtPullData *pull_data,
g_autoptr(GVariant) fallback_object =
g_variant_get_child_value (fallback_objects, i);
- if (!process_one_static_delta_fallback (pull_data,
+ if (!process_one_static_delta_fallback (pull_data, delta_byteswap,
fallback_object,
cancellable, error))
goto out;
@@ -1609,6 +1616,10 @@ process_one_static_delta (OtPullData *pull_data,
header = g_variant_get_child_value (headers, i);
g_variant_get (header, "(u aytt@ay)", &version, &csum_v, &size, &usize, &objects);
+ version = maybe_swap_endian_u32 (delta_byteswap, version);
+ size = maybe_swap_endian_u64 (delta_byteswap, size);
+ usize = maybe_swap_endian_u64 (delta_byteswap, usize);
+
if (version > OSTREE_DELTAPART_VERSION)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
diff --git a/src/libostree/ostree-repo-static-delta-compilation.c
b/src/libostree/ostree-repo-static-delta-compilation.c
index e579e91..2d02e6a 100644
--- a/src/libostree/ostree-repo-static-delta-compilation.c
+++ b/src/libostree/ostree-repo-static-delta-compilation.c
@@ -59,6 +59,7 @@ typedef struct {
guint n_rollsum;
guint n_bsdiff;
guint n_fallback;
+ gboolean swap_endian;
} OstreeStaticDeltaBuilder;
typedef enum {
@@ -1191,7 +1192,8 @@ get_fallback_headers (OstreeRepo *self,
g_variant_new ("(y aytt)",
objtype,
ostree_checksum_to_bytes_v (checksum),
- compressed_size, uncompressed_size));
+ maybe_swap_endian_u64 (builder->swap_endian,
compressed_size),
+ maybe_swap_endian_u64 (builder->swap_endian,
uncompressed_size)));
}
ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder));
@@ -1228,6 +1230,7 @@ get_fallback_headers (OstreeRepo *self,
* - bsdiff-enabled: b: Enable bsdiff compression. Default TRUE.
* - inline-parts: b: Put part data in header, to get a single file delta. Default FALSE.
* - verbose: b: Print diagnostic messages. Default FALSE.
+ * - endianness: b: Deltas use host byte order by default; this option allows choosing (G_BIG_ENDIAN or
G_LITTLE_ENDIAN)
* - filename: ay: Save delta superblock to this filename, and parts in the same directory. Default saves
to repository.
*/
gboolean
@@ -1262,6 +1265,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
g_autoptr(GVariant) fallback_headers = NULL;
g_autoptr(GVariant) detached = NULL;
gboolean inline_parts;
+ guint endianness = G_BYTE_ORDER;
g_autoptr(GFile) tmp_dir = NULL;
builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref);
builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
@@ -1277,6 +1281,11 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
max_chunk_size = 32;
builder.max_chunk_size_bytes = ((guint64)max_chunk_size) * 1000 * 1000;
+ (void) g_variant_lookup (params, "endianness", "u", &endianness);
+ g_return_val_if_fail (endianness == G_BIG_ENDIAN || endianness == G_LITTLE_ENDIAN, FALSE);
+
+ builder.swap_endian = endianness != G_BYTE_ORDER;
+
{ gboolean use_bsdiff;
if (!g_variant_lookup (params, "bsdiff-enabled", "b", &use_bsdiff))
use_bsdiff = TRUE;
@@ -1325,7 +1334,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
}
{ guint8 endianness_char;
- switch (G_BYTE_ORDER)
+
+ switch (endianness)
{
case G_LITTLE_ENDIAN:
endianness_char = 'l';
@@ -1433,10 +1443,10 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
checksum_bytes = g_bytes_new (part_checksum, OSTREE_SHA256_DIGEST_LEN);
objtype_checksum_array = objtype_checksum_array_new (part_builder->objects);
delta_part_header = g_variant_new ("(u aytt@ay)",
- OSTREE_DELTAPART_VERSION,
+ maybe_swap_endian_u32 (builder.swap_endian,
OSTREE_DELTAPART_VERSION),
ot_gvariant_new_ay_bytes (checksum_bytes),
- (guint64) g_variant_get_size (delta_part),
- part_builder->uncompressed_size,
+ maybe_swap_endian_u64 (builder.swap_endian, (guint64)
g_variant_get_size (delta_part)),
+ maybe_swap_endian_u64 (builder.swap_endian,
part_builder->uncompressed_size),
ot_gvariant_new_ay_bytes (objtype_checksum_array));
g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header));
diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c
index 6b1dc57..97cdb8c 100644
--- a/src/libostree/ostree-repo-static-delta-core.c
+++ b/src/libostree/ostree-repo-static-delta-core.c
@@ -589,6 +589,7 @@ _ostree_static_delta_part_open (GInputStream *part_in,
static gboolean
show_one_part (OstreeRepo *self,
+ gboolean swap_endian,
const char *from,
const char *to,
GVariant *meta_entries,
@@ -608,6 +609,8 @@ show_one_part (OstreeRepo *self,
gint part_fd = -1;
g_variant_get_child (meta_entries, i, "(u aytt@ay)", &version, NULL, &size, &usize, &objects);
+ size = maybe_swap_endian_u64 (swap_endian, size);
+ usize = maybe_swap_endian_u64 (swap_endian, usize);
*total_size_ref += size;
*total_usize_ref += usize;
g_print ("PartMeta%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n",
@@ -666,6 +669,45 @@ show_one_part (OstreeRepo *self,
return ret;
}
+OstreeDeltaEndianness
+_ostree_delta_get_endianness (GVariant *superblock)
+{
+ guint8 endianness_char;
+ g_autoptr(GVariant) delta_meta = NULL;
+ g_autoptr(GVariantDict) delta_metadict = NULL;
+
+ delta_meta = g_variant_get_child_value (superblock, 0);
+ delta_metadict = g_variant_dict_new (delta_meta);
+
+ if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char))
+ {
+ switch (endianness_char)
+ {
+ case 'l':
+ return OSTREE_DELTA_ENDIAN_LITTLE;
+ case 'B':
+ return OSTREE_DELTA_ENDIAN_BIG;
+ default:
+ return OSTREE_DELTA_ENDIAN_INVALID;
+ }
+ }
+ return OSTREE_DELTA_ENDIAN_UNKNOWN;
+}
+
+gboolean
+_ostree_delta_needs_byteswap (GVariant *superblock)
+{
+ switch (_ostree_delta_get_endianness (superblock))
+ {
+ case OSTREE_DELTA_ENDIAN_BIG:
+ return G_BYTE_ORDER == G_LITTLE_ENDIAN;
+ case OSTREE_DELTA_ENDIAN_LITTLE:
+ return G_BYTE_ORDER == G_BIG_ENDIAN;
+ default:
+ return FALSE;
+ }
+}
+
gboolean
_ostree_repo_static_delta_dump (OstreeRepo *self,
const char *delta_id,
@@ -678,11 +720,11 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
g_autofree char *superblock_path = NULL;
glnx_fd_close int superblock_fd = -1;
g_autoptr(GVariant) delta_superblock = NULL;
- g_autoptr(GVariant) delta_meta = NULL;
- g_autoptr(GVariantDict) delta_metadict = NULL;
guint64 total_size = 0, total_usize = 0;
guint64 total_fallback_size = 0, total_fallback_usize = 0;
guint i;
+ OstreeDeltaEndianness endianness;
+ gboolean swap_endian = FALSE;
_ostree_parse_delta_name (delta_id, &from, &to);
superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to);
@@ -692,30 +734,34 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
TRUE, &delta_superblock, error))
goto out;
- delta_meta = g_variant_get_child_value (delta_superblock, 0);
- delta_metadict = g_variant_dict_new (delta_meta);
+ g_print ("%s\n", g_variant_print (delta_superblock, 1));
g_print ("Delta: %s\n", delta_id);
- { guint8 endianness_char;
- const char *endianness_description;
+ { const char *endianness_description;
- if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char))
+ endianness = _ostree_delta_get_endianness (delta_superblock);
+
+ switch (endianness)
{
- switch (endianness_char)
- {
- case 'l':
- endianness_description = "little";
- break;
- case 'B':
- endianness_description = "big";
- break;
- default:
- endianness_description = "invalid";
- break;
- }
+ case OSTREE_DELTA_ENDIAN_BIG:
+ endianness_description = "big";
+ if (G_BYTE_ORDER == G_LITTLE_ENDIAN)
+ swap_endian = TRUE;
+ break;
+ case OSTREE_DELTA_ENDIAN_LITTLE:
+ endianness_description = "little";
+ if (G_BYTE_ORDER == G_BIG_ENDIAN)
+ swap_endian = TRUE;
+ break;
+ case OSTREE_DELTA_ENDIAN_UNKNOWN:
+ endianness_description = "unknown";
+ break;
+ case OSTREE_DELTA_ENDIAN_INVALID:
+ endianness_description = "invalid";
+ break;
+ default:
+ g_assert_not_reached ();
}
- else
- endianness_description = "unknown";
g_print ("Endianness: %s\n", endianness_description);
}
@@ -739,6 +785,8 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
{
guint64 size, usize;
g_variant_get_child (fallback, i, "(y aytt)", NULL, NULL, &size, &usize);
+ size = maybe_swap_endian_u64 (swap_endian, size);
+ usize = maybe_swap_endian_u64 (swap_endian, usize);
total_fallback_size += size;
total_fallback_usize += usize;
}
@@ -757,7 +805,7 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
for (i = 0; i < n_parts; i++)
{
- if (!show_one_part (self, from, to, meta_entries, i,
+ if (!show_one_part (self, swap_endian, from, to, meta_entries, i,
&total_size, &total_usize,
cancellable, error))
goto out;
diff --git a/src/libostree/ostree-repo-static-delta-private.h
b/src/libostree/ostree-repo-static-delta-private.h
index 2da000d..d9e5c45 100644
--- a/src/libostree/ostree-repo-static-delta-private.h
+++ b/src/libostree/ostree-repo-static-delta-private.h
@@ -46,10 +46,10 @@ G_BEGIN_DECLS
/**
* OSTREE_STATIC_DELTA_META_ENTRY_FORMAT:
*
- * u: version
+ * u: version (non-canonical endian)
* ay checksum
- * guint64 size: Total size of delta (sum of parts)
- * guint64 usize: Uncompressed size of resulting objects on disk
+ * guint64 size: Total size of delta (sum of parts) (non-canonical endian)
+ * guint64 usize: Uncompressed size of resulting objects on disk (non-canonical endian)
* ARRAY[(guint8 objtype, csum object)]
*
* The checksum is of the delta payload, and each entry in the array
@@ -64,8 +64,8 @@ G_BEGIN_DECLS
*
* y: objtype
* ay: checksum
- * t: compressed size
- * t: uncompressed size
+ * t: compressed size (non-canonical endian)
+ * t: uncompressed size (non-canonical endian)
*
* Object to fetch invididually; includes compressed/uncompressed size.
*/
@@ -79,7 +79,7 @@ G_BEGIN_DECLS
*
* delta-descriptor:
* metadata: a{sv}
- * t: timestamp
+ * t: timestamp (big endian)
* from: ay checksum
* to: ay checksum
* commit: new commit object
@@ -196,4 +196,38 @@ _ostree_repo_static_delta_dump (OstreeRepo *repo,
GCancellable *cancellable,
GError **error);
+/* Used for static deltas which due to a historical mistake are
+ * inconsistent endian.
+ *
+ * https://bugzilla.gnome.org/show_bug.cgi?id=762515
+ */
+static inline guint32
+maybe_swap_endian_u32 (gboolean swap,
+ guint32 v)
+{
+ if (!swap)
+ return v;
+ return GUINT32_SWAP_LE_BE (v);
+}
+
+static inline guint64
+maybe_swap_endian_u64 (gboolean swap,
+ guint64 v)
+{
+ if (!swap)
+ return v;
+ return GUINT64_SWAP_LE_BE (v);
+}
+
+typedef enum {
+ OSTREE_DELTA_ENDIAN_BIG,
+ OSTREE_DELTA_ENDIAN_LITTLE,
+ OSTREE_DELTA_ENDIAN_UNKNOWN,
+ OSTREE_DELTA_ENDIAN_INVALID
+} OstreeDeltaEndianness;
+
+OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock);
+
+gboolean _ostree_delta_needs_byteswap (GVariant *superblock);
+
G_END_DECLS
diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c
index c3c99ba..36fb63f 100644
--- a/src/ostree/ot-builtin-static-delta.c
+++ b/src/ostree/ot-builtin-static-delta.c
@@ -32,7 +32,9 @@ static char *opt_to_rev;
static char *opt_min_fallback_size;
static char *opt_max_bsdiff_size;
static char *opt_max_chunk_size;
+static char *opt_endianness;
static gboolean opt_empty;
+static gboolean opt_swap_endianness;
static gboolean opt_inline;
static gboolean opt_disable_bsdiff;
@@ -60,6 +62,8 @@ static GOptionEntry generate_options[] = {
{ "inline", 0, 0, G_OPTION_ARG_NONE, &opt_inline, "Inline delta parts into main delta", NULL },
{ "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Create delta to revision REV", "REV" },
{ "disable-bsdiff", 0, 0, G_OPTION_ARG_NONE, &opt_disable_bsdiff, "Disable use of bsdiff", NULL },
+ { "set-endianness", 0, 0, G_OPTION_ARG_STRING, &opt_endianness, "Choose metadata endianness ('l' or 'B')",
"ENDIAN" },
+ { "swap-endianness", 0, 0, G_OPTION_ARG_NONE, &opt_swap_endianness, "Swap metadata endianness from host
order", NULL },
{ "min-fallback-size", 0, 0, G_OPTION_ARG_STRING, &opt_min_fallback_size, "Minimum uncompressed size in
megabytes for individual HTTP request", NULL},
{ "max-bsdiff-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_bsdiff_size, "Maximum size in megabytes to
consider bsdiff compression for input files", NULL},
{ "max-chunk-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_chunk_size, "Maximum size of delta chunks in
megabytes", NULL},
@@ -194,6 +198,7 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab
g_autofree char *to_resolved = NULL;
g_autofree char *from_parent_str = NULL;
g_autoptr(GVariantBuilder) parambuilder = NULL;
+ int endianness;
g_assert (opt_to_rev);
@@ -224,6 +229,37 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab
}
if (!ostree_repo_resolve_rev (repo, opt_to_rev, FALSE, &to_resolved, error))
goto out;
+
+ if (opt_endianness)
+ {
+ if (strcmp (opt_endianness, "l") == 0)
+ endianness = G_LITTLE_ENDIAN;
+ else if (strcmp (opt_endianness, "B") == 0)
+ endianness = G_BIG_ENDIAN;
+ else
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid endianness '%s'", opt_endianness);
+ goto out;
+ }
+ }
+ else
+ endianness = G_BYTE_ORDER;
+
+ if (opt_swap_endianness)
+ {
+ switch (endianness)
+ {
+ case G_LITTLE_ENDIAN:
+ endianness = G_BIG_ENDIAN;
+ break;
+ case G_BIG_ENDIAN:
+ endianness = G_LITTLE_ENDIAN;
+ break;
+ default:
+ g_assert_not_reached ();
+ }
+ }
parambuilder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
if (opt_min_fallback_size)
@@ -243,6 +279,8 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab
"inline-parts", g_variant_new_boolean (TRUE));
g_variant_builder_add (parambuilder, "{sv}", "verbose", g_variant_new_boolean (TRUE));
+ if (opt_endianness || opt_swap_endianness)
+ g_variant_builder_add (parambuilder, "{sv}", "endianness", g_variant_new_uint32 (endianness));
g_print ("Generating static delta:\n");
g_print (" From: %s\n", from_resolved ? from_resolved : "empty");
diff --git a/tests/pull-test.sh b/tests/pull-test.sh
index f73b681..9c8b41f 100755
--- a/tests/pull-test.sh
+++ b/tests/pull-test.sh
@@ -113,8 +113,8 @@ ostree --repo=ostree-srv/gnomerepo summary -u
cd ${test_tmpdir}
repo_init
${CMD_PREFIX} ostree --repo=repo pull origin main ${prev_rev}
-${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >out.txt
-assert_file_has_content out.txt 'Delta update: 0/1 parts'
+${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >dry-run-pull.txt
+assert_file_has_content dry-run-pull.txt 'Delta update: 0/1 parts'
rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main)
assert_streq "${prev_rev}" "${rev}"
${CMD_PREFIX} ostree --repo=repo fsck
@@ -143,6 +143,21 @@ assert_not_has_file baz/saucer
echo "ok static delta"
cd ${test_tmpdir}
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate --swap-endianness main
+${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u
+
+repo_init
+${CMD_PREFIX} ostree --repo=repo pull origin main ${prev_rev}
+${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas --dry-run origin main
byteswapped-dry-run-pull.txt
+${CMD_PREFIX} ostree --repo=repo fsck
+
+if ! diff -u dry-run-pull.txt byteswapped-dry-run-pull.txt; then
+ assert_not_reached "byteswapped delta differs in size"
+fi
+
+echo "ok pull byteswapped delta"
+
+cd ${test_tmpdir}
rm ostree-srv/gnomerepo/deltas -rf
ostree --repo=ostree-srv/gnomerepo summary -u
repo_init
diff --git a/tests/test-delta.sh b/tests/test-delta.sh
index 84ce8a7..4679ab8 100755
--- a/tests/test-delta.sh
+++ b/tests/test-delta.sh
@@ -119,6 +119,15 @@ assert_file_has_content show.txt 'Endianness: \(little\|big\)'
echo 'ok show'
+${CMD_PREFIX} ostree --repo=repo static-delta generate --swap-endianness --from=${origrev} --to=${newrev}
+${CMD_PREFIX} ostree --repo=repo static-delta show ${origrev}-${newrev} > show-swapped.txt
+totalsize_orig=$(grep 'Total Size:' show.txt)
+totalsize_swapped=$(grep 'Total Size:' show-swapped.txt)
+assert_not_streq "${totalsize_orig}" ""
+assert_streq "${totalsize_orig}" "${totalsize_swapped}"
+
+echo 'ok generate + show endian swapped'
+
mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=archive-z2
${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev}
${CMD_PREFIX} ostree --repo=repo2 fsck
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]