[ostree/wip/delta2: 13/21] WIP: Static deltas
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree/wip/delta2: 13/21] WIP: Static deltas
- Date: Sun, 4 May 2014 20:55:50 +0000 (UTC)
commit 3c4cca0c4055bf8dee6acb9af0a2d9117b9b4354
Author: Colin Walters <walters verbum org>
Date: Sun Feb 9 16:11:37 2014 -0500
WIP: Static deltas
This is a large code drop to support static deltas. To be rebased.
src/libostree/ostree-core-private.h | 8 +
src/libostree/ostree-core.c | 35 ++
src/libostree/ostree-fetcher.c | 2 +-
src/libostree/ostree-repo-private.h | 9 +
src/libostree/ostree-repo-pull.c | 438 +++++++++++++++++---
.../ostree-repo-static-delta-compilation.c | 119 +++++-
src/libostree/ostree-repo-static-delta-core.c | 115 ++----
src/libostree/ostree-repo-static-delta-private.h | 70 +++-
.../ostree-repo-static-delta-processing.c | 208 +++++++++-
src/libostree/ostree-repo.c | 211 +++++++---
src/libostree/ostree-repo.h | 8 +
src/ostree/ot-builtin-static-delta.c | 22 +
tests/pull-test.sh | 86 ++++-
13 files changed, 1096 insertions(+), 235 deletions(-)
---
diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h
index 851e74b..96902b6 100644
--- a/src/libostree/ostree-core-private.h
+++ b/src/libostree/ostree-core-private.h
@@ -97,6 +97,10 @@ _ostree_get_relative_static_delta_path (const char *from,
const char *to);
char *
+_ostree_get_relative_static_delta_detachedmeta_path (const char *from,
+ const char *to);
+
+char *
_ostree_get_relative_static_delta_part_path (const char *from,
const char *to,
guint i);
@@ -114,5 +118,9 @@ _ostree_loose_path_with_suffix (char *buf,
OstreeRepoMode repo_mode,
const char *suffix);
+GVariant *
+_ostree_detached_metadata_append_gpg_sig (GVariant *existing_metadata,
+ GBytes *signature_bytes);
+
G_END_DECLS
diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c
index b8fa114..a5797e3 100644
--- a/src/libostree/ostree-core.c
+++ b/src/libostree/ostree-core.c
@@ -1325,6 +1325,13 @@ char *
_ostree_get_relative_static_delta_path (const char *from,
const char *to)
{
+ return g_strdup_printf ("deltas/%s-%s/superblock", from, to);
+}
+
+char *
+_ostree_get_relative_static_delta_detachedmeta_path (const char *from,
+ const char *to)
+{
return g_strdup_printf ("deltas/%s-%s/meta", from, to);
}
@@ -1763,3 +1770,31 @@ ostree_commit_get_timestamp (GVariant *commit_variant)
g_variant_get_child (commit_variant, 5, "t", &ret);
return GUINT64_FROM_BE (ret);
}
+
+GVariant *
+_ostree_detached_metadata_append_gpg_sig (GVariant *existing_metadata,
+ GBytes *signature_bytes)
+{
+ GVariantBuilder *builder;
+ gs_unref_variant GVariant *signaturedata = NULL;
+ gs_unref_variant_builder GVariantBuilder *signature_builder = NULL;
+
+ if (existing_metadata)
+ {
+ builder = ot_util_variant_builder_from_variant (existing_metadata, G_VARIANT_TYPE ("a{sv}"));
+ signaturedata = g_variant_lookup_value (existing_metadata, "ostree.gpgsigs", G_VARIANT_TYPE ("aay"));
+ if (signaturedata)
+ signature_builder = ot_util_variant_builder_from_variant (signaturedata, G_VARIANT_TYPE ("aay"));
+ }
+ if (!builder)
+ builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
+ if (!signature_builder)
+ signature_builder = g_variant_builder_new (G_VARIANT_TYPE ("aay"));
+
+ g_variant_builder_add (signature_builder, "@ay", ot_gvariant_new_ay_bytes (signature_bytes));
+
+ g_variant_builder_add (builder, "{sv}", "ostree.gpgsigs", g_variant_builder_end (signature_builder));
+
+ return g_variant_ref_sink (g_variant_builder_end (builder));
+}
+
diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c
index 06023c6..3ac4e92 100644
--- a/src/libostree/ostree-fetcher.c
+++ b/src/libostree/ostree-fetcher.c
@@ -468,7 +468,7 @@ ostree_fetcher_request_uri_with_partial_async (OstreeFetcher *self,
if (local_error != NULL)
{
g_simple_async_result_take_error (pending->result, local_error);
- g_simple_async_result_complete (pending->result);
+ g_simple_async_result_complete_in_idle (pending->result);
}
}
diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h
index b12e116..f7f7c99 100644
--- a/src/libostree/ostree-repo-private.h
+++ b/src/libostree/ostree-repo-private.h
@@ -139,5 +139,14 @@ _ostree_repo_commit_modifier_apply (OstreeRepo *self,
GFileInfo *file_info,
GFileInfo **out_modified_info);
+gboolean
+_ostree_repo_gpg_verify_file_with_metadata (OstreeRepo *self,
+ GFile *path,
+ GVariant *metadata,
+ GFile *keyringdir,
+ GFile *extra_keyring,
+ GCancellable *cancellable,
+ GError **error);
+
G_END_DECLS
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index e3e5a77..d8c0038 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -60,8 +60,11 @@ typedef struct {
guint n_outstanding_metadata_write_requests;
guint n_outstanding_content_fetches;
guint n_outstanding_content_write_requests;
+ guint n_outstanding_deltapart_fetches;
+ guint n_outstanding_deltapart_write_requests;
gint n_requested_metadata;
gint n_requested_content;
+ guint n_fetched_deltaparts;
guint n_fetched_metadata;
guint n_fetched_content;
@@ -79,6 +82,12 @@ typedef struct {
gboolean is_detached_meta;
} FetchObjectData;
+typedef struct {
+ OtPullData *pull_data;
+ GVariant *objects;
+ char *expected_checksum;
+} FetchStaticDeltaData;
+
static SoupURI *
suburi_new (SoupURI *base,
const char *first,
@@ -136,9 +145,11 @@ update_progress (gpointer user_data)
{
OtPullData *pull_data = user_data;
guint outstanding_writes = pull_data->n_outstanding_content_write_requests +
- pull_data->n_outstanding_metadata_write_requests;
+ pull_data->n_outstanding_metadata_write_requests +
+ pull_data->n_outstanding_deltapart_write_requests;
guint outstanding_fetches = pull_data->n_outstanding_content_fetches +
- pull_data->n_outstanding_metadata_fetches;
+ pull_data->n_outstanding_metadata_fetches +
+ pull_data->n_outstanding_deltapart_fetches;
guint64 bytes_transferred = ostree_fetcher_bytes_transferred (pull_data->fetcher);
guint fetched = pull_data->n_fetched_metadata + pull_data->n_fetched_content;
guint requested = pull_data->n_requested_metadata + pull_data->n_requested_content;
@@ -189,9 +200,11 @@ check_outstanding_requests_handle_error (OtPullData *pull_data,
GError *error)
{
gboolean current_fetch_idle = (pull_data->n_outstanding_metadata_fetches == 0 &&
- pull_data->n_outstanding_content_fetches == 0);
+ pull_data->n_outstanding_content_fetches == 0 &&
+ pull_data->n_outstanding_deltapart_fetches == 0);
gboolean current_write_idle = (pull_data->n_outstanding_metadata_write_requests == 0 &&
- pull_data->n_outstanding_content_write_requests == 0);
+ pull_data->n_outstanding_content_write_requests == 0 &&
+ pull_data->n_outstanding_deltapart_write_requests == 0 );
gboolean current_idle = current_fetch_idle && current_write_idle;
throw_async_error (pull_data, error);
@@ -690,7 +703,7 @@ meta_fetch_on_complete (GObject *object,
g_assert (pull_data->n_outstanding_metadata_fetches > 0);
pull_data->n_outstanding_metadata_fetches--;
pull_data->n_fetched_metadata++;
- throw_async_error (pull_data, local_error);
+ check_outstanding_requests_handle_error (pull_data, local_error);
if (local_error)
{
g_variant_unref (fetch_data->object);
@@ -698,6 +711,104 @@ meta_fetch_on_complete (GObject *object,
}
}
+static void
+fetch_static_delta_data_free (gpointer data)
+{
+ FetchStaticDeltaData *fetch_data = data;
+ g_free (fetch_data->expected_checksum);
+ g_variant_unref (fetch_data->objects);
+ g_free (fetch_data);
+}
+
+static void
+on_static_delta_written (GObject *object,
+ GAsyncResult *result,
+ gpointer user_data)
+{
+ FetchStaticDeltaData *fetch_data = user_data;
+ OtPullData *pull_data = fetch_data->pull_data;
+ GError *local_error = NULL;
+ GError **error = &local_error;
+
+ g_debug ("execute static delta part %s complete", fetch_data->expected_checksum);
+
+ if (!_ostree_static_delta_part_execute_finish (pull_data->repo, result, error))
+ goto out;
+
+ out:
+ g_assert (pull_data->n_outstanding_deltapart_write_requests > 0);
+ pull_data->n_outstanding_deltapart_write_requests--;
+ check_outstanding_requests_handle_error (pull_data, local_error);
+ /* Always free state */
+ fetch_static_delta_data_free (fetch_data);
+}
+
+static void
+static_deltapart_fetch_on_complete (GObject *object,
+ GAsyncResult *result,
+ gpointer user_data)
+{
+ FetchStaticDeltaData *fetch_data = user_data;
+ OtPullData *pull_data = fetch_data->pull_data;
+ gs_unref_variant GVariant *metadata = NULL;
+ gs_unref_object GFile *temp_path = NULL;
+ gs_unref_object GInputStream *in = NULL;
+ gs_free char *actual_checksum = NULL;
+ gs_free guint8 *csum = NULL;
+ GError *local_error = NULL;
+ GError **error = &local_error;
+
+ g_debug ("fetch static delta part %s complete", fetch_data->expected_checksum);
+
+ temp_path = ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)object, result, error);
+ if (!temp_path)
+ goto out;
+
+ in = (GInputStream*)g_file_read (temp_path, pull_data->cancellable, error);
+ if (!in)
+ goto out;
+
+ /* TODO - consider making async */
+ if (!ot_gio_checksum_stream (in, &csum, pull_data->cancellable, error))
+ goto out;
+
+ actual_checksum = ostree_checksum_from_bytes (csum);
+
+ if (strcmp (actual_checksum, fetch_data->expected_checksum) != 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Corrupted static delta part; checksum expected='%s' actual='%s'",
+ fetch_data->expected_checksum, actual_checksum);
+ goto out;
+ }
+
+ /* Might as well close the fd here */
+ (void) g_input_stream_close (in, NULL, NULL);
+
+ {
+ gs_unref_bytes GBytes *delta_data
+ = gs_file_map_readonly (temp_path, pull_data->cancellable, error);
+ if (!delta_data)
+ goto out;
+
+ _ostree_static_delta_part_execute_async (pull_data->repo,
+ fetch_data->objects,
+ delta_data,
+ pull_data->cancellable,
+ on_static_delta_written,
+ fetch_data);
+ pull_data->n_outstanding_deltapart_write_requests++;
+ }
+
+ out:
+ g_assert (pull_data->n_outstanding_deltapart_fetches > 0);
+ pull_data->n_outstanding_deltapart_fetches--;
+ pull_data->n_fetched_deltaparts++;
+ check_outstanding_requests_handle_error (pull_data, local_error);
+ if (local_error)
+ fetch_static_delta_data_free (fetch_data);
+}
+
static gboolean
scan_commit_object (OtPullData *pull_data,
const char *checksum,
@@ -957,58 +1068,246 @@ load_remote_repo_config (OtPullData *pull_data,
return ret;
}
-#if 0
static gboolean
-request_static_delta_meta_sync (OtPullData *pull_data,
- const char *ref,
- const char *checksum,
- GVariant **out_delta_meta,
- GCancellable *cancellable,
- GError **error)
+fetch_metadata_to_verify_delta_superblock (OtPullData *pull_data,
+ const char *from_revision,
+ const char *checksum,
+ GBytes *superblock_data,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
- gs_free char *from_revision = NULL;
+ gs_free char *meta_path = _ostree_get_relative_static_delta_detachedmeta_path (from_revision, checksum);
+ gs_unref_bytes GBytes *detached_meta_data = NULL;
SoupURI *target_uri = NULL;
- gs_unref_variant GVariant *ret_delta_meta = NULL;
+ gs_unref_object GFile *temp_input_path = NULL;
+ gs_unref_object GOutputStream *temp_input_stream = NULL;
+ gs_unref_object GInputStream *superblock_in = NULL;
+ gs_unref_variant GVariant *metadata = NULL;
- if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE, &from_revision, error))
- goto out;
+ target_uri = suburi_new (pull_data->base_uri, meta_path, NULL);
- if (from_revision == NULL)
+ if (!fetch_uri_contents_membuf_sync (pull_data, target_uri, FALSE, FALSE,
+ &detached_meta_data,
+ pull_data->cancellable, error))
{
- initiate_commit_scan (pull_data, checksum);
+ g_prefix_error (error, "GPG verification enabled, but failed to fetch metadata: ");
+ goto out;
}
- else
- {
- gs_free char *delta_name = _ostree_get_relative_static_delta_path (from_revision, checksum);
- gs_unref_bytes GBytes *delta_meta_data = NULL;
- gs_unref_variant GVariant *delta_meta = NULL;
- target_uri = suburi_new (pull_data->base_uri, delta_name, NULL);
+ superblock_in = g_memory_input_stream_new_from_bytes (superblock_data);
- if (!fetch_uri_contents_membuf_sync (pull_data, target_uri, FALSE, TRUE,
- &delta_meta_data,
- pull_data->cancellable, error))
- goto out;
+ if (!gs_file_open_in_tmpdir (pull_data->repo->tmp_dir, 0644,
+ &temp_input_path, &temp_input_stream,
+ cancellable, error))
+ goto out;
+
+ if (0 > g_output_stream_splice (temp_input_stream, superblock_in,
+ G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE |
+ G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
+ cancellable, error))
+ goto out;
+
+ metadata = ot_variant_new_from_bytes (G_VARIANT_TYPE ("a{sv}"),
+ detached_meta_data,
+ FALSE);
- if (delta_meta_data)
+ if (!_ostree_repo_gpg_verify_file_with_metadata (pull_data->repo, temp_input_path,
+ metadata, NULL, NULL,
+ cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+static gboolean
+request_static_delta_superblock_sync (OtPullData *pull_data,
+ const char *from_revision,
+ const char *to_revision,
+ GVariant **out_delta_superblock,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_unref_variant GVariant *ret_delta_superblock = NULL;
+ gs_free char *delta_name = _ostree_get_relative_static_delta_path (from_revision, to_revision);
+ gs_unref_bytes GBytes *delta_superblock_data = NULL;
+ gs_unref_bytes GBytes *delta_meta_data = NULL;
+ gs_unref_variant GVariant *delta_superblock = NULL;
+ SoupURI *target_uri = NULL;
+
+ target_uri = suburi_new (pull_data->base_uri, delta_name, NULL);
+
+ if (!fetch_uri_contents_membuf_sync (pull_data, target_uri, FALSE, TRUE,
+ &delta_superblock_data,
+ pull_data->cancellable, error))
+ goto out;
+
+ if (delta_superblock_data)
+ {
+ if (pull_data->gpg_verify)
{
- ret_delta_meta = ot_variant_new_from_bytes ((GVariantType*)OSTREE_STATIC_DELTA_META_FORMAT,
- delta_meta_data, FALSE);
+ if (!fetch_metadata_to_verify_delta_superblock (pull_data,
+ from_revision,
+ to_revision,
+ delta_superblock_data,
+ pull_data->cancellable, error))
+ goto out;
}
+
+ ret_delta_superblock = ot_variant_new_from_bytes ((GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT,
+ delta_superblock_data, FALSE);
}
ret = TRUE;
- gs_transfer_out_value (out_delta_meta, &ret_delta_meta);
+ gs_transfer_out_value (out_delta_superblock, &ret_delta_superblock);
out:
return ret;
}
-#endif
-static void
-process_one_static_delta_meta (OtPullData *pull_data,
- GVariant *delta_meta)
+static gboolean
+process_one_static_delta_fallback (OtPullData *pull_data,
+ GVariant *fallback_object,
+ GCancellable *cancellable,
+ GError **error)
{
+ gboolean ret = FALSE;
+ gs_unref_variant GVariant *csum_v = NULL;
+ gs_free char *checksum = NULL;
+ guint8 objtype_y;
+ OstreeObjectType objtype;
+ gboolean is_stored;
+ guint64 compressed_size, uncompressed_size;
+
+ g_variant_get (fallback_object, "(y aytt)",
+ &objtype_y, &csum_v, &compressed_size, &uncompressed_size);
+ if (!ostree_validate_structureof_objtype (objtype_y, error))
+ goto out;
+ if (!ostree_validate_structureof_csum_v (csum_v, error))
+ goto out;
+
+ objtype = (OstreeObjectType)objtype_y;
+ checksum = ostree_checksum_from_bytes_v (csum_v);
+
+ if (!ostree_repo_has_object (pull_data->repo, objtype, checksum,
+ &is_stored,
+ cancellable, error))
+ goto out;
+
+ if (!is_stored)
+ {
+ if (OSTREE_OBJECT_TYPE_IS_META (objtype))
+ {
+ if (!g_hash_table_lookup (pull_data->requested_metadata, checksum))
+ {
+ gboolean do_fetch_detached;
+ g_hash_table_insert (pull_data->requested_metadata, checksum, checksum);
+
+ do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT);
+ enqueue_one_object_request (pull_data, checksum, objtype, do_fetch_detached);
+ checksum = NULL; /* Transfer ownership */
+ }
+ }
+ else
+ {
+ if (!g_hash_table_lookup (pull_data->requested_content, checksum))
+ {
+ g_hash_table_insert (pull_data->requested_content, checksum, checksum);
+ enqueue_one_object_request (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, FALSE);
+ checksum = NULL; /* Transfer ownership */
+ }
+ }
+ }
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+static gboolean
+process_one_static_delta (OtPullData *pull_data,
+ const char *from_revision,
+ const char *to_revision,
+ GVariant *delta_superblock,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_unref_variant GVariant *headers = NULL;
+ gs_unref_variant GVariant *fallback_objects = NULL;
+ guint i, n;
+
+ /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
+ headers = g_variant_get_child_value (delta_superblock, 3);
+ fallback_objects = g_variant_get_child_value (delta_superblock, 4);
+
+ /* First process the fallbacks */
+ n = g_variant_n_children (fallback_objects);
+ for (i = 0; i < n; i++)
+ {
+ gs_unref_variant GVariant *fallback_object =
+ g_variant_get_child_value (fallback_objects, i);
+
+ if (!process_one_static_delta_fallback (pull_data,
+ fallback_object,
+ cancellable, error))
+ goto out;
+ }
+
+ n = g_variant_n_children (headers);
+ for (i = 0; i < n; i++)
+ {
+ const guchar *csum;
+ gs_unref_variant GVariant *header = NULL;
+ gboolean have_all = FALSE;
+ SoupURI *target_uri = NULL;
+ gs_free char *deltapart_path = NULL;
+ FetchStaticDeltaData *fetch_data;
+ gs_unref_variant GVariant *csum_v = NULL;
+ gs_unref_variant GVariant *objects = NULL;
+ guint64 size, usize;
+
+ header = g_variant_get_child_value (headers, i);
+ g_variant_get (header, "(@aytt ay)", &csum_v, &size, &usize, &objects);
+
+ csum = ostree_checksum_bytes_peek_validate (csum_v, error);
+ if (!csum)
+ goto out;
+
+ if (!_ostree_repo_static_delta_part_have_all_objects (pull_data->repo,
+ objects,
+ &have_all,
+ cancellable, error))
+ goto out;
+
+ if (have_all)
+ {
+ g_debug ("Have all objects from static delta %s-%s part %u",
+ from_revision, to_revision,
+ i);
+ continue;
+ }
+
+ fetch_data = g_new0 (FetchStaticDeltaData, 1);
+ fetch_data->pull_data = pull_data;
+ fetch_data->objects = g_variant_ref (objects);
+ fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v);
+
+ deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i);
+
+ target_uri = suburi_new (pull_data->base_uri, deltapart_path, NULL);
+ ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, target_uri, pull_data->cancellable,
+ static_deltapart_fetch_on_complete,
+ fetch_data);
+ pull_data->n_outstanding_deltapart_fetches++;
+ soup_uri_free (target_uri);
+ }
+
+ ret = TRUE;
+ out:
+ return ret;
}
gboolean
@@ -1025,7 +1324,6 @@ ostree_repo_pull (OstreeRepo *self,
gpointer key, value;
gboolean tls_permissive = FALSE;
OstreeFetcherConfigFlags fetcher_flags = 0;
- guint i;
gs_free char *remote_key = NULL;
gs_free char *path = NULL;
gs_free char *baseurl = NULL;
@@ -1220,15 +1518,37 @@ ostree_repo_pull (OstreeRepo *self,
g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
- const char *checksum = value;
- if (!scan_one_metadata_object (pull_data, checksum, OSTREE_OBJECT_TYPE_COMMIT,
- 0, pull_data->cancellable, error))
+ gs_free char *from_revision = NULL;
+ const char *ref = key;
+ const char *to_revision = value;
+ GVariant *delta_superblock = NULL;
+
+ if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE,
+ &from_revision, error))
goto out;
- }
- for (i = 0; i < pull_data->static_delta_metas->len; i++)
- {
- process_one_static_delta_meta (pull_data, pull_data->static_delta_metas->pdata[i]);
+ if (from_revision && g_strcmp0 (from_revision, to_revision) != 0)
+ {
+ if (!request_static_delta_superblock_sync (pull_data, from_revision, to_revision,
+ &delta_superblock, cancellable, error))
+ goto out;
+ }
+
+ if (!delta_superblock)
+ {
+ g_debug ("no delta superblock for %s-%s", from_revision, to_revision);
+ if (!scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT,
+ 0, pull_data->cancellable, error))
+ goto out;
+ }
+ else
+ {
+ g_debug ("processing delta superblock for %s-%s", from_revision, to_revision);
+ if (!process_one_static_delta (pull_data, from_revision, to_revision,
+ delta_superblock,
+ cancellable, error))
+ goto out;
+ }
}
/* Now await work completion */
@@ -1259,7 +1579,6 @@ ostree_repo_pull (OstreeRepo *self,
else
{
ostree_repo_transaction_set_ref (pull_data->repo, pull_data->remote_name, ref, checksum);
-
}
}
@@ -1272,21 +1591,36 @@ ostree_repo_pull (OstreeRepo *self,
if (bytes_transferred > 0 && pull_data->progress)
{
guint shift;
- gs_free char *msg = NULL;
+ GString *buf = g_string_new ("");
+ gboolean join = FALSE;
if (bytes_transferred < 1024)
shift = 1;
else
shift = 1024;
- msg = g_strdup_printf ("%u metadata, %u content objects fetched; %" G_GUINT64_FORMAT " %s transferred
in %u seconds",
- pull_data->n_fetched_metadata, pull_data->n_fetched_content,
- (guint64)(bytes_transferred / shift),
- shift == 1 ? "B" : "KiB",
- (guint) ((end_time - start_time) / G_USEC_PER_SEC));
- ostree_async_progress_set_status (pull_data->progress, msg);
+ if (pull_data->n_fetched_metadata > 0 ||
+ pull_data->n_fetched_content > 0)
+ g_string_append_printf (buf, "%s%u metadata, %u content objects fetched",
+ join ? "; " : "",
+ pull_data->n_fetched_metadata, pull_data->n_fetched_content), join = TRUE;
+ if (pull_data->n_fetched_deltaparts)
+ g_string_append_printf (buf, "%s%u delta parts fetched",
+ join ? "; " : "",
+ pull_data->n_fetched_deltaparts), join = TRUE;
+
+ g_string_append_printf (buf, "%s%" G_GUINT64_FORMAT " %s transferred in %u seconds",
+ join ? "; " : "",
+ (guint64)(bytes_transferred / shift),
+ shift == 1 ? "B" : "KiB",
+ (guint) ((end_time - start_time) / G_USEC_PER_SEC));
+ ostree_async_progress_set_status (pull_data->progress, buf->str);
+ g_string_free (buf, TRUE);
}
+ if (pull_data->progress)
+ ostree_async_progress_finish (pull_data->progress);
+
ret = TRUE;
out:
if (pull_data->main_context)
diff --git a/src/libostree/ostree-repo-static-delta-compilation.c
b/src/libostree/ostree-repo-static-delta-compilation.c
index 722b9bb..0ebaa34 100644
--- a/src/libostree/ostree-repo-static-delta-compilation.c
+++ b/src/libostree/ostree-repo-static-delta-compilation.c
@@ -38,6 +38,8 @@ typedef struct {
typedef struct {
GPtrArray *parts;
+ GPtrArray *fallback_objects;
+ guint64 loose_compressed_size;
} OstreeStaticDeltaBuilder;
static void
@@ -148,6 +150,8 @@ generate_delta_lowlatency (OstreeRepo *repo,
g_hash_table_insert (new_reachable_objects, g_variant_ref (serialized_key), serialized_key);
}
+
+ g_printerr ("%u new reachable objects\n", g_hash_table_size (new_reachable_objects));
current_part = allocate_part (builder);
@@ -162,6 +166,7 @@ generate_delta_lowlatency (OstreeRepo *repo,
gs_unref_object GInputStream *content_stream = NULL;
gsize bytes_read;
const guint readlen = 4096;
+ guint64 compressed_size;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
@@ -170,7 +175,18 @@ generate_delta_lowlatency (OstreeRepo *repo,
cancellable, error))
goto out;
- current_part->uncompressed_size += content_size;
+ /* Fall back to plain HTTP-based fetch for large objects;
+ * in the future we should try an rsync-style rolling checksum
+ * against a previous version, if any.
+ */
+ if (content_size > OSTREE_STATIC_DELTA_PART_MAX_SIZE_BYTES)
+ {
+ g_printerr ("fallback for %s\n",
+ ostree_object_to_string (checksum, objtype));
+ g_ptr_array_add (builder->fallback_objects,
+ ostree_object_name_serialize (checksum, objtype));
+ continue;
+ }
/* Ensure we have at least one object per delta, even if a given
* object is larger.
@@ -181,6 +197,14 @@ generate_delta_lowlatency (OstreeRepo *repo,
current_part = allocate_part (builder);
}
+ if (!ostree_repo_query_object_storage_size (repo, objtype, checksum,
+ &compressed_size,
+ cancellable, error))
+ goto out;
+ builder->loose_compressed_size += compressed_size;
+
+ current_part->uncompressed_size += content_size;
+
g_ptr_array_add (current_part->objects, g_variant_ref (serialized_key));
object_payload_start = current_part->payload->len;
@@ -213,7 +237,6 @@ generate_delta_lowlatency (OstreeRepo *repo,
g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_WRITE);
_ostree_write_varuint64 (current_part->operations, object_payload_start);
_ostree_write_varuint64 (current_part->operations, content_size);
- g_printerr ("write %" G_GUINT64_FORMAT " %" G_GUINT64_FORMAT "\n", (guint64) object_payload_start,
(guint64)(content_size));
g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_CLOSE);
}
@@ -222,6 +245,71 @@ generate_delta_lowlatency (OstreeRepo *repo,
return ret;
}
+static gboolean
+get_fallback_headers (OstreeRepo *self,
+ OstreeStaticDeltaBuilder *builder,
+ GVariant **out_headers,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ guint i;
+ gs_unref_variant GVariant *ret_headers = NULL;
+ gs_unref_variant_builder GVariantBuilder *fallback_builder = NULL;
+
+ fallback_builder = g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT));
+
+ for (i = 0; i < builder->fallback_objects->len; i++)
+ {
+ GVariant *serialized = builder->fallback_objects->pdata[i];
+ const char *checksum;
+ OstreeObjectType objtype;
+ guint64 compressed_size;
+ guint64 uncompressed_size;
+
+ ostree_object_name_deserialize (serialized, &checksum, &objtype);
+
+ if (OSTREE_OBJECT_TYPE_IS_META (objtype))
+ {
+ if (!ostree_repo_load_object_stream (self, objtype, checksum,
+ NULL, &uncompressed_size,
+ cancellable, error))
+ goto out;
+ compressed_size = uncompressed_size;
+ }
+ else
+ {
+ gs_unref_object GFileInfo *file_info = NULL;
+
+ if (!ostree_repo_query_object_storage_size (self, OSTREE_OBJECT_TYPE_FILE,
+ checksum,
+ &compressed_size,
+ cancellable, error))
+ goto out;
+
+ if (!ostree_repo_load_file (self, checksum,
+ NULL, &file_info, NULL,
+ cancellable, error))
+ goto out;
+
+ uncompressed_size = g_file_info_get_size (file_info);
+ }
+
+ g_variant_builder_add_value (fallback_builder,
+ g_variant_new ("(y aytt)",
+ objtype,
+ ostree_checksum_to_bytes_v (checksum),
+ compressed_size, uncompressed_size));
+ }
+
+ ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder));
+
+ ret = TRUE;
+ gs_transfer_out_value (out_headers, &ret_headers);
+ out:
+ return ret;
+}
+
/**
* ostree_repo_static_delta_generate:
* @self: Repo
@@ -250,6 +338,8 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
OstreeStaticDeltaBuilder builder = { 0, };
guint i;
GVariant *metadata_source;
+ guint64 total_compressed_size = 0;
+ guint64 total_uncompressed_size = 0;
gs_unref_variant_builder GVariantBuilder *part_headers = NULL;
gs_unref_ptrarray GPtrArray *part_tempfiles = NULL;
gs_unref_variant GVariant *delta_descriptor = NULL;
@@ -257,8 +347,10 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
gs_unref_object GFile *descriptor_path = NULL;
gs_unref_object GFile *descriptor_dir = NULL;
gs_unref_variant GVariant *tmp_metadata = NULL;
+ gs_unref_variant GVariant *fallback_headers = NULL;
builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref);
+ builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
/* Ignore optimization flags */
if (!generate_delta_lowlatency (self, from, to, &builder,
@@ -333,6 +425,14 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
ot_gvariant_new_ay_bytes (objtype_checksum_array));
g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header));
g_ptr_array_add (part_tempfiles, g_object_ref (part_tempfile));
+
+ total_compressed_size += g_variant_get_size (delta_part);
+ total_uncompressed_size += part_builder->uncompressed_size;
+
+ g_printerr ("part %u n:%u compressed:%" G_GUINT64_FORMAT " uncompressed:%" G_GUINT64_FORMAT "\n",
+ i, part_builder->objects->len,
+ g_variant_get_size (delta_part),
+ part_builder->uncompressed_size);
}
descriptor_relpath = _ostree_get_relative_static_delta_path (from, to);
@@ -365,21 +465,32 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
metadata_source = tmp_metadata;
}
+ if (!get_fallback_headers (self, &builder, &fallback_headers,
+ cancellable, error))
+ goto out;
+
{
GDateTime *now = g_date_time_new_now_utc ();
- delta_descriptor = g_variant_new ("(@(a(ss)a(say))taya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT ")",
+ delta_descriptor = g_variant_new ("(@(a(ss)a(say))tay"
+ "a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT
+ "@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")",
metadata_source,
GUINT64_TO_BE (g_date_time_to_unix (now)),
g_variant_builder_new (G_VARIANT_TYPE ("ay")),
- part_headers);
+ part_headers,
+ fallback_headers);
g_date_time_unref (now);
}
+ g_printerr ("delta compressed=%" G_GUINT64_FORMAT " loose=%" G_GUINT64_FORMAT "\n",
+ total_compressed_size, builder.loose_compressed_size);
+
if (!ot_util_variant_save (descriptor_path, delta_descriptor, cancellable, error))
goto out;
ret = TRUE;
out:
g_clear_pointer (&builder.parts, g_ptr_array_unref);
+ g_clear_pointer (&builder.fallback_objects, g_ptr_array_unref);
return ret;
}
diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c
index dad5fdb..7eb211b 100644
--- a/src/libostree/ostree-repo-static-delta-core.c
+++ b/src/libostree/ostree-repo-static-delta-core.c
@@ -99,7 +99,7 @@ ostree_repo_list_static_delta_names (OstreeRepo *self,
name = gs_file_get_basename_cached (child);
{
- gs_unref_object GFile *meta_path = g_file_get_child (child, "meta");
+ gs_unref_object GFile *meta_path = g_file_get_child (child, "superblock");
if (g_file_query_exists (meta_path, NULL))
{
@@ -115,12 +115,12 @@ ostree_repo_list_static_delta_names (OstreeRepo *self,
return ret;
}
-static gboolean
-have_all_objects (OstreeRepo *repo,
- GVariant *checksum_array,
- gboolean *out_have_all,
- GCancellable *cancellable,
- GError **error)
+gboolean
+_ostree_repo_static_delta_part_have_all_objects (OstreeRepo *repo,
+ GVariant *checksum_array,
+ gboolean *out_have_all,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
guint8 *checksums_data;
@@ -160,31 +160,6 @@ have_all_objects (OstreeRepo *repo,
return ret;
}
-static gboolean
-zlib_uncompress_data (GBytes *data,
- GBytes **out_uncompressed,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- gs_unref_object GMemoryInputStream *memin = (GMemoryInputStream*)g_memory_input_stream_new_from_bytes
(data);
- gs_unref_object GMemoryOutputStream *memout = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0,
g_realloc, g_free);
- gs_unref_object GConverter *zlib_decomp =
- (GConverter*) g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW);
- gs_unref_object GInputStream *convin = g_converter_input_stream_new ((GInputStream*)memin, zlib_decomp);
-
- if (0 > g_output_stream_splice ((GOutputStream*)memout, convin,
- G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE |
- G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
- cancellable, error))
- goto out;
-
- ret = TRUE;
- *out_uncompressed = g_memory_output_stream_steal_as_bytes (memout);
- out:
- return ret;
-}
-
/**
* ostree_repo_static_delta_execute_offline:
* @self: Repo
@@ -196,7 +171,7 @@ zlib_uncompress_data (GBytes *data,
* Given a directory representing an already-downloaded static delta
* on disk, apply it, generating a new commit. The directory must be
* named with the form "FROM-TO", where both are checksums, and it
- * must contain a file named "meta", along with at least one part.
+ * must contain a file named "superblock", along with at least one part.
*/
gboolean
ostree_repo_static_delta_execute_offline (OstreeRepo *self,
@@ -207,14 +182,23 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
{
gboolean ret = FALSE;
guint i, n;
- gs_unref_object GFile *meta_file = g_file_get_child (dir, "meta");
+ gs_unref_object GFile *meta_file = g_file_get_child (dir, "superblock");
gs_unref_variant GVariant *meta = NULL;
gs_unref_variant GVariant *headers = NULL;
+ gs_unref_variant GVariant *fallback = NULL;
- if (!ot_util_variant_map (meta_file, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_META_FORMAT),
+ if (!ot_util_variant_map (meta_file, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT),
FALSE, &meta, error))
goto out;
+ fallback = g_variant_get_child_value (meta, 4);
+ if (g_variant_n_children (fallback) > 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Cannot execute delta offline: contains nonempty http fallback entries");
+ goto out;
+ }
+
headers = g_variant_get_child_value (meta, 3);
n = g_variant_n_children (headers);
for (i = 0; i < n; i++)
@@ -227,14 +211,14 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
gs_unref_variant GVariant *csum_v = NULL;
gs_unref_variant GVariant *objects = NULL;
gs_unref_object GFile *part_path = NULL;
- gs_unref_variant GVariant *part = NULL;
gs_unref_object GInputStream *raw_in = NULL;
gs_unref_object GInputStream *in = NULL;
header = g_variant_get_child_value (headers, i);
g_variant_get (header, "(@aytt ay)", &csum_v, &size, &usize, &objects);
- if (!have_all_objects (self, objects, &have_all, cancellable, error))
+ if (!_ostree_repo_static_delta_part_have_all_objects (self, objects, &have_all,
+ cancellable, error))
goto out;
/* If we already have these objects, don't bother executing the
@@ -255,70 +239,25 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
if (!skip_validation)
{
- gs_unref_object GInputStream *tmp_in = NULL;
- gs_free guchar *actual_checksum = NULL;
-
- tmp_in = (GInputStream*)g_file_read (part_path, cancellable, error);
- if (!tmp_in)
+ gs_free char *expected_checksum = ostree_checksum_from_bytes (csum);
+ if (!_ostree_static_delta_part_validate (self, part_path, i,
+ expected_checksum,
+ cancellable, error))
goto out;
-
- if (!ot_gio_checksum_stream (tmp_in, &actual_checksum,
- cancellable, error))
- goto out;
-
- if (ostree_cmp_checksum_bytes (csum, actual_checksum) != 0)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Checksum mismatch in static delta %s part %u",
- gs_file_get_path_cached (dir), i);
- goto out;
- }
}
{
GMappedFile *mfile = gs_file_map_noatime (part_path, cancellable, error);
gs_unref_bytes GBytes *bytes = NULL;
- gs_unref_bytes GBytes *payload = NULL;
- gsize partlen;
- const guint8*partdata;
if (!mfile)
goto out;
bytes = g_mapped_file_get_bytes (mfile);
g_mapped_file_unref (mfile);
-
- partdata = g_bytes_get_data (bytes, &partlen);
-
- if (partlen < 1)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Corrupted 0 length byte part %s/%i",
- gs_file_get_basename_cached (dir),
- i);
- goto out;
- }
-
- switch (partdata[0])
- {
- case 0:
- payload = g_bytes_new_from_bytes (bytes, 1, partlen - 1);
- break;
- case 'g':
- {
- gs_unref_bytes GBytes *subbytes = g_bytes_new_from_bytes (bytes, 1, partlen - 1);
- if (!zlib_uncompress_data (subbytes, &payload,
- cancellable, error))
- goto out;
- }
- break;
- }
-
- part = ot_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT),
- payload, FALSE);
-
- if (!_ostree_static_delta_part_execute (self, objects, part, cancellable, error))
+ if (!_ostree_static_delta_part_execute (self, objects, bytes,
+ cancellable, error))
{
g_prefix_error (error, "executing delta part %i: ", i);
goto out;
diff --git a/src/libostree/ostree-repo-static-delta-private.h
b/src/libostree/ostree-repo-static-delta-private.h
index ec9e841..3329b3e 100644
--- a/src/libostree/ostree-repo-static-delta-private.h
+++ b/src/libostree/ostree-repo-static-delta-private.h
@@ -32,6 +32,8 @@ G_BEGIN_DECLS
/**
* OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT:
*
+ * y compression type (0: none, 'z': zlib)
+ * ---
* ay data source
* ay operations
*/
@@ -51,8 +53,21 @@ G_BEGIN_DECLS
#define OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "(ayttay)"
+
/**
- * OSTREE_STATIC_DELTA_META_FORMAT:
+ * OSTREE_STATIC_DELTA_FALLBACK_FORMAT:
+ *
+ * y: objtype
+ * ay: checksum
+ * t: compressed size
+ * t: uncompressed size
+ *
+ * Object to fetch invididually; includes compressed/uncompressed size.
+ */
+#define OSTREE_STATIC_DELTA_FALLBACK_FORMAT "(yaytt)"
+
+/**
+ * OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT:
*
* A .delta object is a custom binary format. It has the following high
* level form:
@@ -62,6 +77,7 @@ G_BEGIN_DECLS
* timestamp: guint64
* ARRAY[(csum from, csum to)]: ay
* ARRAY[delta-meta-entry]
+ * array[fallback]
*
* The metadata would include things like a version number, as well as
* extended verification data like a GPG signature.
@@ -70,22 +86,51 @@ G_BEGIN_DECLS
* fetched and applied before this one. This is a fairly generic
* recursion mechanism that would potentially allow saving significant
* storage space on the server.
+ *
+ * The heart of the static delta: the array of delta parts.
+ *
+ * Finally, we have the fallback array, which is the set of objects to
+ * fetch individually - the compiler determined it wasn't worth
+ * duplicating the space.
*/
-#define OSTREE_STATIC_DELTA_META_FORMAT "(a{sv}taya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT ")"
+#define OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT "(a{sv}taya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "a"
OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")"
+
+gboolean _ostree_static_delta_part_validate (OstreeRepo *repo,
+ GFile *part_path,
+ guint part_offset,
+ const char *expected_checksum,
+ GCancellable *cancellable,
+ GError **error);
gboolean _ostree_static_delta_part_execute (OstreeRepo *repo,
GVariant *header,
- GVariant *part,
+ GBytes *partdata,
GCancellable *cancellable,
GError **error);
+gboolean _ostree_static_delta_part_execute_raw (OstreeRepo *repo,
+ GVariant *header,
+ GVariant *part,
+ GCancellable *cancellable,
+ GError **error);
+
+void _ostree_static_delta_part_execute_async (OstreeRepo *repo,
+ GVariant *header,
+ GBytes *partdata,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data);
+
+gboolean _ostree_static_delta_part_execute_finish (OstreeRepo *repo,
+ GAsyncResult *result,
+ GError **error);
+
typedef enum {
- OSTREE_STATIC_DELTA_OP_FETCH = 1,
- OSTREE_STATIC_DELTA_OP_WRITE = 2,
- OSTREE_STATIC_DELTA_OP_GUNZIP = 3,
- OSTREE_STATIC_DELTA_OP_CLOSE = 4,
- OSTREE_STATIC_DELTA_OP_READOBJECT = 5,
- OSTREE_STATIC_DELTA_OP_READPAYLOAD = 6
+ OSTREE_STATIC_DELTA_OP_WRITE = 1,
+ OSTREE_STATIC_DELTA_OP_GUNZIP = 2,
+ OSTREE_STATIC_DELTA_OP_CLOSE = 3,
+ OSTREE_STATIC_DELTA_OP_READOBJECT = 4,
+ OSTREE_STATIC_DELTA_OP_READPAYLOAD = 5
} OstreeStaticDeltaOpCode;
gboolean
@@ -93,5 +138,12 @@ _ostree_static_delta_parse_checksum_array (GVariant *array,
guint8 **out_checksums_array,
guint *out_n_checksums,
GError **error);
+
+gboolean
+_ostree_repo_static_delta_part_have_all_objects (OstreeRepo *repo,
+ GVariant *checksum_array,
+ gboolean *out_have_all,
+ GCancellable *cancellable,
+ GError **error);
G_END_DECLS
diff --git a/src/libostree/ostree-repo-static-delta-processing.c
b/src/libostree/ostree-repo-static-delta-processing.c
index a373f00..01dbd1b 100644
--- a/src/libostree/ostree-repo-static-delta-processing.c
+++ b/src/libostree/ostree-repo-static-delta-processing.c
@@ -64,14 +64,12 @@ typedef struct {
GCancellable *cancellable, \
GError **error);
-OPPROTO(fetch)
OPPROTO(write)
OPPROTO(gunzip)
OPPROTO(close)
#undef OPPROTO
static OstreeStaticDeltaOperation op_dispatch_table[] = {
- { "fetch", dispatch_fetch },
{ "write", dispatch_write },
{ "gunzip", dispatch_gunzip },
{ "close", dispatch_close },
@@ -110,13 +108,47 @@ open_output_target_csum (OstreeRepo *repo,
return ret;
}
+gboolean
+_ostree_static_delta_part_validate (OstreeRepo *repo,
+ GFile *part_path,
+ guint part_offset,
+ const char *expected_checksum,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_unref_object GInputStream *tmp_in = NULL;
+ gs_free guchar *actual_checksum_bytes = NULL;
+ gs_free gchar *actual_checksum = NULL;
+
+ tmp_in = (GInputStream*)g_file_read (part_path, cancellable, error);
+ if (!tmp_in)
+ goto out;
+
+ if (!ot_gio_checksum_stream (tmp_in, &actual_checksum_bytes,
+ cancellable, error))
+ goto out;
+
+ actual_checksum = ostree_checksum_from_bytes (actual_checksum_bytes);
+ if (strcmp (actual_checksum, expected_checksum) != 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Checksum mismatch in static delta part %u; expected=%s actual=%s",
+ part_offset, expected_checksum, actual_checksum);
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ return ret;
+}
gboolean
-_ostree_static_delta_part_execute (OstreeRepo *repo,
- GVariant *objects,
- GVariant *part,
- GCancellable *cancellable,
- GError **error)
+_ostree_static_delta_part_execute_raw (OstreeRepo *repo,
+ GVariant *objects,
+ GVariant *part,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
guint8 *checksums_data;
@@ -172,14 +204,162 @@ _ostree_static_delta_part_execute (OstreeRepo *repo,
}
static gboolean
-dispatch_fetch (OstreeRepo *repo,
- StaticDeltaExecutionState *state,
- GCancellable *cancellable,
- GError **error)
+zlib_uncompress_data (GBytes *data,
+ GBytes **out_uncompressed,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_unref_object GMemoryInputStream *memin = (GMemoryInputStream*)g_memory_input_stream_new_from_bytes
(data);
+ gs_unref_object GMemoryOutputStream *memout = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0,
g_realloc, g_free);
+ gs_unref_object GConverter *zlib_decomp =
+ (GConverter*) g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW);
+ gs_unref_object GInputStream *convin = g_converter_input_stream_new ((GInputStream*)memin, zlib_decomp);
+
+ if (0 > g_output_stream_splice ((GOutputStream*)memout, convin,
+ G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE |
+ G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
+ cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ *out_uncompressed = g_memory_output_stream_steal_as_bytes (memout);
+ out:
+ return ret;
+}
+
+gboolean
+_ostree_static_delta_part_execute (OstreeRepo *repo,
+ GVariant *header,
+ GBytes *part_bytes,
+ GCancellable *cancellable,
+ GError **error)
{
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
- "Static delta fetch opcode is not implemented in this version");
- return FALSE;
+ gboolean ret = FALSE;
+ gsize partlen;
+ const guint8*partdata;
+ gs_unref_bytes GBytes *part_payload_bytes = NULL;
+ gs_unref_bytes GBytes *payload_data = NULL;
+ gs_unref_variant GVariant *payload = NULL;
+ guint8 comptype;
+
+ partdata = g_bytes_get_data (part_bytes, &partlen);
+
+ if (partlen < 1)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Corrupted 0 length delta part");
+ goto out;
+ }
+
+ /* First byte is compression type */
+ comptype = partdata[0];
+ /* Then the rest may be compressed or uncompressed */
+ part_payload_bytes = g_bytes_new_from_bytes (part_bytes, 1, partlen - 1);
+ switch (comptype)
+ {
+ case 0:
+ /* No compression */
+ payload_data = g_bytes_ref (part_payload_bytes);
+ break;
+ case 'g':
+ {
+ if (!zlib_uncompress_data (part_payload_bytes, &payload_data,
+ cancellable, error))
+ goto out;
+ }
+ break;
+ default:
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid compression type '%u'", comptype);
+ goto out;
+ }
+
+ payload = ot_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT),
+ payload_data, FALSE);
+ if (!_ostree_static_delta_part_execute_raw (repo, header, payload,
+ cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+typedef struct {
+ OstreeRepo *repo;
+ GVariant *header;
+ GBytes *partdata;
+ GCancellable *cancellable;
+ GSimpleAsyncResult *result;
+} StaticDeltaPartExecuteAsyncData;
+
+static void
+static_delta_part_execute_async_data_free (gpointer user_data)
+{
+ StaticDeltaPartExecuteAsyncData *data = user_data;
+
+ g_clear_object (&data->repo);
+ g_variant_unref (data->header);
+ g_bytes_unref (data->partdata);
+ g_clear_object (&data->cancellable);
+ g_free (data);
+}
+
+static void
+static_delta_part_execute_thread (GSimpleAsyncResult *res,
+ GObject *object,
+ GCancellable *cancellable)
+{
+ GError *error = NULL;
+ StaticDeltaPartExecuteAsyncData *data;
+
+ data = g_simple_async_result_get_op_res_gpointer (res);
+ if (!_ostree_static_delta_part_execute (data->repo,
+ data->header,
+ data->partdata,
+ cancellable, &error))
+ g_simple_async_result_take_error (res, error);
+}
+
+void
+_ostree_static_delta_part_execute_async (OstreeRepo *repo,
+ GVariant *header,
+ GBytes *partdata,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data)
+{
+ StaticDeltaPartExecuteAsyncData *asyncdata;
+
+ asyncdata = g_new0 (StaticDeltaPartExecuteAsyncData, 1);
+ asyncdata->repo = g_object_ref (repo);
+ asyncdata->header = g_variant_ref (header);
+ asyncdata->partdata = g_bytes_ref (partdata);
+ asyncdata->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
+
+ asyncdata->result = g_simple_async_result_new ((GObject*) repo,
+ callback, user_data,
+ _ostree_static_delta_part_execute_async);
+
+ g_simple_async_result_set_op_res_gpointer (asyncdata->result, asyncdata,
+ static_delta_part_execute_async_data_free);
+ g_simple_async_result_run_in_thread (asyncdata->result, static_delta_part_execute_thread,
G_PRIORITY_DEFAULT, cancellable);
+ g_object_unref (asyncdata->result);
+}
+
+gboolean
+_ostree_static_delta_part_execute_finish (OstreeRepo *repo,
+ GAsyncResult *result,
+ GError **error)
+{
+ GSimpleAsyncResult *simple = G_SIMPLE_ASYNC_RESULT (result);
+
+ g_warn_if_fail (g_simple_async_result_get_source_tag (simple) == _ostree_static_delta_part_execute_async);
+
+ if (g_simple_async_result_propagate_error (simple, error))
+ return FALSE;
+ return TRUE;
}
static gboolean
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index f859540..5653287 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -1518,9 +1518,8 @@ ostree_repo_append_gpg_signature (OstreeRepo *self,
{
gboolean ret = FALSE;
gs_unref_variant GVariant *metadata = NULL;
+ gs_unref_variant GVariant *new_metadata = NULL;
gs_unref_variant_builder GVariantBuilder *builder = NULL;
- gs_unref_variant_builder GVariantBuilder *signature_builder = NULL;
- gs_unref_variant GVariant *signaturedata = NULL;
if (!ostree_repo_read_commit_detached_metadata (self,
commit_checksum,
@@ -1533,27 +1532,11 @@ ostree_repo_append_gpg_signature (OstreeRepo *self,
goto out;
}
- if (metadata)
- {
- builder = ot_util_variant_builder_from_variant (metadata, G_VARIANT_TYPE ("a{sv}"));
- signaturedata = g_variant_lookup_value (metadata, "ostree.gpgsigs", G_VARIANT_TYPE ("aay"));
- if (signaturedata)
- signature_builder = ot_util_variant_builder_from_variant (signaturedata, G_VARIANT_TYPE ("aay"));
- }
- if (!builder)
- builder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"));
- if (!signature_builder)
- signature_builder = g_variant_builder_new (G_VARIANT_TYPE ("aay"));
-
- g_variant_builder_add (signature_builder, "@ay", ot_gvariant_new_ay_bytes (signature_bytes));
-
- g_variant_builder_add (builder, "{sv}", "ostree.gpgsigs", g_variant_builder_end (signature_builder));
-
- metadata = g_variant_builder_end (builder);
+ new_metadata = _ostree_detached_metadata_append_gpg_sig (metadata, signature_bytes);
if (!ostree_repo_write_commit_detached_metadata (self,
commit_checksum,
- metadata,
+ new_metadata,
cancellable,
error))
{
@@ -1567,33 +1550,20 @@ ostree_repo_append_gpg_signature (OstreeRepo *self,
return ret;
}
-/**
- * ostree_repo_sign_commit:
- * @self: Self
- * @commit_checksum: SHA256 of given commit to sign
- * @key_id: Use this GPG key id
- * @homedir: (allow-none): GPG home directory, or %NULL
- * @cancellable: A #GCancellable
- * @error: a #GError
- *
- * Add a GPG signature to a commit.
- */
-gboolean
-ostree_repo_sign_commit (OstreeRepo *self,
- const gchar *commit_checksum,
- const gchar *key_id,
- const gchar *homedir,
- GCancellable *cancellable,
- GError **error)
+static gboolean
+sign_data (OstreeRepo *self,
+ GBytes *input_data,
+ const gchar *key_id,
+ const gchar *homedir,
+ GBytes **out_signature,
+ GCancellable *cancellable,
+ GError **error)
{
#ifdef HAVE_GPGME
gboolean ret = FALSE;
- gs_unref_object GFile *commit_path = NULL;
- gs_free gchar *commit_filename = NULL;
gs_unref_object GFile *tmp_signature_file = NULL;
gs_unref_object GOutputStream *tmp_signature_output = NULL;
- gs_unref_variant GVariant *commit_variant = NULL;
- gs_unref_bytes GBytes *signature_bytes = NULL;
+ gs_unref_bytes GBytes *ret_signature = NULL;
gpgme_ctx_t context;
gpgme_engine_info_t info;
gpgme_error_t err;
@@ -1603,10 +1573,6 @@ ostree_repo_sign_commit (OstreeRepo *self,
int signature_fd = -1;
GMappedFile *signature_file = NULL;
- if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT,
- commit_checksum, &commit_variant, error))
- goto out;
-
if (!gs_file_open_in_tmpdir (self->tmp_dir, 0644,
&tmp_signature_file, &tmp_signature_output,
cancellable, error))
@@ -1661,13 +1627,16 @@ ostree_repo_sign_commit (OstreeRepo *self,
goto out;
}
- if ((err = gpgme_data_new_from_mem (&commit_buffer, g_variant_get_data (commit_variant),
- g_variant_get_size (commit_variant), FALSE)) != GPG_ERR_NO_ERROR)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Failed to create buffer from commit file");
- goto out;
- }
+ {
+ gsize len;
+ const char *buf = g_bytes_get_data (input_data, &len);
+ if ((err = gpgme_data_new_from_mem (&commit_buffer, buf, len, FALSE)) != GPG_ERR_NO_ERROR)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Failed to create buffer from commit file");
+ goto out;
+ }
+ }
signature_fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)tmp_signature_output);
if (signature_fd < 0)
@@ -1698,13 +1667,10 @@ ostree_repo_sign_commit (OstreeRepo *self,
signature_file = gs_file_map_noatime (tmp_signature_file, cancellable, error);
if (!signature_file)
goto out;
- signature_bytes = g_mapped_file_get_bytes (signature_file);
+ ret_signature = g_mapped_file_get_bytes (signature_file);
- if (!ostree_repo_append_gpg_signature (self, commit_checksum, signature_bytes,
- cancellable, error))
- goto out;
-
ret = TRUE;
+ gs_transfer_out_value (out_signature, &ret_signature);
out:
if (commit_buffer)
gpgme_data_release (commit_buffer);
@@ -1724,7 +1690,134 @@ out:
#endif
}
-static gboolean
+/**
+ * ostree_repo_sign_commit:
+ * @self: Self
+ * @commit_checksum: SHA256 of given commit to sign
+ * @key_id: Use this GPG key id
+ * @homedir: (allow-none): GPG home directory, or %NULL
+ * @cancellable: A #GCancellable
+ * @error: a #GError
+ *
+ * Add a GPG signature to a commit.
+ */
+gboolean
+ostree_repo_sign_commit (OstreeRepo *self,
+ const gchar *commit_checksum,
+ const gchar *key_id,
+ const gchar *homedir,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_unref_bytes GBytes *commit_data = NULL;
+ gs_unref_bytes GBytes *signature_data = NULL;
+ gs_unref_variant GVariant *commit_variant = NULL;
+
+ if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT,
+ commit_checksum, &commit_variant, error))
+ goto out;
+
+ /* This has the same lifecycle as the variant, so we can just
+ * use static.
+ */
+ signature_data = g_bytes_new_static (g_variant_get_data (commit_variant),
+ g_variant_get_size (commit_variant));
+
+ if (!sign_data (self, signature_data, key_id, homedir,
+ &signature_data,
+ cancellable, error))
+ goto out;
+
+ if (!ostree_repo_append_gpg_signature (self, commit_checksum, signature_data,
+ cancellable, error))
+ goto out;
+
+ ret = TRUE;
+out:
+ return ret;
+}
+
+/**
+ * ostree_repo_sign_delta:
+ * @self: Self
+ * @from_commit: SHA256 of starting commit to sign
+ * @to_commit: SHA256 of target commit to sign
+ * @key_id: Use this GPG key id
+ * @homedir: (allow-none): GPG home directory, or %NULL
+ * @cancellable: A #GCancellable
+ * @error: a #GError
+ *
+ * Add a GPG signature to a static delta.
+ */
+gboolean
+ostree_repo_sign_delta (OstreeRepo *self,
+ const gchar *from_commit,
+ const gchar *to_commit,
+ const gchar *key_id,
+ const gchar *homedir,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_unref_bytes GBytes *delta_data = NULL;
+ gs_unref_bytes GBytes *signature_data = NULL;
+ gs_unref_variant GVariant *commit_variant = NULL;
+ gs_free char *delta_path = NULL;
+ gs_unref_object GFile *delta_file = NULL;
+ gs_unref_object char *detached_metadata_relpath = NULL;
+ gs_unref_object GFile *detached_metadata_path = NULL;
+ gs_unref_variant GVariant *existing_detached_metadata = NULL;
+ gs_unref_variant GVariant *normalized = NULL;
+ gs_unref_variant GVariant *new_metadata = NULL;
+ GError *temp_error = NULL;
+
+ detached_metadata_relpath =
+ _ostree_get_relative_static_delta_detachedmeta_path (from_commit, to_commit);
+ detached_metadata_path = g_file_resolve_relative_path (self->repodir, detached_metadata_relpath);
+
+ delta_path = _ostree_get_relative_static_delta_path (from_commit, to_commit);
+ delta_file = g_file_resolve_relative_path (self->repodir, delta_path);
+ delta_data = gs_file_map_readonly (delta_file, cancellable, error);
+ if (!delta_data)
+ goto out;
+
+ if (!ot_util_variant_map (detached_metadata_path, G_VARIANT_TYPE ("a{sv}"),
+ TRUE, &existing_detached_metadata, &temp_error))
+ {
+ if (g_error_matches (temp_error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+ {
+ g_clear_error (&temp_error);
+ }
+ else
+ {
+ g_propagate_error (error, temp_error);
+ goto out;
+ }
+ }
+
+ if (!sign_data (self, delta_data, key_id, homedir,
+ &signature_data,
+ cancellable, error))
+ goto out;
+
+ new_metadata = _ostree_detached_metadata_append_gpg_sig (existing_detached_metadata, signature_data);
+
+ normalized = g_variant_get_normal_form (new_metadata);
+
+ if (!g_file_replace_contents (detached_metadata_path,
+ g_variant_get_data (normalized),
+ g_variant_get_size (normalized),
+ NULL, FALSE, 0, NULL,
+ cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+gboolean
_ostree_repo_gpg_verify_file_with_metadata (OstreeRepo *self,
GFile *path,
GVariant *metadata,
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index f522fce..26aa6ae 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -521,6 +521,14 @@ gboolean ostree_repo_sign_commit (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
+gboolean ostree_repo_sign_delta (OstreeRepo *self,
+ const gchar *from_commit,
+ const gchar *to_commit,
+ const gchar *key_id,
+ const gchar *homedir,
+ GCancellable *cancellable,
+ GError **error);
+
gboolean ostree_repo_append_gpg_signature (OstreeRepo *self,
const gchar *commit_checksum,
GBytes *signature_bytes,
diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c
index be535a6..34305a4 100644
--- a/src/ostree/ot-builtin-static-delta.c
+++ b/src/ostree/ot-builtin-static-delta.c
@@ -27,11 +27,15 @@
static char *opt_from_rev;
static char *opt_to_rev;
static char *opt_apply;
+static char **opt_key_ids;
+static char *opt_gpg_homedir;
static GOptionEntry options[] = {
{ "from", 0, 0, G_OPTION_ARG_STRING, &opt_from_rev, "Create delta from revision REV", "REV" },
{ "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Create delta to revision REV", "REV" },
{ "apply", 0, 0, G_OPTION_ARG_FILENAME, &opt_apply, "Apply delta from PATH", "PATH" },
+ { "gpg-sign", 0, 0, G_OPTION_ARG_STRING_ARRAY, &opt_key_ids, "GPG Key ID to sign the delta with",
"key-id"},
+ { "gpg-homedir", 0, 0, G_OPTION_ARG_STRING, &opt_gpg_homedir, "GPG Homedir to use when looking for
keyrings", "homedir"},
{ NULL }
};
@@ -113,6 +117,24 @@ ostree_builtin_static_delta (int argc, char **argv, OstreeRepo *repo, GCancellab
from_resolved, to_resolved, NULL,
cancellable, error))
goto out;
+
+ if (opt_key_ids)
+ {
+ char **iter;
+
+ for (iter = opt_key_ids; iter && *iter; iter++)
+ {
+ const char *keyid = *iter;
+
+ if (!ostree_repo_sign_delta (repo,
+ from_resolved, to_resolved,
+ keyid,
+ opt_gpg_homedir,
+ cancellable,
+ error))
+ goto out;
+ }
+ }
}
else
{
diff --git a/tests/pull-test.sh b/tests/pull-test.sh
index 8231cef..3829e4c 100755
--- a/tests/pull-test.sh
+++ b/tests/pull-test.sh
@@ -19,21 +19,31 @@
set -e
-cd ${test_tmpdir}
-mkdir repo
-${CMD_PREFIX} ostree --repo=repo init
-${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat
httpd-address)/ostree/gnomerepo
+function repo_init() {
+ cd ${test_tmpdir}
+ rm repo -rf
+ mkdir repo
+ ${CMD_PREFIX} ostree --repo=repo init
+ ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat
httpd-address)/ostree/gnomerepo
+}
+
+function verify_initial_contents() {
+ rm checkout-origin-main -rf
+ $OSTREE checkout origin/main checkout-origin-main
+ cd checkout-origin-main
+ assert_file_has_content firstfile '^first$'
+ assert_file_has_content baz/cow '^moo$'
+}
+
# Try both syntaxes
+repo_init
${CMD_PREFIX} ostree --repo=repo pull origin main
${CMD_PREFIX} ostree --repo=repo pull origin:main
${CMD_PREFIX} ostree --repo=repo fsck
echo "ok pull"
cd ${test_tmpdir}
-$OSTREE checkout origin/main checkout-origin-main
-cd checkout-origin-main
-assert_file_has_content firstfile '^first$'
-assert_file_has_content baz/cow '^moo$'
+verify_initial_contents
echo "ok pull contents"
cd ${test_tmpdir}
@@ -43,3 +53,63 @@ ${CMD_PREFIX} ostree --repo=repo fsck
$OSTREE show --print-detached-metadata-key=SIGNATURE main > main-meta
assert_file_has_content main-meta "HANCOCK"
echo "ok pull detached metadata"
+
+cd ${test_tmpdir}
+repo_init
+${CMD_PREFIX} ostree --repo=repo pull origin main
+${CMD_PREFIX} ostree --repo=repo fsck
+# Generate a delta from old to current, even though we aren't going to
+# use it.
+ostree --repo=ostree-srv/gnomerepo static-delta main
+
+rm main-files -rf
+ostree --repo=ostree-srv/gnomerepo checkout main main-files
+cd main-files
+echo "an added file for static deltas" > added-file
+echo "modified file for static deltas" > baz/cow
+rm baz/saucer
+ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main -s 'static delta test'
+cd ..
+rm main-files -rf
+# Generate delta that we'll use
+ostree --repo=ostree-srv/gnomerepo static-delta main
+
+cd ${test_tmpdir}
+${CMD_PREFIX} ostree --repo=repo pull origin main
+${CMD_PREFIX} ostree --repo=repo fsck
+
+rm checkout-origin-main -rf
+$OSTREE checkout origin:main checkout-origin-main
+cd checkout-origin-main
+assert_file_has_content firstfile '^first$'
+assert_file_has_content baz/cow "modified file for static deltas"
+assert_not_has_file baz/saucer
+
+echo "ok static delta"
+
+cd ${test_tmpdir}
+rm main-files -rf
+ostree --repo=ostree-srv/gnomerepo checkout main main-files
+cd main-files
+# Make a file larger than 16M for testing
+dd if=/dev/zero of=test-bigfile count=1 seek=42678
+echo "further modified file for static deltas" > baz/cow
+ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main -s '2nd static delta test'
+cd ..
+rm main-files -rf
+ostree --repo=ostree-srv/gnomerepo static-delta main
+
+cd ${test_tmpdir}
+${CMD_PREFIX} ostree --repo=repo pull origin main
+${CMD_PREFIX} ostree --repo=repo fsck
+
+rm checkout-origin-main -rf
+$OSTREE checkout origin:main checkout-origin-main
+cd checkout-origin-main
+assert_has_file test-bigfile
+stat --format=%s test-bigfile > bigfile-size
+assert_file_has_content bigfile-size 21851648
+assert_file_has_content baz/cow "further modified file for static deltas"
+assert_not_has_file baz/saucer
+
+echo "ok static delta 2"
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
