[ostree/wip/thin-commits] pull: Implement a metadata-only pull mode
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree/wip/thin-commits] pull: Implement a metadata-only pull mode
- Date: Mon, 4 Nov 2013 01:14:14 +0000 (UTC)
commit 574c82303fc29ffa4323987da00b62f969c579c1
Author: Vivek Dasmohapatra <vivek collabora co uk>
Date: Wed Aug 28 14:41:46 2013 +0100
pull: Implement a metadata-only pull mode
This just fetches the top level commit object. The intention is to
allow this minimal pre-fetch to be used to determine the size of an
upcoming pull in advance. Invoking pull again without the
METADATA_ONLY flag fetches the content.
Prior to metadata-only pulls, having the top level object was assumed
to mean you had the whole tree - this is not the case anymore, so we
explicitly store the commits named as "committhin".
src/libostree/ostree-repo-commit.c | 116 ++++++++++++++++++++++++++++++++---
src/libostree/ostree-repo-private.h | 1 +
src/libostree/ostree-repo-pull.c | 67 +++++++++++++++------
src/libostree/ostree-repo.c | 60 +++++++++++++++++-
src/libostree/ostree-repo.h | 27 ++++++++-
src/ostree/ot-builtin-pull-local.c | 44 ++++++++++---
src/ostree/ot-builtin-pull.c | 6 ++
7 files changed, 278 insertions(+), 43 deletions(-)
---
diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c
index 9c39938..e652f38 100644
--- a/src/libostree/ostree-repo-commit.c
+++ b/src/libostree/ostree-repo-commit.c
@@ -300,6 +300,7 @@ write_object (OstreeRepo *self,
const char *expected_checksum,
GInputStream *input,
guint64 file_object_length,
+ gboolean is_thin_commit,
guchar **out_csum,
GCancellable *cancellable,
GError **error)
@@ -491,6 +492,17 @@ write_object (OstreeRepo *self,
if (do_commit)
{
+ /* Override the loose path here for thin commits; we name them
+ * as "committhin", even though we checked for just "commit"
+ * above. This is mildly lame in that attempting to store thin
+ * commits will rewrite the existing object, but it doesn't
+ * really matter. Trying to do better would result in even
+ * *more* conditionals in this already heavily conditional
+ * function.
+ */
+ if (objtype == OSTREE_OBJECT_TYPE_COMMIT && is_thin_commit)
+ _ostree_loose_path_with_suffix (loose_objpath, actual_checksum, objtype, self->mode, "thin");
+
if (!commit_loose_object_trusted (self, objtype, loose_objpath,
temp_file, temp_filename,
is_symlink, file_info,
@@ -977,7 +989,7 @@ ostree_repo_write_metadata (OstreeRepo *self,
normalized = g_variant_get_normal_form (object);
input = ot_variant_read (normalized);
- return write_object (self, objtype, expected_checksum, input, 0, out_csum,
+ return write_object (self, objtype, expected_checksum, input, 0, FALSE, out_csum,
cancellable, error);
}
@@ -1004,7 +1016,7 @@ ostree_repo_write_metadata_stream_trusted (OstreeRepo *self,
GError **error)
{
/* Ignore provided length for now */
- return write_object (self, objtype, checksum, object_input, 0, NULL,
+ return write_object (self, objtype, checksum, object_input, 0, FALSE, NULL,
cancellable, error);
}
@@ -1034,13 +1046,48 @@ ostree_repo_write_metadata_trusted (OstreeRepo *self,
normalized = g_variant_get_normal_form (variant);
input = ot_variant_read (normalized);
- return write_object (self, type, checksum, input, 0, NULL,
+ return write_object (self, type, checksum, input, 0, FALSE, NULL,
+ cancellable, error);
+}
+
+/**
+ * ostree_repo_write_commit_trusted:
+ * @self: Repo
+ * @checksum: Checksum for new commit object
+ * @variant: Commit object
+ * @is_thin_commit: If %TRUE, the commit is "thin"; i.e. other metadata/content is not stored
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Store the commit object @variant; the provided @checksum is
+ * trusted.
+ *
+ * Set @is_thin_commit if the other metadata and content objects
+ * referenced by @object will not also be stored.
+ */
+gboolean
+ostree_repo_write_commit_trusted (OstreeRepo *self,
+ const char *checksum,
+ GVariant *variant,
+ gboolean is_thin_commit,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gs_unref_object GInputStream *input = NULL;
+ gs_unref_variant GVariant *normalized = NULL;
+
+ normalized = g_variant_get_normal_form (variant);
+ input = ot_variant_read (normalized);
+
+ return write_object (self, OSTREE_OBJECT_TYPE_COMMIT, checksum, input,
+ 0, is_thin_commit, NULL,
cancellable, error);
}
typedef struct {
OstreeRepo *repo;
OstreeObjectType objtype;
+ gboolean is_thin_commit;
char *expected_checksum;
GVariant *object;
GCancellable *cancellable;
@@ -1069,12 +1116,17 @@ write_metadata_thread (GSimpleAsyncResult *res,
{
GError *error = NULL;
WriteMetadataAsyncData *data;
+ gs_unref_object GInputStream *input = NULL;
+ gs_unref_variant GVariant *normalized = NULL;
data = g_simple_async_result_get_op_res_gpointer (res);
- if (!ostree_repo_write_metadata (data->repo, data->objtype, data->expected_checksum,
- data->object,
- &data->result_csum,
- cancellable, &error))
+
+ normalized = g_variant_get_normal_form (data->object);
+ input = ot_variant_read (normalized);
+
+ if (!write_object (data->repo, data->objtype, data->expected_checksum,
+ input, 0, data->is_thin_commit, &data->result_csum,
+ cancellable, &error))
g_simple_async_result_take_error (res, error);
}
@@ -1140,6 +1192,52 @@ ostree_repo_write_metadata_finish (OstreeRepo *self,
return TRUE;
}
+/**
+ * ostree_repo_write_commit_async:
+ * @self: Repo
+ * @expected_checksum: (allow-none): If provided, validate content against this checksum
+ * @object: Commit object
+ * @is_thin_commit: If %TRUE, the commit is "thin"; i.e. other metadata/content is not stored
+ * @cancellable: Cancellable
+ * @callback: Invoked when metadata is writed
+ * @user_data: Data for @callback
+ *
+ * Asynchronously store the commit object @object. If provided, the
+ * checksum @expected_checksum will be verified.
+ *
+ * Set @is_thin_commit if the other metadata and content objects
+ * referenced by @object will not also be stored.
+ */
+void
+ostree_repo_write_commit_async (OstreeRepo *self,
+ const char *expected_checksum,
+ GVariant *object,
+ gboolean is_thin_commit,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data)
+{
+ WriteMetadataAsyncData *asyncdata;
+
+ asyncdata = g_new0 (WriteMetadataAsyncData, 1);
+ asyncdata->is_thin_commit = is_thin_commit;
+ asyncdata->repo = g_object_ref (self);
+ asyncdata->objtype = OSTREE_OBJECT_TYPE_COMMIT;
+ asyncdata->expected_checksum = g_strdup (expected_checksum);
+ asyncdata->object = g_variant_ref (object);
+ asyncdata->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
+
+ asyncdata->result = g_simple_async_result_new ((GObject*) self,
+ callback, user_data,
+ ostree_repo_write_metadata_async);
+
+ g_simple_async_result_set_op_res_gpointer (asyncdata->result, asyncdata,
+ write_metadata_async_data_free);
+ g_simple_async_result_run_in_thread (asyncdata->result, write_metadata_thread, G_PRIORITY_DEFAULT,
cancellable);
+ g_object_unref (asyncdata->result);
+
+}
+
gboolean
_ostree_repo_write_directory_meta (OstreeRepo *self,
GFileInfo *file_info,
@@ -1201,7 +1299,7 @@ ostree_repo_write_content_trusted (OstreeRepo *self,
GError **error)
{
return write_object (self, OSTREE_OBJECT_TYPE_FILE, checksum,
- object_input, length, NULL,
+ object_input, length, FALSE, NULL,
cancellable, error);
}
@@ -1229,7 +1327,7 @@ ostree_repo_write_content (OstreeRepo *self,
GError **error)
{
return write_object (self, OSTREE_OBJECT_TYPE_FILE, expected_checksum,
- object_input, length, out_csum,
+ object_input, length, FALSE, out_csum,
cancellable, error);
}
diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h
index 0a3b0a0..45f2698 100644
--- a/src/libostree/ostree-repo-private.h
+++ b/src/libostree/ostree-repo-private.h
@@ -137,5 +137,6 @@ _ostree_repo_commit_modifier_apply (OstreeRepo *self,
GFileInfo *file_info,
GFileInfo **out_modified_info);
+
G_END_DECLS
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 271d44d..65d543c 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -693,9 +693,19 @@ meta_fetch_on_complete (GObject *object,
FALSE, &metadata, error))
goto out;
- ostree_repo_write_metadata_async (pull_data->repo, objtype, checksum, metadata,
- pull_data->cancellable,
- on_metadata_writed, fetch_data);
+ if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
+ {
+ gboolean is_thin = (pull_data->flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0;
+ ostree_repo_write_commit_async (pull_data->repo, checksum, metadata, is_thin,
+ pull_data->cancellable,
+ on_metadata_writed, fetch_data);
+ }
+ else
+ {
+ ostree_repo_write_metadata_async (pull_data->repo, objtype, checksum, metadata,
+ pull_data->cancellable,
+ on_metadata_writed, fetch_data);
+ }
pull_data->n_outstanding_metadata_write_requests++;
}
@@ -743,24 +753,28 @@ scan_commit_object (OtPullData *pull_data,
}
#endif
- if (!ostree_repo_load_variant (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT, checksum,
- &commit, error))
+ if (!ostree_repo_load_commit (pull_data->repo, checksum, &commit, NULL,
+ cancellable, error))
goto out;
/* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */
g_variant_get_child (commit, 6, "@ay", &tree_contents_csum);
g_variant_get_child (commit, 7, "@ay", &tree_meta_csum);
- if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (tree_contents_csum),
- OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1,
- cancellable, error))
- goto out;
+ // If this is a commit only pull, don't grab the top dirtree/dirmeta:
+ if (!(pull_data->flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY))
+ {
+ if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (tree_contents_csum),
+ OSTREE_OBJECT_TYPE_DIR_TREE, recursion_depth + 1,
+ cancellable, error))
+ goto out;
+
+ if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (tree_meta_csum),
+ OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1,
+ cancellable, error))
+ goto out;
+ }
- if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (tree_meta_csum),
- OSTREE_OBJECT_TYPE_DIR_META, recursion_depth + 1,
- cancellable, error))
- goto out;
-
ret = TRUE;
out:
if (iter)
@@ -781,6 +795,7 @@ scan_one_metadata_object (OtPullData *pull_data,
gs_free char *tmp_checksum = NULL;
gboolean is_requested;
gboolean is_stored;
+ gboolean is_thin_commit = FALSE;
tmp_checksum = ostree_checksum_from_bytes (csum);
object = ostree_object_name_serialize (tmp_checksum, objtype);
@@ -789,9 +804,19 @@ scan_one_metadata_object (OtPullData *pull_data,
return TRUE;
is_requested = g_hash_table_lookup (pull_data->requested_metadata, tmp_checksum) != NULL;
- if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored,
- cancellable, error))
- goto out;
+ if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
+ {
+ if (!ostree_repo_load_commit (pull_data->repo, tmp_checksum,
+ NULL, &is_thin_commit,
+ cancellable, error))
+ goto out;
+ }
+ else
+ {
+ if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored,
+ cancellable, error))
+ goto out;
+ }
if (!is_stored && !is_requested)
{
@@ -809,7 +834,13 @@ scan_one_metadata_object (OtPullData *pull_data,
}
else if (is_stored)
{
- if (pull_data->transaction_resuming || is_requested)
+ /* Scan if it was already stored,
+ and one of the following:
+ - Resuming an interrupted pull
+ - This object was explicitly requested
+ - The commit object is thin, and we might want to make it non-thin
+ */
+ if (pull_data->transaction_resuming || is_requested || is_thin_commit)
{
switch (objtype)
{
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index eb5bc53..353324f 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -810,6 +810,7 @@ load_metadata_internal (OstreeRepo *self,
const char *sha256,
gboolean error_if_not_found,
GVariant **out_variant,
+ gboolean check_thin,
GInputStream **out_stream,
guint64 *out_size,
GCancellable *cancellable,
@@ -823,7 +824,10 @@ load_metadata_internal (OstreeRepo *self,
g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (objtype), FALSE);
- _ostree_loose_path (loose_path_buf, sha256, objtype, self->mode);
+ if (objtype == OSTREE_OBJECT_TYPE_COMMIT && check_thin)
+ _ostree_loose_path_with_suffix (loose_path_buf, sha256, objtype, self->mode, "thin");
+ else
+ _ostree_loose_path (loose_path_buf, sha256, objtype, self->mode);
if (!openat_allow_noent (self->objects_dir_fd, loose_path_buf, &fd,
cancellable, error))
@@ -1112,7 +1116,7 @@ ostree_repo_load_object_stream (OstreeRepo *self,
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
{
- if (!load_metadata_internal (self, objtype, checksum, TRUE, NULL,
+ if (!load_metadata_internal (self, objtype, checksum, TRUE, NULL, FALSE,
&ret_input, &size,
cancellable, error))
goto out;
@@ -1325,7 +1329,7 @@ ostree_repo_load_variant_if_exists (OstreeRepo *self,
GError **error)
{
return load_metadata_internal (self, objtype, sha256, FALSE,
- out_variant, NULL, NULL, NULL, error);
+ out_variant, FALSE, NULL, NULL, NULL, error);
}
/**
@@ -1347,7 +1351,55 @@ ostree_repo_load_variant (OstreeRepo *self,
GError **error)
{
return load_metadata_internal (self, objtype, sha256, TRUE,
- out_variant, NULL, NULL, NULL, error);
+ out_variant, FALSE, NULL, NULL, NULL, error);
+}
+
+/**
+ * ostree_repo_load_commit:
+ * @self: Repo
+ * @sha256: Checksum string
+ * @out_variant: (out) (transfer full): Metadata object
+ * @out_is_thin: (out): Whether or not the commit is "thin"; i.e. only storing the commit itself
+ * @error: Error
+ *
+ * Load the metadata object @sha256 of type @objtype, storing the
+ * result in @out_variant.
+ */
+gboolean
+ostree_repo_load_commit (OstreeRepo *self,
+ const char *sha256,
+ GVariant **out_variant,
+ gboolean *out_is_thin,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_unref_variant GVariant *ret_variant = NULL;
+
+ if (!load_metadata_internal (self, OSTREE_OBJECT_TYPE_COMMIT, sha256, FALSE,
+ &ret_variant, FALSE, NULL, NULL,
+ cancellable, error))
+ goto out;
+
+ if (ret_variant != NULL)
+ {
+ if (out_is_thin)
+ *out_is_thin = FALSE;
+ }
+ else
+ {
+ if (!load_metadata_internal (self, OSTREE_OBJECT_TYPE_COMMIT, sha256, TRUE,
+ &ret_variant, TRUE, NULL, NULL,
+ cancellable, error))
+ goto out;
+ if (out_is_thin)
+ *out_is_thin = TRUE;
+ }
+
+ ret = TRUE;
+ gs_transfer_out_value (out_variant, &ret_variant);
+ out:
+ return ret;
}
/**
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 6a97f6b..7ced5a1 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -155,6 +155,15 @@ gboolean ostree_repo_write_metadata_finish (OstreeRepo *self,
guchar **out_csum,
GError **error);
+
+void ostree_repo_write_commit_async (OstreeRepo *self,
+ const char *expected_checksum,
+ GVariant *object,
+ gboolean is_thin_commit,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data);
+
gboolean ostree_repo_write_content (OstreeRepo *self,
const char *expected_checksum,
GInputStream *object_input,
@@ -163,6 +172,13 @@ gboolean ostree_repo_write_content (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
+gboolean ostree_repo_write_commit_trusted (OstreeRepo *self,
+ const char *checksum,
+ GVariant *variant,
+ gboolean is_thin_commit,
+ GCancellable *cancellable,
+ GError **error);
+
gboolean ostree_repo_write_metadata_trusted (OstreeRepo *self,
OstreeObjectType objtype,
const char *checksum,
@@ -210,6 +226,13 @@ gboolean ostree_repo_list_refs (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
+gboolean ostree_repo_load_commit (OstreeRepo *self,
+ const char *sha256,
+ GVariant **out_variant,
+ gboolean *out_is_thin,
+ GCancellable *cancellable,
+ GError **error);
+
gboolean ostree_repo_load_variant (OstreeRepo *self,
OstreeObjectType objtype,
const char *sha256,
@@ -453,9 +476,11 @@ gboolean ostree_repo_prune (OstreeRepo *self,
/**
* OstreeRepoPullFlags:
* @OSTREE_REPO_PULL_FLAGS_NONE: No special options for pull
+ * @OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY: Only fetch the commit object
*/
typedef enum {
- OSTREE_REPO_PULL_FLAGS_NONE
+ OSTREE_REPO_PULL_FLAGS_NONE = 0x00,
+ OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY = 0x01,
} OstreeRepoPullFlags;
gboolean ostree_repo_pull (OstreeRepo *self,
diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c
index 0feeddf..504b1f6 100644
--- a/src/ostree/ot-builtin-pull-local.c
+++ b/src/ostree/ot-builtin-pull-local.c
@@ -30,9 +30,11 @@
#include "otutil.h"
static char *opt_remote;
+gboolean opt_commit_only;
static GOptionEntry options[] = {
{ "remote", 0, 0, G_OPTION_ARG_STRING, &opt_remote, "Add REMOTE to refspec", "REMOTE" },
+ { "commit-only", 'm', 0, G_OPTION_ARG_NONE, &opt_commit_only, "Download only the commit", NULL },
{ NULL }
};
@@ -61,16 +63,17 @@ import_one_object (OtLocalCloneData *data,
GError **error)
{
gboolean ret = FALSE;
- guint64 length;
- gs_unref_object GInputStream *object = NULL;
-
- if (!ostree_repo_load_object_stream (data->src_repo, objtype, checksum,
- &object, &length,
- cancellable, error))
- goto out;
if (objtype == OSTREE_OBJECT_TYPE_FILE)
{
+ gs_unref_object GInputStream *object = NULL;
+ guint64 length;
+
+ if (!ostree_repo_load_object_stream (data->src_repo, objtype, checksum,
+ &object, &length,
+ cancellable, error))
+ goto out;
+
if (!ostree_repo_write_content_trusted (data->dest_repo, checksum,
object, length,
cancellable, error))
@@ -78,9 +81,16 @@ import_one_object (OtLocalCloneData *data,
}
else
{
+ gs_unref_variant GVariant *metadata = NULL;
+
if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
{
+ gboolean is_thin = FALSE;
gs_unref_variant GVariant *detached_meta = NULL;
+
+ if (!ostree_repo_load_commit (data->src_repo, checksum, &metadata, &is_thin,
+ cancellable, error))
+ goto out;
if (!ostree_repo_read_commit_detached_metadata (data->src_repo,
checksum, &detached_meta,
@@ -94,11 +104,23 @@ import_one_object (OtLocalCloneData *data,
cancellable, error))
goto out;
}
+
+ if (!ostree_repo_write_commit_trusted (data->dest_repo,
+ checksum, metadata, is_thin,
+ cancellable, error))
+ goto out;
+ }
+ else
+ {
+ if (!ostree_repo_load_variant (data->src_repo, objtype, checksum, &metadata,
+ error))
+ goto out;
+
+ if (!ostree_repo_write_metadata_trusted (data->dest_repo, objtype,
+ checksum, metadata,
+ cancellable, error))
+ goto out;
}
- if (!ostree_repo_write_metadata_stream_trusted (data->dest_repo, objtype,
- checksum, object, length,
- cancellable, error))
- goto out;
}
g_atomic_int_inc (&data->n_objects_copied);
diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c
index 67305fd..99628f4 100644
--- a/src/ostree/ot-builtin-pull.c
+++ b/src/ostree/ot-builtin-pull.c
@@ -27,7 +27,10 @@
#include "ostree.h"
#include "otutil.h"
+gboolean opt_commit_only;
+
static GOptionEntry options[] = {
+ { "commit-only", 'm', 0, G_OPTION_ARG_NONE, &opt_commit_only, "Download only the commit", NULL },
{ NULL }
};
@@ -64,6 +67,9 @@ ostree_builtin_pull (int argc, char **argv, OstreeRepo *repo, GCancellable *canc
g_ptr_array_add (refs_to_fetch, NULL);
}
+ if (opt_commit_only)
+ pullflags |= OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY;
+
console = gs_console_get ();
if (console)
progress = ostree_async_progress_new_and_connect (ot_common_pull_progress, console);
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
