[ostree/wip/async-pull] [wip] Asynchronous metadata fetch
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree/wip/async-pull] [wip] Asynchronous metadata fetch
- Date: Sat, 15 Sep 2012 18:44:16 +0000 (UTC)
commit 83ea6bcdab059d326af381ffdef885c08691d2f9
Author: Colin Walters <walters verbum org>
Date: Tue Aug 28 09:41:09 2012 -0400
[wip] Asynchronous metadata fetch
src/ostree/ostree-pull.c | 673 ++++++++++++++++++++++++++++------------------
1 files changed, 411 insertions(+), 262 deletions(-)
---
diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c
index 0115b8b..74fcff6 100644
--- a/src/ostree/ostree-pull.c
+++ b/src/ostree/ostree-pull.c
@@ -66,6 +66,7 @@
#include "ostree-fetcher.h"
+
gboolean verbose;
gint opt_packfile_threshold = 66;
gboolean opt_related;
@@ -92,10 +93,17 @@ typedef struct {
GHashTable *file_checksums_to_fetch;
GMainLoop *loop;
+ GCancellable *cancellable;
/* Used in meta fetch phase */
guint outstanding_uri_requests;
- guint outstanding_meta_requests;
+
+ GThread *metadata_scan_thread;
+ GAsyncQueue *metadata_objects_to_scan;
+ volatile gint outstanding_meta_requests;
+ GHashTable *scanned_metadata; /* Maps object name to itself */
+
+ GHashTable *commit_to_depth; /* Maps commit checksum to depth we should fetch */
/* Used in content fetch phase */
guint outstanding_filemeta_requests;
@@ -115,6 +123,17 @@ suburi_new (SoupURI *base,
const char *first,
...) G_GNUC_NULL_TERMINATED;
+static gboolean scan_one_metadata_object (OtPullData *pull_data,
+ const guchar *csum,
+ OstreeObjectType objtype,
+ GCancellable *cancellable,
+ GError **error);
+static gboolean scan_one_metadata_object_v_name (OtPullData *pull_data,
+ GVariant *object,
+ GCancellable *cancellable,
+ GError **error);
+
+
static SoupURI *
suburi_new (SoupURI *base,
const char *first,
@@ -188,16 +207,9 @@ uri_fetch_update_status (gpointer user_data)
}
static void
-check_outstanding_requests_handle_error (OtPullData *pull_data,
- GError *error)
+throw_async_error (OtPullData *pull_data,
+ GError *error)
{
- if (pull_data->outstanding_uri_requests == 0 &&
- pull_data->outstanding_meta_requests == 0 &&
- pull_data->outstanding_filemeta_requests == 0 &&
- pull_data->outstanding_filecontent_requests == 0 &&
- pull_data->outstanding_checksum_requests == 0 &&
- (pull_data->loose_files == NULL || g_hash_table_size (pull_data->loose_files) == 0))
- g_main_loop_quit (pull_data->loop);
if (error)
{
if (!pull_data->caught_error)
@@ -214,6 +226,20 @@ check_outstanding_requests_handle_error (OtPullData *pull_data,
}
static void
+check_outstanding_requests_handle_error (OtPullData *pull_data,
+ GError *error)
+{
+ if (pull_data->outstanding_uri_requests == 0 &&
+ pull_data->outstanding_meta_requests == 0 &&
+ pull_data->outstanding_filemeta_requests == 0 &&
+ pull_data->outstanding_filecontent_requests == 0 &&
+ pull_data->outstanding_checksum_requests == 0 &&
+ (pull_data->loose_files == NULL || g_hash_table_size (pull_data->loose_files) == 0))
+ g_main_loop_quit (pull_data->loop);
+ throw_async_error (pull_data, error);
+}
+
+static void
run_mainloop_monitor_fetcher (OtPullData *pull_data)
{
GSource *update_timeout = NULL;
@@ -554,34 +580,6 @@ fetch_and_cache_pack_indexes (OtPullData *pull_data,
}
static gboolean
-fetch_loose_object (OtPullData *pull_data,
- const char *checksum,
- OstreeObjectType objtype,
- GFile **out_temp_path,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lfree char *objpath = NULL;
- ot_lobj GFile *ret_temp_path = NULL;
- SoupURI *obj_uri = NULL;
-
- objpath = ostree_get_relative_object_path (checksum, objtype);
- obj_uri = suburi_new (pull_data->base_uri, objpath, NULL);
-
- if (!fetch_uri (pull_data, obj_uri, ostree_object_type_to_string (objtype), &ret_temp_path,
- cancellable, error))
- goto out;
-
- ret = TRUE;
- ot_transfer_out_value (out_temp_path, &ret_temp_path);
- out:
- if (obj_uri)
- soup_uri_free (obj_uri);
- return ret;
-}
-
-static gboolean
find_object_ensure_indexes (OtPullData *pull_data,
const char *checksum,
OstreeObjectType objtype,
@@ -626,126 +624,11 @@ find_object_ensure_indexes (OtPullData *pull_data,
}
static gboolean
-find_object_ensure_pack_data (OtPullData *pull_data,
- const char *checksum,
- OstreeObjectType objtype,
- gboolean *out_is_stored,
- GFile **out_remote_pack_path,
- guint64 *out_remote_pack_offset,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- gboolean ret_is_stored;
- ot_lfree char *remote_pack_checksum = NULL;
- ot_lobj GFile *ret_remote_pack_path = NULL;
-
- if (!find_object_ensure_indexes (pull_data, checksum, objtype, &ret_is_stored,
- &remote_pack_checksum, out_remote_pack_offset,
- cancellable, error))
- goto out;
-
- if (!ret_is_stored)
- {
- if (remote_pack_checksum)
- {
- if (!fetch_one_pack_file (pull_data, remote_pack_checksum, OSTREE_OBJECT_TYPE_IS_META (objtype),
- &ret_remote_pack_path, cancellable, error))
- goto out;
- }
- }
-
- ret = TRUE;
- if (out_is_stored)
- *out_is_stored = ret_is_stored;
- ot_transfer_out_value (out_remote_pack_path, &ret_remote_pack_path);
- /* offset set above */
- out:
- return ret;
-}
-
-static gboolean
-fetch_and_store_metadata (OtPullData *pull_data,
- const char *checksum,
- OstreeObjectType objtype,
- GVariant **out_variant,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- guint64 pack_offset = 0;
- gboolean is_stored;
- ot_lvariant GVariant *ret_variant = NULL;
- ot_lobj GFile *remote_pack_path = NULL;
- ot_lobj GFile *temp_path = NULL;
- ot_lobj GInputStream *input = NULL;
- ot_lvariant GVariant *pack_entry = NULL;
- ot_lvariant GVariant *metadata = NULL;
- GMappedFile *pack_map = NULL;
-
- g_assert (OSTREE_OBJECT_TYPE_IS_META (objtype));
-
- if (!find_object_ensure_pack_data (pull_data, checksum, objtype,
- &is_stored, &remote_pack_path, &pack_offset,
- cancellable, error))
- goto out;
-
- if (remote_pack_path != NULL)
- {
- g_assert (!is_stored);
-
- pack_map = g_mapped_file_new (ot_gfile_get_path_cached (remote_pack_path), FALSE, error);
- if (!pack_map)
- goto out;
-
- if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map),
- g_mapped_file_get_length (pack_map),
- pack_offset, FALSE, TRUE, &pack_entry,
- cancellable, error))
- goto out;
-
- g_variant_get_child (pack_entry, 2, "v", &metadata);
-
- input = ot_variant_read (metadata);
- }
- else if (!is_stored)
- {
- if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error))
- goto out;
-
- input = (GInputStream*)g_file_read (temp_path, cancellable, error);
- if (!input)
- goto out;
- }
-
- if (input)
- {
- g_assert (remote_pack_path != NULL || !is_stored);
- if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, input,
- cancellable, error))
- goto out;
- }
-
- if (!ostree_repo_load_variant (pull_data->repo, objtype, checksum,
- &ret_variant, error))
- goto out;
-
- ret = TRUE;
- ot_transfer_out_value (out_variant, &ret_variant);
- out:
- if (temp_path)
- (void) ot_gfile_unlink (temp_path, NULL, NULL);
- if (pack_map)
- g_mapped_file_unref (pack_map);
- return ret;
-}
-
-static gboolean
-fetch_and_store_tree_metadata_recurse (OtPullData *pull_data,
- int depth,
- const char *rev,
- GCancellable *cancellable,
- GError **error)
+scan_dirtree_object (OtPullData *pull_data,
+ int depth,
+ const char *checksum,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
int i, n;
@@ -762,8 +645,8 @@ fetch_and_store_tree_metadata_recurse (OtPullData *pull_data,
goto out;
}
- if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_DIR_TREE,
- &tree, cancellable, error))
+ if (!ostree_repo_load_variant (pull_data->repo, OSTREE_OBJECT_TYPE_DIR_TREE, checksum,
+ &tree, error))
goto out;
/* PARSE OSTREE_SERIALIZED_TREE_VARIANT */
@@ -802,15 +685,14 @@ fetch_and_store_tree_metadata_recurse (OtPullData *pull_data,
if (!ot_util_filename_validate (dirname, error))
goto out;
- g_free (tmp_checksum);
- tmp_checksum = ostree_checksum_from_bytes_v (meta_csum);
- if (!fetch_and_store_metadata (pull_data, tmp_checksum, OSTREE_OBJECT_TYPE_DIR_META,
- NULL, cancellable, error))
+ if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (tree_csum),
+ OSTREE_OBJECT_TYPE_DIR_TREE,
+ cancellable, error))
goto out;
-
- g_free (tmp_checksum);
- tmp_checksum = ostree_checksum_from_bytes_v (tree_csum);
- if (!fetch_and_store_tree_metadata_recurse (pull_data, depth+1, tmp_checksum, cancellable, error))
+
+ if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (meta_csum),
+ OSTREE_OBJECT_TYPE_DIR_META,
+ cancellable, error))
goto out;
}
@@ -820,94 +702,6 @@ fetch_and_store_tree_metadata_recurse (OtPullData *pull_data,
}
static gboolean
-fetch_and_store_commit_metadata_recurse (OtPullData *pull_data,
- int parent_depth,
- int related_depth,
- const char *rev,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- ot_lvariant GVariant *commit = NULL;
- ot_lvariant GVariant *related_objects = NULL;
- ot_lvariant GVariant *tree_contents_csum = NULL;
- ot_lvariant GVariant *tree_meta_csum = NULL;
- ot_lfree char *tmp_checksum = NULL;
- GVariantIter *iter = NULL;
-
- if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_COMMIT,
- &commit, cancellable, error))
- goto out;
-
- /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */
- g_variant_get_child (commit, 6, "@ay", &tree_contents_csum);
- g_variant_get_child (commit, 7, "@ay", &tree_meta_csum);
-
- g_free (tmp_checksum);
- tmp_checksum = ostree_checksum_from_bytes_v (tree_meta_csum);
- if (!fetch_and_store_metadata (pull_data, tmp_checksum, OSTREE_OBJECT_TYPE_DIR_META,
- NULL, cancellable, error))
- goto out;
-
- g_free (tmp_checksum);
- tmp_checksum = ostree_checksum_from_bytes_v (tree_contents_csum);
- if (!fetch_and_store_tree_metadata_recurse (pull_data, 0, tmp_checksum,
- cancellable, error))
- goto out;
-
- if (opt_related)
- {
- const char *name;
- ot_lvariant GVariant *csum_v = NULL;
-
- if (parent_depth > OSTREE_MAX_RECURSION
- || related_depth > OSTREE_MAX_RECURSION)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Exceeded maximum recursion");
- goto out;
- }
-
- related_objects = g_variant_get_child_value (commit, 2);
- iter = g_variant_iter_new (related_objects);
-
- while (g_variant_iter_loop (iter, "(&s ay)", &name, &csum_v))
- {
- ot_lfree char *checksum = ostree_checksum_from_bytes_v (csum_v);
-
- /* Pass opt_depth here to ensure we aren't fetching parents of related */
- if (!fetch_and_store_commit_metadata_recurse (pull_data, opt_depth,
- related_depth + 1, checksum,
- cancellable, error))
- goto out;
- }
- }
-
- if (parent_depth < opt_depth)
- {
- ot_lvariant GVariant *parent_csum_v = NULL;
-
- parent_csum_v = g_variant_get_child_value (commit, 1);
-
- if (g_variant_n_children (parent_csum_v) > 0)
- {
- ot_lfree char *checksum = ostree_checksum_from_bytes_v (parent_csum_v);
-
- if (!fetch_and_store_commit_metadata_recurse (pull_data, parent_depth + 1,
- 0, checksum,
- cancellable, error))
- goto out;
- }
- }
-
- ret = TRUE;
- out:
- if (iter)
- g_variant_iter_free (iter);
- return ret;
-}
-
-static gboolean
fetch_ref_contents (OtPullData *pull_data,
const char *ref,
char **out_contents,
@@ -1366,6 +1160,343 @@ fetch_content (OtPullData *pull_data,
return ret;
}
+typedef struct {
+ OtPullData *pull_data;
+ GVariant *object;
+} IdleFetchMetadataObjectData;
+
+static void
+meta_fetch_on_complete (GObject *object,
+ GAsyncResult *result,
+ gpointer user_data)
+{
+ IdleFetchMetadataObjectData *fetch_data = user_data;
+ OtPullData *pull_data = fetch_data->pull_data;
+ ot_lobj GFile *temp_path = NULL;
+ ot_lobj GInputStream *input = NULL;
+ const char *checksum;
+ OstreeObjectType objtype;
+ GError *local_error = NULL;
+ GError **error = &local_error;
+
+ temp_path = ostree_fetcher_request_uri_finish ((OstreeFetcher*)object, result, error);
+ if (!temp_path)
+ goto out;
+
+ ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
+
+ input = (GInputStream*)g_file_read (temp_path, pull_data->cancellable, error);
+ if (!input)
+ goto out;
+
+ if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, input,
+ pull_data->cancellable, error))
+ goto out;
+
+ g_async_queue_push (pull_data->metadata_objects_to_scan,
+ g_variant_ref (fetch_data->object));
+ if (g_atomic_int_dec_and_test (&pull_data->outstanding_meta_requests))
+ g_async_queue_push (pull_data->metadata_objects_to_scan,
+ NULL);
+
+ out:
+ (void) ot_gfile_unlink (temp_path, NULL, NULL);
+ throw_async_error (pull_data, local_error);
+ g_variant_unref (fetch_data->object);
+ g_free (fetch_data);
+}
+
+static gboolean
+idle_fetch_metadata_object (gpointer data)
+{
+ IdleFetchMetadataObjectData *fetch_data = data;
+ OtPullData *pull_data = fetch_data->pull_data;
+ ot_lfree char *objpath = NULL;
+ const char *checksum;
+ OstreeObjectType objtype;
+ SoupURI *obj_uri = NULL;
+
+ ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
+
+ objpath = ostree_get_relative_object_path (checksum, objtype);
+ obj_uri = suburi_new (pull_data->base_uri, objpath, NULL);
+
+ ostree_fetcher_request_uri_async (pull_data->fetcher, obj_uri, pull_data->cancellable,
+ meta_fetch_on_complete, fetch_data);
+ soup_uri_free (obj_uri);
+
+ return FALSE;
+}
+
+/**
+ * queue_metadata_object_fetch:
+ *
+ * Pass a request to the main thread to fetch a metadata object.
+ */
+static void
+queue_metadata_object_fetch (OtPullData *pull_data,
+ GVariant *object)
+{
+ IdleFetchMetadataObjectData *fetch_data = g_new (IdleFetchMetadataObjectData, 1);
+ fetch_data->pull_data = pull_data;
+ fetch_data->object = g_variant_ref (object);
+ g_atomic_int_inc (&(fetch_data->pull_data->outstanding_meta_requests));
+ g_idle_add (idle_fetch_metadata_object, fetch_data);
+}
+
+static gboolean
+scan_commit_object (OtPullData *pull_data,
+ guint recursion_depth,
+ const char *checksum,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gboolean depth_found;
+ gpointer depth_ptr;
+ guint avail_depth, traversed;
+ ot_lvariant GVariant *commit = NULL;
+ ot_lvariant GVariant *related_objects = NULL;
+ ot_lvariant GVariant *tree_contents_csum = NULL;
+ ot_lvariant GVariant *tree_meta_csum = NULL;
+ ot_lfree char *tmp_checksum = NULL;
+ GVariantIter *iter = NULL;
+
+ if (recursion_depth > OSTREE_MAX_RECURSION)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Exceeded maximum recursion");
+ goto out;
+ }
+
+ if (!ostree_repo_load_variant (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT, checksum,
+ &commit, error))
+ goto out;
+
+ /* PARSE OSTREE_SERIALIZED_COMMIT_VARIANT */
+ g_variant_get_child (commit, 6, "@ay", &tree_contents_csum);
+ g_variant_get_child (commit, 7, "@ay", &tree_meta_csum);
+
+ if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (tree_contents_csum),
+ OSTREE_OBJECT_TYPE_DIR_TREE,
+ cancellable, error))
+ goto out;
+
+ if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (tree_meta_csum),
+ OSTREE_OBJECT_TYPE_DIR_META,
+ cancellable, error))
+ goto out;
+
+ traversed = 0;
+ while (traversed < opt_depth)
+ {
+ traversed++;
+ depth_found = g_hash_table_lookup_extended (pull_data->commit_to_depth, checksum, NULL,
+ &depth_ptr);
+ if (depth_found)
+ break;
+ }
+ if (depth_found)
+ avail_depth = GPOINTER_TO_UINT (depth_ptr);
+ else
+ avail_depth = opt_depth;
+
+ if (traversed <= avail_depth)
+ {
+ ot_lvariant GVariant *parent_csum_v = NULL;
+
+ parent_csum_v = g_variant_get_child_value (commit, 1);
+
+ if (g_variant_n_children (parent_csum_v) > 0)
+ {
+ g_hash_table_insert (pull_data->commit_to_depth,
+ ostree_checksum_from_bytes_v (parent_csum_v),
+ GUINT_TO_POINTER (avail_depth - traversed));
+ if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (parent_csum_v),
+ OSTREE_OBJECT_TYPE_COMMIT,
+ cancellable, error))
+ goto out;
+ }
+ }
+
+ if (opt_related)
+ {
+ const char *name;
+ ot_lvariant GVariant *csum_v = NULL;
+
+ related_objects = g_variant_get_child_value (commit, 2);
+ iter = g_variant_iter_new (related_objects);
+
+ while (g_variant_iter_loop (iter, "(&s ay)", &name, &csum_v))
+ {
+ g_hash_table_insert (pull_data->commit_to_depth,
+ ostree_checksum_from_bytes_v (csum_v),
+ GUINT_TO_POINTER (avail_depth - traversed));
+ if (!scan_one_metadata_object (pull_data, ostree_checksum_bytes_peek (csum_v),
+ OSTREE_OBJECT_TYPE_COMMIT,
+ cancellable, error))
+ goto out;
+ }
+ }
+
+ ret = TRUE;
+ out:
+ if (iter)
+ g_variant_iter_free (iter);
+ return ret;
+}
+
+static gboolean
+scan_one_metadata_object (OtPullData *pull_data,
+ const guchar *csum,
+ OstreeObjectType objtype,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ ot_lvariant GVariant *object = NULL;
+ ot_lfree char *tmp_checksum = NULL;
+ gboolean is_stored;
+
+ tmp_checksum = ostree_checksum_from_bytes (csum);
+ object = ostree_object_name_serialize (tmp_checksum, objtype);
+
+ if (g_hash_table_lookup (pull_data->scanned_metadata, object))
+ return TRUE;
+
+ if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored,
+ cancellable, error))
+ goto out;
+
+ if (!is_stored)
+ {
+ queue_metadata_object_fetch (pull_data, object);
+ }
+ else
+ {
+ switch (objtype)
+ {
+ case OSTREE_OBJECT_TYPE_COMMIT:
+ if (!scan_commit_object (pull_data, 0, tmp_checksum, pull_data->cancellable,
+ error))
+ goto out;
+ break;
+ case OSTREE_OBJECT_TYPE_DIR_META:
+ break;
+ case OSTREE_OBJECT_TYPE_DIR_TREE:
+ if (!scan_dirtree_object (pull_data, 0, tmp_checksum, pull_data->cancellable,
+ error))
+ goto out;
+ break;
+ case OSTREE_OBJECT_TYPE_FILE:
+ g_assert_not_reached ();
+ break;
+ }
+
+ }
+
+ g_hash_table_insert (pull_data->scanned_metadata, g_variant_ref (object), object);
+
+ ret = TRUE;
+ out:
+ return ret;
+
+}
+
+static gboolean
+scan_one_metadata_object_v_name (OtPullData *pull_data,
+ GVariant *object,
+ GCancellable *cancellable,
+ GError **error)
+{
+ OstreeObjectType objtype;
+ const char *checksum = NULL;
+ ot_lfree guchar *csum = NULL;
+
+ ostree_object_name_deserialize (object, &checksum, &objtype);
+ csum = ostree_checksum_to_bytes (checksum);
+
+ return scan_one_metadata_object (pull_data, csum, objtype,
+ cancellable, error);
+}
+
+typedef struct {
+ OtPullData *pull_data;
+ GError *error;
+} IdleThrowErrorData;
+
+static gboolean
+idle_throw_error (gpointer user_data)
+{
+ IdleThrowErrorData *data = user_data;
+
+ throw_async_error (data->pull_data, data->error);
+
+ g_free (data);
+ return FALSE;
+}
+
+static gboolean
+idle_quit_loop (gpointer user_data)
+{
+ GMainLoop *loop = user_data;
+ g_main_loop_quit (loop);
+ return FALSE;
+}
+
+/**
+ * scan_metadata_thread_main:
+ *
+ * This thread starts from the given commit objects put into the
+ * queue, traversing them.
+ *
+ * If we're missing an object from one of
+ * them, we queue a request to the main thread to fetch it. When it's
+ * fetched, we get passed the object back and scan it.
+ */
+static gpointer
+scan_metadata_thread_main (gpointer user_data)
+{
+ OtPullData *pull_data = user_data;
+ GError *local_error = NULL;
+ GError **error = &local_error;
+
+ pull_data->scanned_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
+ (GDestroyNotify)g_variant_unref, NULL);
+
+ while (TRUE)
+ {
+ ot_lvariant GVariant *item = NULL;
+
+ item = g_async_queue_pop (pull_data->metadata_objects_to_scan);
+
+ if (item == NULL)
+ {
+ if (g_atomic_int_get (&(pull_data->outstanding_meta_requests)) == 0)
+ {
+ item = g_async_queue_try_pop (pull_data->metadata_objects_to_scan);
+ if (!item)
+ break;
+ }
+ }
+
+ if (!scan_one_metadata_object_v_name (pull_data, item, pull_data->cancellable, error))
+ goto out;
+ }
+
+ g_main_context_invoke (NULL, idle_quit_loop, pull_data->loop);
+
+ out:
+ if (local_error)
+ {
+ IdleThrowErrorData *throwdata = g_new0 (IdleThrowErrorData, 1);
+ throwdata->pull_data = pull_data;
+ throwdata->error = local_error;
+ g_main_context_invoke (NULL, idle_throw_error, throwdata);
+ }
+ return NULL;
+}
+
static gboolean
parse_ref_summary (const char *contents,
GHashTable **out_refs,
@@ -1502,6 +1633,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
pull_data->repo = repo;
pull_data->file_checksums_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+ pull_data->commit_to_depth = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+
if (argc < 2)
{
ot_util_usage_error (context, "REMOTE must be specified", error);
@@ -1526,6 +1659,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
goto out;
}
+ pull_data->metadata_objects_to_scan = g_async_queue_new ();
+
requested_refs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
updated_refs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
commits_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
@@ -1610,15 +1745,17 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
goto out;
g_print ("Analyzing objects needed...\n");
+
+ g_thread_new ("metadatascan", scan_metadata_thread_main, pull_data);
g_hash_table_iter_init (&hash_iter, commits_to_fetch);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *commit = value;
-
- if (!fetch_and_store_commit_metadata_recurse (pull_data, 0, 0, commit,
- cancellable, error))
- goto out;
+
+ g_hash_table_insert (pull_data->commit_to_depth, g_strdup (commit), GUINT_TO_POINTER (opt_depth));
+ g_async_queue_push (pull_data->metadata_objects_to_scan,
+ ostree_object_name_serialize (commit, OSTREE_OBJECT_TYPE_COMMIT));
}
g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch);
@@ -1649,13 +1786,20 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
if (!ostree_validate_checksum_string (sha256, error))
goto out;
- if (!fetch_and_store_commit_metadata_recurse (pull_data, 0, 0, sha256, cancellable, error))
- goto out;
+ g_hash_table_insert (pull_data->commit_to_depth, g_strdup (sha256), GUINT_TO_POINTER (opt_depth));
+ g_async_queue_push (pull_data->metadata_objects_to_scan,
+ ostree_object_name_serialize (sha256, OSTREE_OBJECT_TYPE_COMMIT));
g_hash_table_insert (updated_refs, g_strdup (ref), g_strdup (sha256));
}
}
+ /* Push a marker saying that we're done with requests */
+ g_async_queue_push (pull_data->metadata_objects_to_scan, NULL);
+
+ /* Handle queued metadata requests here */
+ g_main_loop_run (pull_data->loop);
+
if (!fetch_content (pull_data, cancellable, error))
goto out;
@@ -1698,6 +1842,11 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
g_free (pull_data->remote_name);
if (pull_data->base_uri)
soup_uri_free (pull_data->base_uri);
+ if (pull_data->metadata_scan_thread)
+ g_thread_join (pull_data->metadata_scan_thread);
+ if (pull_data->metadata_objects_to_scan)
+ g_async_queue_unref (pull_data->metadata_objects_to_scan);
+ g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->file_checksums_to_fetch, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->cached_meta_pack_indexes, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->cached_data_pack_indexes, (GDestroyNotify) g_ptr_array_unref);
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
