[ostree] repo: Store transaction file, use it to optimize for pull resumes
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree] repo: Store transaction file, use it to optimize for pull resumes
- Date: Mon, 8 Jul 2013 14:31:38 +0000 (UTC)
commit 56f8584fe4034c2cc018f8d48139a97498e34d2f
Author: Colin Walters <walters verbum org>
Date: Mon Jul 8 09:05:34 2013 -0400
repo: Store transaction file, use it to optimize for pull resumes
If pull is interrupted, we may have downloaded an arbitrary subset of
the requested objects. Previously, we handled this by scanning for
all objects each time.
However, there's an easy optimization - this patch creates a lock file
in the repo. If we don't see that file when starting a pull, we know
we don't need to stat() every file; presence of a dirtree object for
example implies the existence of everything it references.
src/libostree/ostree-repo.c | 36 +++++++++++++++++++++++++++++--
src/libostree/ostree-repo.h | 1 +
src/ostree/ostree-pull.c | 41 ++++++++++++++++++++---------------
src/ostree/ot-builtin-commit.c | 2 +-
src/ostree/ot-builtin-pull-local.c | 8 ++++--
5 files changed, 63 insertions(+), 25 deletions(-)
---
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 0975e72..016d361 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -54,6 +54,8 @@ struct OstreeRepo {
GFile *remote_cache_dir;
GFile *config_file;
+ GFile *transaction_lock_path;
+
GMutex cache_lock;
GPtrArray *cached_meta_indexes;
GPtrArray *cached_content_indexes;
@@ -106,6 +108,9 @@ ostree_repo_finalize (GObject *object)
g_clear_object (&self->uncompressed_objects_dir);
g_clear_object (&self->remote_cache_dir);
g_clear_object (&self->config_file);
+
+ g_clear_object (&self->transaction_lock_path);
+
if (self->loose_object_devino_hash)
g_hash_table_destroy (self->loose_object_devino_hash);
if (self->updated_uncompressed_dirs)
@@ -1357,14 +1362,34 @@ devino_cache_lookup (OstreeRepo *self,
gboolean
ostree_repo_prepare_transaction (OstreeRepo *self,
gboolean enable_commit_hardlink_scan,
+ gboolean *out_transaction_resume,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
+ gboolean ret_transaction_resume = FALSE;
+ gs_free char *transaction_str = NULL;
g_return_val_if_fail (self->in_transaction == FALSE, FALSE);
+ if (self->transaction_lock_path == NULL)
+ self->transaction_lock_path = g_file_resolve_relative_path (self->repodir, "transaction");
+
+ if (g_file_query_file_type (self->transaction_lock_path, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL) ==
G_FILE_TYPE_SYMBOLIC_LINK)
+ ret_transaction_resume = TRUE;
+ else
+ ret_transaction_resume = FALSE;
+
self->in_transaction = TRUE;
+ if (ret_transaction_resume)
+ {
+ if (!ot_gfile_ensure_unlinked (self->transaction_lock_path, cancellable, error))
+ goto out;
+ }
+ transaction_str = g_strdup_printf ("pid=%llu", (unsigned long long) getpid ());
+ if (!g_file_make_symbolic_link (self->transaction_lock_path, transaction_str,
+ cancellable, error))
+ goto out;
if (enable_commit_hardlink_scan)
{
@@ -1376,6 +1401,8 @@ ostree_repo_prepare_transaction (OstreeRepo *self,
}
ret = TRUE;
+ if (out_transaction_resume)
+ *out_transaction_resume = ret_transaction_resume;
out:
return ret;
}
@@ -1389,12 +1416,15 @@ ostree_repo_commit_transaction (OstreeRepo *self,
g_return_val_if_fail (self->in_transaction == TRUE, FALSE);
- ret = TRUE;
- /* out: */
- self->in_transaction = FALSE;
+ if (!ot_gfile_ensure_unlinked (self->transaction_lock_path, cancellable, error))
+ goto out;
+
if (self->loose_object_devino_hash)
g_hash_table_remove_all (self->loose_object_devino_hash);
+ self->in_transaction = FALSE;
+ ret = TRUE;
+ out:
return ret;
}
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 69b4dff..6417b91 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -78,6 +78,7 @@ GFile * ostree_repo_get_file_object_path (OstreeRepo *self,
gboolean ostree_repo_prepare_transaction (OstreeRepo *self,
gboolean enable_commit_hardlink_scan,
+ gboolean *out_transaction_resume,
GCancellable *cancellable,
GError **error);
diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c
index 16af072..eb7cc40 100644
--- a/src/ostree/ostree-pull.c
+++ b/src/ostree/ostree-pull.c
@@ -108,6 +108,7 @@ typedef struct {
GMainLoop *loop;
GCancellable *cancellable;
+ gboolean transaction_resuming;
volatile gint n_scanned_metadata;
guint outstanding_uri_requests;
@@ -822,7 +823,7 @@ scan_one_metadata_object (OtPullData *pull_data,
if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored,
cancellable, error))
goto out;
-
+
if (!is_stored && !is_requested)
{
char *duped_checksum = g_strdup (tmp_checksum);
@@ -834,23 +835,26 @@ scan_one_metadata_object (OtPullData *pull_data,
}
else if (is_stored)
{
- switch (objtype)
+ if (pull_data->transaction_resuming || is_requested)
{
- case OSTREE_OBJECT_TYPE_COMMIT:
- if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth,
- pull_data->cancellable, error))
- goto out;
- break;
- case OSTREE_OBJECT_TYPE_DIR_META:
- break;
- case OSTREE_OBJECT_TYPE_DIR_TREE:
- if (!scan_dirtree_object (pull_data, tmp_checksum, recursion_depth,
- pull_data->cancellable, error))
- goto out;
- break;
- case OSTREE_OBJECT_TYPE_FILE:
- g_assert_not_reached ();
- break;
+ switch (objtype)
+ {
+ case OSTREE_OBJECT_TYPE_COMMIT:
+ if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth,
+ pull_data->cancellable, error))
+ goto out;
+ break;
+ case OSTREE_OBJECT_TYPE_DIR_META:
+ break;
+ case OSTREE_OBJECT_TYPE_DIR_TREE:
+ if (!scan_dirtree_object (pull_data, tmp_checksum, recursion_depth,
+ pull_data->cancellable, error))
+ goto out;
+ break;
+ case OSTREE_OBJECT_TYPE_FILE:
+ g_assert_not_reached ();
+ break;
+ }
}
g_hash_table_insert (pull_data->scanned_metadata, g_variant_ref (object), object);
g_atomic_int_inc (&pull_data->n_scanned_metadata);
@@ -1347,7 +1351,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
}
}
- if (!ostree_repo_prepare_transaction (pull_data->repo, FALSE, NULL, error))
+ if (!ostree_repo_prepare_transaction (pull_data->repo, FALSE, &pull_data->transaction_resuming,
+ cancellable, error))
goto out;
pull_data->metadata_objects_to_fetch = ot_waitable_queue_new ();
diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c
index f01815b..b79a3a3 100644
--- a/src/ostree/ot-builtin-commit.c
+++ b/src/ostree/ot-builtin-commit.c
@@ -351,7 +351,7 @@ ostree_builtin_commit (int argc, char **argv, GFile *repo_path, GError **error)
goto out;
}
- if (!ostree_repo_prepare_transaction (repo, TRUE, cancellable, error))
+ if (!ostree_repo_prepare_transaction (repo, TRUE, NULL, cancellable, error))
goto out;
in_transaction = TRUE;
diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c
index edb19b9..471306f 100644
--- a/src/ostree/ot-builtin-pull-local.c
+++ b/src/ostree/ot-builtin-pull-local.c
@@ -170,6 +170,7 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err
int i;
GHashTableIter hash_iter;
gpointer key, value;
+ gboolean transaction_resuming = FALSE;
gs_unref_hashtable GHashTable *objects = NULL;
gs_unref_object GFile *src_f = NULL;
gs_unref_object GFile *src_repo_dir = NULL;
@@ -247,6 +248,10 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err
}
}
+ if (!ostree_repo_prepare_transaction (data->dest_repo, FALSE, &transaction_resuming,
+ cancellable, error))
+ goto out;
+
g_print ("Enumerating objects...\n");
source_objects = ostree_traverse_new_reachable ();
@@ -275,9 +280,6 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err
}
}
- if (!ostree_repo_prepare_transaction (data->dest_repo, FALSE, cancellable, error))
- goto out;
-
data->n_objects_to_check = g_hash_table_size (source_objects);
g_hash_table_iter_init (&hash_iter, source_objects);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]