[ostree] repo: Store transaction file, use it to optimize for pull resumes



commit 56f8584fe4034c2cc018f8d48139a97498e34d2f
Author: Colin Walters <walters verbum org>
Date:   Mon Jul 8 09:05:34 2013 -0400

    repo: Store transaction file, use it to optimize for pull resumes
    
    If pull is interrupted, we may have downloaded an arbitrary subset of
    the requested objects.  Previously, we handled this by scanning for
    all objects each time.
    
    However, there's an easy optimization - this patch creates a lock file
    in the repo.  If we don't see that file when starting a pull, we know
    we don't need to stat() every file; presence of a dirtree object for
    example implies the existence of everything it references.

 src/libostree/ostree-repo.c        |   36 +++++++++++++++++++++++++++++--
 src/libostree/ostree-repo.h        |    1 +
 src/ostree/ostree-pull.c           |   41 ++++++++++++++++++++---------------
 src/ostree/ot-builtin-commit.c     |    2 +-
 src/ostree/ot-builtin-pull-local.c |    8 ++++--
 5 files changed, 63 insertions(+), 25 deletions(-)
---
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 0975e72..016d361 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -54,6 +54,8 @@ struct OstreeRepo {
   GFile *remote_cache_dir;
   GFile *config_file;
 
+  GFile *transaction_lock_path;
+
   GMutex cache_lock;
   GPtrArray *cached_meta_indexes;
   GPtrArray *cached_content_indexes;
@@ -106,6 +108,9 @@ ostree_repo_finalize (GObject *object)
   g_clear_object (&self->uncompressed_objects_dir);
   g_clear_object (&self->remote_cache_dir);
   g_clear_object (&self->config_file);
+
+  g_clear_object (&self->transaction_lock_path);
+
   if (self->loose_object_devino_hash)
     g_hash_table_destroy (self->loose_object_devino_hash);
   if (self->updated_uncompressed_dirs)
@@ -1357,14 +1362,34 @@ devino_cache_lookup (OstreeRepo           *self,
 gboolean
 ostree_repo_prepare_transaction (OstreeRepo     *self,
                                  gboolean        enable_commit_hardlink_scan,
+                                 gboolean       *out_transaction_resume,
                                  GCancellable   *cancellable,
                                  GError        **error)
 {
   gboolean ret = FALSE;
+  gboolean ret_transaction_resume = FALSE;
+  gs_free char *transaction_str = NULL;
 
   g_return_val_if_fail (self->in_transaction == FALSE, FALSE);
 
+  if (self->transaction_lock_path == NULL)
+    self->transaction_lock_path = g_file_resolve_relative_path (self->repodir, "transaction");
+
+  if (g_file_query_file_type (self->transaction_lock_path, G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, NULL) == 
G_FILE_TYPE_SYMBOLIC_LINK)
+    ret_transaction_resume = TRUE;
+  else
+    ret_transaction_resume = FALSE;
+
   self->in_transaction = TRUE;
+  if (ret_transaction_resume)
+    {
+      if (!ot_gfile_ensure_unlinked (self->transaction_lock_path, cancellable, error))
+        goto out;
+    }
+  transaction_str = g_strdup_printf ("pid=%llu", (unsigned long long) getpid ());
+  if (!g_file_make_symbolic_link (self->transaction_lock_path, transaction_str,
+                                  cancellable, error))
+    goto out;
 
   if (enable_commit_hardlink_scan)
     {
@@ -1376,6 +1401,8 @@ ostree_repo_prepare_transaction (OstreeRepo     *self,
     }
 
   ret = TRUE;
+  if (out_transaction_resume)
+    *out_transaction_resume = ret_transaction_resume;
  out:
   return ret;
 }
@@ -1389,12 +1416,15 @@ ostree_repo_commit_transaction (OstreeRepo     *self,
 
   g_return_val_if_fail (self->in_transaction == TRUE, FALSE);
 
-  ret = TRUE;
-  /* out: */
-  self->in_transaction = FALSE;
+  if (!ot_gfile_ensure_unlinked (self->transaction_lock_path, cancellable, error))
+    goto out;
+
   if (self->loose_object_devino_hash)
     g_hash_table_remove_all (self->loose_object_devino_hash);
 
+  self->in_transaction = FALSE;
+  ret = TRUE;
+ out:
   return ret;
 }
 
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 69b4dff..6417b91 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -78,6 +78,7 @@ GFile *       ostree_repo_get_file_object_path (OstreeRepo   *self,
 
 gboolean      ostree_repo_prepare_transaction (OstreeRepo     *self,
                                                gboolean        enable_commit_hardlink_scan,
+                                               gboolean       *out_transaction_resume,
                                                GCancellable   *cancellable,
                                                GError        **error);
 
diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c
index 16af072..eb7cc40 100644
--- a/src/ostree/ostree-pull.c
+++ b/src/ostree/ostree-pull.c
@@ -108,6 +108,7 @@ typedef struct {
   GMainLoop    *loop;
   GCancellable *cancellable;
 
+  gboolean      transaction_resuming;
   volatile gint n_scanned_metadata;
   guint         outstanding_uri_requests;
 
@@ -822,7 +823,7 @@ scan_one_metadata_object (OtPullData         *pull_data,
   if (!ostree_repo_has_object (pull_data->repo, objtype, tmp_checksum, &is_stored,
                                cancellable, error))
     goto out;
-  
+
   if (!is_stored && !is_requested)
     {
       char *duped_checksum = g_strdup (tmp_checksum);
@@ -834,23 +835,26 @@ scan_one_metadata_object (OtPullData         *pull_data,
     }
   else if (is_stored)
     {
-      switch (objtype)
+      if (pull_data->transaction_resuming || is_requested)
         {
-        case OSTREE_OBJECT_TYPE_COMMIT:
-          if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth,
-                                   pull_data->cancellable, error))
-            goto out;
-          break;
-        case OSTREE_OBJECT_TYPE_DIR_META:
-          break;
-        case OSTREE_OBJECT_TYPE_DIR_TREE:
-          if (!scan_dirtree_object (pull_data, tmp_checksum, recursion_depth,
-                                    pull_data->cancellable, error))
-            goto out;
-          break;
-        case OSTREE_OBJECT_TYPE_FILE:
-          g_assert_not_reached ();
-          break;
+          switch (objtype)
+            {
+            case OSTREE_OBJECT_TYPE_COMMIT:
+              if (!scan_commit_object (pull_data, tmp_checksum, recursion_depth,
+                                       pull_data->cancellable, error))
+                goto out;
+              break;
+            case OSTREE_OBJECT_TYPE_DIR_META:
+              break;
+            case OSTREE_OBJECT_TYPE_DIR_TREE:
+              if (!scan_dirtree_object (pull_data, tmp_checksum, recursion_depth,
+                                        pull_data->cancellable, error))
+                goto out;
+              break;
+            case OSTREE_OBJECT_TYPE_FILE:
+              g_assert_not_reached ();
+              break;
+            }
         }
       g_hash_table_insert (pull_data->scanned_metadata, g_variant_ref (object), object);
       g_atomic_int_inc (&pull_data->n_scanned_metadata);
@@ -1347,7 +1351,8 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
         }
     }
 
-  if (!ostree_repo_prepare_transaction (pull_data->repo, FALSE, NULL, error))
+  if (!ostree_repo_prepare_transaction (pull_data->repo, FALSE, &pull_data->transaction_resuming,
+                                        cancellable, error))
     goto out;
 
   pull_data->metadata_objects_to_fetch = ot_waitable_queue_new ();
diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c
index f01815b..b79a3a3 100644
--- a/src/ostree/ot-builtin-commit.c
+++ b/src/ostree/ot-builtin-commit.c
@@ -351,7 +351,7 @@ ostree_builtin_commit (int argc, char **argv, GFile *repo_path, GError **error)
         goto out;
     }
 
-  if (!ostree_repo_prepare_transaction (repo, TRUE, cancellable, error))
+  if (!ostree_repo_prepare_transaction (repo, TRUE, NULL, cancellable, error))
     goto out;
 
   in_transaction = TRUE;
diff --git a/src/ostree/ot-builtin-pull-local.c b/src/ostree/ot-builtin-pull-local.c
index edb19b9..471306f 100644
--- a/src/ostree/ot-builtin-pull-local.c
+++ b/src/ostree/ot-builtin-pull-local.c
@@ -170,6 +170,7 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err
   int i;
   GHashTableIter hash_iter;
   gpointer key, value;
+  gboolean transaction_resuming = FALSE;
   gs_unref_hashtable GHashTable *objects = NULL;
   gs_unref_object GFile *src_f = NULL;
   gs_unref_object GFile *src_repo_dir = NULL;
@@ -247,6 +248,10 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err
         }
     }
 
+  if (!ostree_repo_prepare_transaction (data->dest_repo, FALSE, &transaction_resuming,
+                                        cancellable, error))
+    goto out;
+
   g_print ("Enumerating objects...\n");
 
   source_objects = ostree_traverse_new_reachable ();
@@ -275,9 +280,6 @@ ostree_builtin_pull_local (int argc, char **argv, GFile *repo_path, GError **err
         }
     }
 
-  if (!ostree_repo_prepare_transaction (data->dest_repo, FALSE, cancellable, error))
-    goto out;
-
   data->n_objects_to_check = g_hash_table_size (source_objects);
   g_hash_table_iter_init (&hash_iter, source_objects);
   while (g_hash_table_iter_next (&hash_iter, &key, &value))


[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]