[ostree] pull: Add a --dry-run option for static deltas

[Colin Walters <walters src gnome org>]

commit 4beb5f4eaf619f1c47762bb872099e24f3efe2f0
Author: Colin Walters <walters verbum org>
Date:   Fri Feb 19 12:28:07 2016 -0500

    pull: Add a --dry-run option for static deltas
    
    One of the design goals with deltas was not just wire efficiency,
    but also having all the data up front about how much data would
    be transferred before starting.
    
    Let's expose that better by adding a `dry-run` option to the pull API.
    This requires static deltas to be useful.  Basically we simply call
    the progress callback once with the data from the superblock.

 src/libostree/ostree-repo-pull.c |   40 +++++++++++++++++++++--
 src/ostree/ot-builtin-pull.c     |   66 +++++++++++++++++++++++++++++++++++--
 tests/pull-test.sh               |   14 ++++++++
 3 files changed, 113 insertions(+), 7 deletions(-)
---
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index a7db45f..efa424f 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -48,6 +48,8 @@ typedef struct {
   GCancellable *cancellable;
   OstreeAsyncProgress *progress;
 
+  gboolean      dry_run;
+  gboolean      dry_run_emitted_progress;
   gboolean      legacy_transaction_resuming;
   enum {
     OSTREE_PULL_PHASE_FETCHING_REFS,
@@ -78,6 +80,7 @@ typedef struct {
   guint             n_outstanding_deltapart_write_requests;
   guint             n_total_deltaparts;
   guint64           total_deltapart_size;
+  guint64           total_deltapart_usize;
   gint              n_requested_metadata;
   gint              n_requested_content;
   guint             n_fetched_deltaparts;
@@ -227,6 +230,8 @@ update_progress (gpointer user_data)
                                   pull_data->n_total_deltaparts);
   ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-size",
                                     pull_data->total_deltapart_size);
+  ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-usize",
+                                    pull_data->total_deltapart_usize);
   ostree_async_progress_set_uint (pull_data->progress, "total-delta-superblocks",
                                   pull_data->static_delta_superblocks->len);
 
@@ -243,6 +248,9 @@ update_progress (gpointer user_data)
   else
     ostree_async_progress_set_status (pull_data->progress, NULL);
 
+  if (pull_data->dry_run)
+    pull_data->dry_run_emitted_progress = TRUE;
+
   return TRUE;
 }
 
@@ -262,6 +270,9 @@ pull_termination_condition (OtPullData          *pull_data)
   if (pull_data->caught_error)
     return TRUE;
 
+  if (pull_data->dry_run)
+    return pull_data->dry_run_emitted_progress;
+
   switch (pull_data->phase)
     {
     case OSTREE_PULL_PHASE_FETCHING_REFS:
@@ -1451,11 +1462,18 @@ process_one_static_delta_fallback (OtPullData   *pull_data,
   if (!ostree_validate_structureof_csum_v (csum_v, error))
     goto out;
 
+  pull_data->total_deltapart_size += compressed_size;
+  pull_data->total_deltapart_usize += uncompressed_size;
+
+  if (pull_data->dry_run)
+    {
+      ret = TRUE;
+      goto out;
+    }
+
   objtype = (OstreeObjectType)objtype_y;
   checksum = ostree_checksum_from_bytes_v (csum_v);
 
-  pull_data->total_deltapart_size += compressed_size;
-
   if (!ostree_repo_has_object (pull_data->repo, objtype, checksum,
                                &is_stored,
                                cancellable, error))
@@ -1524,6 +1542,7 @@ process_one_static_delta (OtPullData   *pull_data,
     }
 
   /* Write the to-commit object */
+  if (!pull_data->dry_run)
   {
     g_autoptr(GVariant) to_csum_v = NULL;
     g_autofree char *to_checksum = NULL;
@@ -1626,7 +1645,11 @@ process_one_static_delta (OtPullData   *pull_data,
       }
 
       pull_data->total_deltapart_size += size;
+      pull_data->total_deltapart_usize += usize;
 
+      if (pull_data->dry_run)
+        continue;
+      
       fetch_data = g_new0 (FetchStaticDeltaData, 1);
       fetch_data->pull_data = pull_data;
       fetch_data->objects = g_variant_ref (objects);
@@ -1780,6 +1803,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
       (void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas);
       (void) g_variant_lookup (options, "require-static-deltas", "b", &require_static_deltas);
       (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids);
+      (void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run);
     }
 
   g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE);
@@ -1790,6 +1814,10 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
     g_return_val_if_fail (dir_to_pull[0] == '/', FALSE);
 
   g_return_val_if_fail (!(disable_static_deltas && require_static_deltas), FALSE);
+  /* We only do dry runs with static deltas, because we don't really have any
+   * in-advance information for bare fetches.
+   */
+  g_return_val_if_fail (!pull_data->dry_run || require_static_deltas, FALSE);
 
   pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0;
   pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0;
@@ -2243,7 +2271,7 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
 
   if (pull_data->progress)
     {
-      update_timeout = g_timeout_source_new_seconds (1);
+      update_timeout = g_timeout_source_new_seconds (pull_data->dry_run ? 0 : 1);
       g_source_set_priority (update_timeout, G_PRIORITY_HIGH);
       g_source_set_callback (update_timeout, update_progress, pull_data, NULL);
       g_source_attach (update_timeout, pull_data->main_context);
@@ -2256,6 +2284,12 @@ ostree_repo_pull_with_options (OstreeRepo             *self,
 
   if (pull_data->caught_error)
     goto out;
+
+  if (pull_data->dry_run)
+    {
+      ret = TRUE;
+      goto out;
+    }
   
   g_assert_cmpint (pull_data->n_outstanding_metadata_fetches, ==, 0);
   g_assert_cmpint (pull_data->n_outstanding_metadata_write_requests, ==, 0);
diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c
index 7fa673c..7c91890 100644
--- a/src/ostree/ot-builtin-pull.c
+++ b/src/ostree/ot-builtin-pull.c
@@ -30,6 +30,7 @@
 static gboolean opt_disable_fsync;
 static gboolean opt_mirror;
 static gboolean opt_commit_only;
+static gboolean opt_dry_run;
 static gboolean opt_disable_static_deltas;
 static gboolean opt_require_static_deltas;
 static char* opt_subpath;
@@ -42,6 +43,7 @@ static GOptionEntry options[] = {
    { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", 
NULL },
    { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL },
    { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Only pull the provided subpath", NULL },
+   { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Only print information on what will be downloaded 
(requires static deltas)", NULL },
    { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", 
"DEPTH" },
    { NULL }
  };
@@ -62,6 +64,39 @@ gpg_verify_result_cb (OstreeRepo *repo,
   gs_console_begin_status_line (console, "", NULL, NULL);
 }
 
+static gboolean printed_console_progress;
+
+static void
+dry_run_console_progress_changed (OstreeAsyncProgress *progress,
+                                  gpointer             user_data)
+{
+  guint fetched_delta_parts, total_delta_parts;
+  guint64 total_delta_part_size, total_delta_part_usize;
+  GString *buf;
+
+  g_assert (!printed_console_progress);
+  printed_console_progress = TRUE;
+
+  fetched_delta_parts = ostree_async_progress_get_uint (progress, "fetched-delta-parts");
+  total_delta_parts = ostree_async_progress_get_uint (progress, "total-delta-parts");
+  total_delta_part_size = ostree_async_progress_get_uint64 (progress, "total-delta-part-size");
+  total_delta_part_usize = ostree_async_progress_get_uint64 (progress, "total-delta-part-usize");
+
+  buf = g_string_new ("");
+
+  { g_autofree char *formatted_size =
+      g_format_size (total_delta_part_size);
+    g_autofree char *formatted_usize =
+      g_format_size (total_delta_part_usize);
+
+    g_string_append_printf (buf, "Delta update: %u/%u parts, %s to transfer, %s uncompressed",
+                            fetched_delta_parts, total_delta_parts,
+                            formatted_size, formatted_usize);
+  }
+  g_print ("%s\n", buf->str);
+  g_string_free (buf, TRUE);
+}
+
 gboolean
 ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **error)
 {
@@ -99,6 +134,13 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
   if (opt_commit_only)
     pullflags |= OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY;
 
+  if (opt_dry_run && !opt_require_static_deltas)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "--dry-run requires --require-static-deltas");
+      goto out;
+    }
+
   if (strchr (argv[1], ':') == NULL)
     {
       remote = g_strdup (argv[1]);
@@ -149,11 +191,21 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
       g_ptr_array_add (refs_to_fetch, NULL);
     }
 
-  console = gs_console_get ();
-  if (console)
+  if (!opt_dry_run)
     {
-      gs_console_begin_status_line (console, "", NULL, NULL);
-      progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, 
console);
+      console = gs_console_get ();
+      if (console)
+        {
+          gs_console_begin_status_line (console, "", NULL, NULL);
+          progress = ostree_async_progress_new_and_connect 
(ostree_repo_pull_default_console_progress_changed, console);
+          signal_handler_id = g_signal_connect (repo, "gpg-verify-result",
+                                                G_CALLBACK (gpg_verify_result_cb),
+                                                console);
+        }
+    }
+  else
+    {
+      progress = ostree_async_progress_new_and_connect (dry_run_console_progress_changed, console);
       signal_handler_id = g_signal_connect (repo, "gpg-verify-result",
                                             G_CALLBACK (gpg_verify_result_cb),
                                             console);
@@ -180,6 +232,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
     g_variant_builder_add (&builder, "{s v}", "require-static-deltas",
                            g_variant_new_variant (g_variant_new_boolean (opt_require_static_deltas)));
 
+    g_variant_builder_add (&builder, "{s v}", "dry-run",
+                           g_variant_new_variant (g_variant_new_boolean (opt_dry_run)));
+
     if (override_commit_ids)
       g_variant_builder_add (&builder, "{s v}", "override-commit-ids",
                              g_variant_new_variant (g_variant_new_strv ((const 
char*const*)override_commit_ids->pdata, override_commit_ids->len)));
@@ -192,6 +247,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **
   if (progress)
     ostree_async_progress_finish (progress);
 
+  if (opt_dry_run)
+    g_assert (printed_console_progress);
+
   ret = TRUE;
  out:
   if (signal_handler_id > 0)
diff --git a/tests/pull-test.sh b/tests/pull-test.sh
index 04272d5..f73b681 100755
--- a/tests/pull-test.sh
+++ b/tests/pull-test.sh
@@ -107,12 +107,24 @@ rm main-files -rf
 # Generate delta that we'll use
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate main
 prev_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main^)
+new_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main)
 ostree --repo=ostree-srv/gnomerepo summary -u
 
 cd ${test_tmpdir}
 repo_init
 ${CMD_PREFIX} ostree --repo=repo pull origin main ${prev_rev}
+${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >out.txt
+assert_file_has_content out.txt 'Delta update: 0/1 parts'
+rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main)
+assert_streq "${prev_rev}" "${rev}"
+${CMD_PREFIX} ostree --repo=repo fsck
+
+cd ${test_tmpdir}
+repo_init
+${CMD_PREFIX} ostree --repo=repo pull origin main ${prev_rev}
 ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main
+rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main)
+assert_streq "${new_rev}" "${rev}"
 ${CMD_PREFIX} ostree --repo=repo fsck
 
 cd ${test_tmpdir}
@@ -140,6 +152,8 @@ fi
 assert_file_has_content err.txt "deltas required, but none found"
 ${CMD_PREFIX} ostree --repo=repo fsck
 
+echo "ok delta required but don't exist"
+
 cd ${test_tmpdir}
 rm main-files -rf
 ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main main-files