GNOME.org
 

[ostree/wip/metalinks: 2/2] More metalink work, plus a test case

commit b8ba2402a16ca39243f343a79f2da2aab716cc80
Author: Colin Walters <walters verbum org>
Date:   Tue Aug 19 14:56:38 2014 -0400

    More metalink work, plus a test case

 src/libostree/ostree-core.h      |    6 +-
 src/libostree/ostree-metalink.c  |   13 +++-
 src/libostree/ostree-repo-pull.c |  143 ++++++++++++++++++++++++++-----------
 src/libostree/ostree-repo.c      |   15 +++-
 src/libotutil/ot-variant-utils.c |  121 ++++++++++++++++++++++++++++++++
 src/libotutil/ot-variant-utils.h |   10 +++
 src/ostree/ot-builtin-summary.c  |    1 -
 tests/test-pull-metalink.sh      |    1 +
 8 files changed, 257 insertions(+), 53 deletions(-)
---
diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h
index 8ed8d2d..c1b81d7 100644
--- a/src/libostree/ostree-core.h
+++ b/src/libostree/ostree-core.h
@@ -120,10 +120,10 @@ typedef enum {
 /**
  * OSTREE_SUMMARY_GVARIANT_FORMAT:
  *
- * a(ayay) - Array of (checksum, commit data)
- * refs: a{s(aya{sv})} - Map of ref name -> (latest commit, additional metadata)
+ * a(ayay) - Array of (checksum, commit data), sorted by checksum
+ * refs: a(s(aya{sv})) - Map of ref name -> (latest commit, additional metadata), sorted by ref name
  */
-#define OSTREE_SUMMARY_GVARIANT_STRING "(a(ayay)a{s(aya{sv})})"
+#define OSTREE_SUMMARY_GVARIANT_STRING "(a(ayay)a(s(aya{sv})))"
 #define OSTREE_SUMMARY_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_SUMMARY_GVARIANT_STRING)
 
 /**
diff --git a/src/libostree/ostree-metalink.c b/src/libostree/ostree-metalink.c
index 2c0541e..94057d7 100644
--- a/src/libostree/ostree-metalink.c
+++ b/src/libostree/ostree-metalink.c
@@ -327,8 +327,9 @@ _ostree_metalink_finalize (GObject *object)
 
   self = OSTREE_METALINK (object);
 
+  g_object_unref (self->fetcher);
   g_free (self->requested_file);
-  g_clear_object (&self->uri);
+  soup_uri_free (self->uri);
 
   G_OBJECT_CLASS (_ostree_metalink_parent_class)->finalize (object);
 }
@@ -354,9 +355,10 @@ _ostree_metalink_new (OstreeFetcher  *fetcher,
 {
   OstreeMetalink *self = (OstreeMetalink*)g_object_new (OSTREE_TYPE_METALINK, NULL);
 
+  self->fetcher = g_object_ref (fetcher);
   self->requested_file = g_strdup (requested_file);
   self->max_size = max_size;
-  self->uri = g_object_ref (uri);
+  self->uri = soup_uri_copy (uri);
  
   return self;
 }
@@ -685,12 +687,19 @@ _ostree_metalink_request_sync (OstreeMetalink         *self,
   gboolean ret = FALSE;
   GMainContext *sync_context = g_main_context_new ();
   MetalinkSyncCallState state = { 0, };
+
+  state.error = error;
+  state.running = TRUE;
+
+  g_main_context_push_thread_default (sync_context);
   
   _ostree_metalink_request_async (self, cancellable, on_async_result, &state);
   
   while (state.running)
     g_main_context_iteration (sync_context, TRUE);
 
+  g_main_context_pop_thread_default (sync_context);
+
   ret = state.success;
   if (sync_context)
     g_main_context_unref (sync_context);
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 6df3688..ba906cb 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -52,6 +52,7 @@ typedef struct {
   
   gboolean          gpg_verify;
 
+  GVariant         *summary;
   GPtrArray        *static_delta_metas;
   GHashTable       *scanned_metadata; /* Maps object name to itself */
   GHashTable       *requested_metadata; /* Maps object name to itself */
@@ -480,6 +481,41 @@ fetch_ref_contents (OtPullData    *pull_data,
   return ret;
 }
 
+static gboolean
+lookup_commit_checksum_from_summary (OtPullData    *pull_data,
+                                     const char    *ref,
+                                     char         **out_checksum,
+                                     GError       **error)
+{
+  gboolean ret = FALSE;
+  gs_unref_variant GVariant *commits = g_variant_get_child_value (pull_data->summary, 0);
+  gs_unref_variant GVariant *refs = g_variant_get_child_value (pull_data->summary, 1);
+  gs_unref_variant GVariant *refdata = NULL;
+  gs_unref_variant GVariant *commit_data = NULL;
+  gs_unref_variant GVariant *commit_csum_v = NULL;
+  gs_unref_bytes GBytes *commit_bytes = NULL;
+  int i;
+  
+  if (!ot_variant_bsearch_str (refs, ref, &i))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "No such branch '%s' in repository summary",
+                   ref);
+      goto out;
+    }
+      
+  refdata = g_variant_get_child_value (refs, i);
+  commit_csum_v = g_variant_get_child_value (refdata, 0);
+
+  if (!ostree_validate_structureof_csum_v (commit_csum_v, error))
+    goto out;
+
+  ret = TRUE;
+  *out_checksum = ostree_checksum_from_bytes_v (commit_csum_v);
+ out:
+  return ret;
+}
+
 static void
 content_fetch_on_write_complete (GObject        *object,
                                  GAsyncResult   *result,
@@ -1030,7 +1066,7 @@ ostree_repo_pull (OstreeRepo               *self,
   gs_free char *remote_key = NULL;
   gs_free char *path = NULL;
   gs_free char *baseurl = NULL;
-  gs_free char *metalink_url = NULL;
+  gs_free char *metalink_url_str = NULL;
   gs_unref_hashtable GHashTable *requested_refs_to_fetch = NULL;
   gs_unref_hashtable GHashTable *commits_to_fetch = NULL;
   gs_free char *remote_mode_str = NULL;
@@ -1073,38 +1109,6 @@ ostree_repo_pull (OstreeRepo               *self,
       goto out;
     }
 
-  if (!repo_get_string_key_inherit (self, remote_key, "metalink", &metalink_url, error))
-    goto out;
-
-  if (!metalink_url)
-    {
-      if (!repo_get_string_key_inherit (self, remote_key, "url", &baseurl, error))
-        goto out;
-
-      pull_data->base_uri = soup_uri_new (baseurl);
-
-      if (!pull_data->base_uri)
-        {
-          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                       "Failed to parse url '%s'", baseurl);
-          goto out;
-        }
-    }
-  else
-    {
-      gs_unref_object GFile *metalink_data = NULL;
-      SoupURI *metalink_uri = soup_uri_new (metalink_url);
-      metalink = _ostree_metalink_new (pull_data->fetcher, metalink_url, OSTREE_MAX_METADATA_SIZE, 
metalink_uri);
-      g_object_unref (metalink_uri);
-
-      if (!_ostree_metalink_request_sync (metalink, &pull_data->base_uri,
-                                          &metalink_data,
-                                          cancellable, error))
-        goto out;
-
-      
-    }
-
 #ifdef HAVE_GPGME
   if (!ot_keyfile_get_boolean_with_default (config, remote_key, "gpg-verify",
                                             TRUE, &pull_data->gpg_verify, error))
@@ -1178,6 +1182,48 @@ ostree_repo_pull (OstreeRepo               *self,
       }
   }
 
+  if (!repo_get_string_key_inherit (self, remote_key, "metalink", &metalink_url_str, error))
+    goto out;
+
+  if (!metalink_url_str)
+    {
+      if (!repo_get_string_key_inherit (self, remote_key, "url", &baseurl, error))
+        goto out;
+
+      pull_data->base_uri = soup_uri_new (baseurl);
+
+      if (!pull_data->base_uri)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Failed to parse url '%s'", baseurl);
+          goto out;
+        }
+    }
+  else
+    {
+      gs_unref_object GFile *metalink_data = NULL;
+      SoupURI *metalink_uri = soup_uri_new (metalink_url_str);
+      
+      if (!metalink_uri)
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Invalid metalink URL: %s", metalink_url_str);
+          goto out;
+        }
+      
+      metalink = _ostree_metalink_new (pull_data->fetcher, "summary", OSTREE_MAX_METADATA_SIZE, 
metalink_uri);
+      soup_uri_free (metalink_uri);
+
+      if (!_ostree_metalink_request_sync (metalink, &pull_data->base_uri,
+                                          &metalink_data,
+                                          cancellable, error))
+        goto out;
+
+      if (!ot_util_variant_map (metalink_data, OSTREE_SUMMARY_GVARIANT_FORMAT, FALSE,
+                                &pull_data->summary, error))
+        goto out;
+    }
+
   if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
     goto out;
 
@@ -1207,7 +1253,6 @@ ostree_repo_pull (OstreeRepo               *self,
       for (strviter = refs_to_fetch; *strviter; strviter++)
         {
           const char *branch = *strviter;
-          char *contents;
 
           if (ostree_validate_checksum_string (branch, NULL))
             {
@@ -1216,11 +1261,7 @@ ostree_repo_pull (OstreeRepo               *self,
             }
           else
             {
-              if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
-                goto out;
-      
-              /* Transfer ownership of contents */
-              g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents);
+              g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL);
             }
         }
     }
@@ -1240,14 +1281,29 @@ ostree_repo_pull (OstreeRepo               *self,
       for (;branches_iter && *branches_iter; branches_iter++)
         {
           const char *branch = *branches_iter;
-          char *contents;
               
+          g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL);
+        }
+    }
+
+  while (g_hash_table_iter_next (&hash_iter, &key, &value))
+    {
+      const char *branch = key;
+      char *contents;
+
+      if (pull_data->summary)
+        {
+          if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, error))
+            goto out;
+        }
+      else
+        {
           if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
             goto out;
-
-          /* Transfer ownership of contents */
-          g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents);
         }
+      
+      /* Transfer ownership of contents */
+      g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents);
     }
 
   pull_data->phase = OSTREE_PULL_PHASE_FETCHING_OBJECTS;
@@ -1348,6 +1404,7 @@ ostree_repo_pull (OstreeRepo               *self,
   g_free (pull_data->remote_name);
   if (pull_data->base_uri)
     soup_uri_free (pull_data->base_uri);
+  g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref);
   g_clear_pointer (&pull_data->static_delta_metas, (GDestroyNotify) g_ptr_array_unref);
   g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref);
   g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref);
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 6ef5397..f642bd5 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -2282,6 +2282,7 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
                                 GError        **error)
 {
   gboolean ret = FALSE;
+  gs_unref_object GFile *summary_path = NULL;
   gs_unref_hashtable GHashTable *refs = NULL;
   gs_unref_hashtable GHashTable *commits = NULL;
   gs_unref_variant_builder GVariantBuilder *refs_builder = NULL;
@@ -2289,7 +2290,6 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
   gs_unref_variant GVariant *summary = NULL;
   GList *ordered_keys = NULL;
   GList *iter = NULL;
-  GHashTableIter hashiter;
 
   if (!ostree_repo_list_refs (self, NULL, &refs, cancellable, error))
     goto out;
@@ -2320,7 +2320,7 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
           g_hash_table_insert (commits, g_strdup (commit), g_variant_ref (commit_content));
         }
 
-      csum_v = ostree_checksum_to_bytes_v (commit);
+      csum_v = g_variant_ref_sink (ostree_checksum_to_bytes_v (commit));
 
       g_variant_builder_add_value (refs_builder, 
                                    g_variant_new ("(s(@ay a{sv}))", ref, csum_v,
@@ -2343,9 +2343,9 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
 
       g_assert (commit_content);
 
-      csum_v = ostree_checksum_to_bytes_v (commit);
+      csum_v = g_variant_ref_sink (ostree_checksum_to_bytes_v (commit));
       commit_data = g_variant_get_data_as_bytes (commit_content);
-      commit_data_v = g_variant_new_from_bytes (G_VARIANT_TYPE ("ay"), commit_data, TRUE);
+      commit_data_v = g_variant_ref_sink (g_variant_new_from_bytes (G_VARIANT_TYPE ("ay"), commit_data, 
TRUE));
 
       g_variant_builder_add_value (commits_builder, 
                                    g_variant_new ("(@ay ay)", csum_v, commit_data_v));
@@ -2354,11 +2354,18 @@ ostree_repo_regenerate_summary (OstreeRepo     *self,
   {
     gs_unref_variant_builder GVariantBuilder *summary_builder =
       g_variant_builder_new (OSTREE_SUMMARY_GVARIANT_FORMAT);
+
     g_variant_builder_add_value (summary_builder, g_variant_builder_end (commits_builder));
     g_variant_builder_add_value (summary_builder, g_variant_builder_end (refs_builder));
     summary = g_variant_builder_end (summary_builder);
+    g_variant_ref_sink (summary);
   }
 
+  summary_path = g_file_get_child (self->repodir, "summary");
+
+  if (!ot_util_variant_save (summary_path, summary, cancellable, error))
+    goto out;
+
   ret = TRUE;
  out:
   if (ordered_keys)
diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c
index db79616..031be2a 100644
--- a/src/libotutil/ot-variant-utils.c
+++ b/src/libotutil/ot-variant-utils.c
@@ -288,3 +288,124 @@ ot_variant_new_from_bytes (const GVariantType  *type,
                                   (GDestroyNotify)g_bytes_unref, bytes);
 #endif
 }
+
+/**
+ * ot_variant_bsearch_str:
+ * @array: A GVariant array whose first element must be a string
+ * @str: Search for this string
+ * @out_pos: Output position
+ *
+ *
+ * Binary search in a GVariant array, which must be of the form 'a(s...)',
+ * where '...' may be anything.  The array elements must be sorted.
+ *
+ * Returns: %TRUE if found, %FALSE otherwise
+ */
+gboolean
+ot_variant_bsearch_str (GVariant   *array,
+                        const char *str,
+                        int        *out_pos)
+{
+  gsize imax, imin;
+  gsize imid;
+  gsize n;
+
+  n = g_variant_n_children (array);
+  if (n == 0)
+    return FALSE;
+
+  imax = n - 1;
+  imin = 0;
+  while (imax >= imin)
+    {
+      gs_unref_variant GVariant *child = NULL;
+      const char *cur;
+      int cmp;
+
+      imid = (imin + imax) / 2;
+
+      child = g_variant_get_child_value (array, imid);
+      g_variant_get_child (child, 0, "&s", &cur, NULL);      
+
+      cmp = strcmp (cur, str);
+      if (cmp < 0)
+        imin = imid + 1;
+      else if (cmp > 0)
+        {
+          if (imid == 0)
+            break;
+          imax = imid - 1;
+        }
+      else
+        {
+          *out_pos = imid;
+          return TRUE;
+        }
+    }
+
+  *out_pos = imid;
+  return FALSE;
+}
+
+/**
+ * ot_variant_bsearch_bytes:
+ * 
+ * Like ot_variant_bsearch, but with binary data like checksums.
+ */
+gboolean
+ot_variant_bsearch_bytes (GVariant   *array,
+                          GBytes     *bytes,
+                          int        *out_pos)
+{
+  gsize imax, imin;
+  gsize imid;
+  gsize n;
+  const guint8 *bytes_data;
+  gsize bytes_len;
+
+  bytes_data = g_bytes_get_data (bytes, &bytes_len);
+
+  n = g_variant_n_children (array);
+  if (n == 0)
+    return FALSE;
+
+  imax = n - 1;
+  imin = 0;
+  while (imax >= imin)
+    {
+      gs_unref_variant GVariant *child = NULL;
+      gs_unref_variant GVariant *child_first = NULL;
+      const guint8 *variant_data;
+      gsize variant_size;
+      int cmp;
+
+      imid = (imin + imax) / 2;
+
+      child = g_variant_get_child_value (array, imid);
+      child_first = g_variant_get_child_value (child, 0);      
+
+      variant_data = g_variant_get_data (child_first);
+      variant_size = g_variant_get_size (child_first);
+
+      if (variant_size != bytes_len)
+        cmp = 1;
+      else
+        cmp = memcmp (variant_data, bytes_data, bytes_len);
+      if (cmp < 0)
+        imin = imid + 1;
+      else if (cmp > 0)
+        {
+          if (imid == 0)
+            break;
+          imax = imid - 1;
+        }
+      else
+        {
+          *out_pos = imid;
+          return TRUE;
+        }
+    }
+
+  *out_pos = imid;
+  return FALSE;
+}
diff --git a/src/libotutil/ot-variant-utils.h b/src/libotutil/ot-variant-utils.h
index 429bceb..28f4096 100644
--- a/src/libotutil/ot-variant-utils.h
+++ b/src/libotutil/ot-variant-utils.h
@@ -72,5 +72,15 @@ ot_variant_new_from_bytes (const GVariantType  *type,
                            GBytes        *bytes,
                            gboolean       trusted);
 
+gboolean
+ot_variant_bsearch_str (GVariant   *array,
+                        const char *str,
+                        int        *out_pos);
+
+gboolean
+ot_variant_bsearch_bytes (GVariant   *array,
+                          GBytes     *bytes,
+                          int        *out_pos);
+
 G_END_DECLS
 
diff --git a/src/ostree/ot-builtin-summary.c b/src/ostree/ot-builtin-summary.c
index 17feb34..667d6fe 100644
--- a/src/ostree/ot-builtin-summary.c
+++ b/src/ostree/ot-builtin-summary.c
@@ -36,7 +36,6 @@ ostree_builtin_summary (int argc, char **argv, OstreeRepo *repo, GCancellable *c
 {
   gboolean ret = FALSE;
   GOptionContext *context;
-  gs_unref_ptrarray GPtrArray *delta_names = NULL;
 
   context = g_option_context_new ("Manage summary metadata");
   g_option_context_add_main_entries (context, options, NULL);
diff --git a/tests/test-pull-metalink.sh b/tests/test-pull-metalink.sh
index 56769b5..ccd17b7 100755
--- a/tests/test-pull-metalink.sh
+++ b/tests/test-pull-metalink.sh
@@ -62,5 +62,6 @@ mkdir repo
 ${CMD_PREFIX} ostree --repo=repo init
 ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin metalink=$(cat 
metalink-httpd-address)/metalink.xml
 ${CMD_PREFIX} ostree --repo=repo pull origin:main
+${CMD_PREFIX} ostree --repo=repo rev-parse origin:main
 ${CMD_PREFIX} ostree --repo=repo fsck
 echo "ok pull via metalink"
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]