GNOME.org
 

[ostree/wip/metalinks] Move sync fetches into pull code, add more tests

commit 9dedead2ceafe9d19f66fc2856400da294f1ba54
Author: Colin Walters <walters verbum org>
Date:   Tue Aug 19 23:26:55 2014 -0400

    Move sync fetches into pull code, add more tests

 src/libostree/ostree-metalink.c   |   54 +++---------------------------
 src/libostree/ostree-metalink.h   |    7 +---
 src/libostree/ostree-repo-pull.c  |   66 ++++++++++++++++++++++++++++++++++---
 src/libotutil/ot-checksum-utils.c |    2 +-
 tests/test-pull-metalink.sh       |   52 ++++++++++++++++++++++++++++-
 5 files changed, 120 insertions(+), 61 deletions(-)
---
diff --git a/src/libostree/ostree-metalink.c b/src/libostree/ostree-metalink.c
index 38fcde2..89a8435 100644
--- a/src/libostree/ostree-metalink.c
+++ b/src/libostree/ostree-metalink.c
@@ -339,9 +339,11 @@ metalink_parser_text (GMarkupParseContext *context,
           switch (self->in_verification_type)
             {
             case G_CHECKSUM_SHA256:
+              g_free (self->verification_sha256);
               self->verification_sha256 = g_strndup (text, text_len);
               break;
             case G_CHECKSUM_SHA512:
+              g_free (self->verification_sha512);
               self->verification_sha512 = g_strndup (text, text_len);
               break;
             default:
@@ -692,6 +694,7 @@ _ostree_metalink_request_finish (OstreeMetalink         *self,
 
   if (g_task_propagate_boolean ((GTask*)result, error))
     {
+      g_assert_cmpint (request->current_url_index, <, request->urls->len);
       *out_target_uri = request->urls->pdata[request->current_url_index];
       *out_data = g_object_ref (request->result);
       return TRUE;
@@ -700,53 +703,8 @@ _ostree_metalink_request_finish (OstreeMetalink         *self,
     return FALSE;
 }
 
-typedef struct
-{
-  gboolean                running;
-  gboolean                success;
-  SoupURI               **out_target_uri;
-  GFile                 **out_data;
-  GError                **error;
-} MetalinkSyncCallState;
-
-static void
-on_async_result (GObject          *src,
-                 GAsyncResult     *result,
-                 gpointer          user_data)
-{
-  MetalinkSyncCallState *state = user_data;
-
-  state->success = _ostree_metalink_request_finish ((OstreeMetalink*)src, result,
-                                                    state->out_target_uri, state->out_data,
-                                                    state->error);
-  state->running = FALSE;
-}
-
-gboolean
-_ostree_metalink_request_sync (OstreeMetalink         *self,
-                               SoupURI               **out_target_uri,
-                               GFile                 **out_data,
-                               GCancellable           *cancellable,
-                               GError                **error)
+SoupURI *
+_ostree_metalink_get_uri (OstreeMetalink        *self)
 {
-  gboolean ret = FALSE;
-  GMainContext *sync_context = g_main_context_new ();
-  MetalinkSyncCallState state = { 0, };
-
-  state.error = error;
-  state.running = TRUE;
-
-  g_main_context_push_thread_default (sync_context);
-  
-  _ostree_metalink_request_async (self, cancellable, on_async_result, &state);
-  
-  while (state.running)
-    g_main_context_iteration (sync_context, TRUE);
-
-  g_main_context_pop_thread_default (sync_context);
-
-  ret = state.success;
-  if (sync_context)
-    g_main_context_unref (sync_context);
-  return ret;
+  return self->uri;
 }
diff --git a/src/libostree/ostree-metalink.h b/src/libostree/ostree-metalink.h
index 09065b6..0c26ade 100644
--- a/src/libostree/ostree-metalink.h
+++ b/src/libostree/ostree-metalink.h
@@ -48,12 +48,7 @@ OstreeMetalink *_ostree_metalink_new (OstreeFetcher  *fetcher,
                                       guint64         max_size,
                                       SoupURI        *uri);
 
-gboolean _ostree_metalink_request_sync (OstreeMetalink         *self,
-                                        SoupURI               **out_target_uri,
-                                        GFile                 **out_data,
-                                        GCancellable           *cancellable,
-                                        GError                **error);
-
+SoupURI *_ostree_metalink_get_uri (OstreeMetalink         *self);
 
 void _ostree_metalink_request_async (OstreeMetalink         *self,
                                      GCancellable          *cancellable,
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index ba906cb..346f059 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -362,6 +362,52 @@ fetch_uri_contents_utf8_sync (OtPullData  *pull_data,
   return ret;
 }
 
+typedef struct
+{
+  OtPullData             *pull_data;
+  SoupURI               **out_target_uri;
+  GFile                 **out_data;
+  gboolean                running;
+  gboolean                success;
+} FetchMetalinkSyncData;
+
+static void
+on_metalink_fetched (GObject          *src,
+                     GAsyncResult     *result,
+                     gpointer          user_data)
+{
+  FetchMetalinkSyncData *data = user_data;
+
+  data->success = _ostree_metalink_request_finish ((OstreeMetalink*)src, result,
+                                                   data->out_target_uri, data->out_data,
+                                                   data->pull_data->async_error);
+  data->running = FALSE;
+}
+
+static gboolean
+request_metalink_sync (OtPullData             *pull_data,
+                       OstreeMetalink         *metalink,
+                       SoupURI               **out_target_uri,
+                       GFile                 **out_data,
+                       GCancellable           *cancellable,
+                       GError                **error)
+{
+  FetchMetalinkSyncData data = { 0, };
+
+  data.pull_data = pull_data;
+  data.out_target_uri = out_target_uri;
+  data.out_data = out_data;
+  data.running = TRUE;
+
+  pull_data->fetching_sync_uri = _ostree_metalink_get_uri (metalink);
+  _ostree_metalink_request_async (metalink, cancellable, on_metalink_fetched, &data);
+  
+  while (data.running)
+    g_main_context_iteration (NULL, TRUE);
+
+  return data.success;
+}
+
 static void
 enqueue_one_object_request (OtPullData        *pull_data,
                             const char        *checksum,
@@ -491,6 +537,7 @@ lookup_commit_checksum_from_summary (OtPullData    *pull_data,
   gs_unref_variant GVariant *commits = g_variant_get_child_value (pull_data->summary, 0);
   gs_unref_variant GVariant *refs = g_variant_get_child_value (pull_data->summary, 1);
   gs_unref_variant GVariant *refdata = NULL;
+  gs_unref_variant GVariant *reftargetdata = NULL;
   gs_unref_variant GVariant *commit_data = NULL;
   gs_unref_variant GVariant *commit_csum_v = NULL;
   gs_unref_bytes GBytes *commit_bytes = NULL;
@@ -505,7 +552,8 @@ lookup_commit_checksum_from_summary (OtPullData    *pull_data,
     }
       
   refdata = g_variant_get_child_value (refs, i);
-  commit_csum_v = g_variant_get_child_value (refdata, 0);
+  reftargetdata = g_variant_get_child_value (refdata, 1);
+  commit_csum_v = g_variant_get_child_value (reftargetdata, 0);
 
   if (!ostree_validate_structureof_csum_v (commit_csum_v, error))
     goto out;
@@ -1203,6 +1251,7 @@ ostree_repo_pull (OstreeRepo               *self,
     {
       gs_unref_object GFile *metalink_data = NULL;
       SoupURI *metalink_uri = soup_uri_new (metalink_url_str);
+      SoupURI *target_uri = NULL;
       
       if (!metalink_uri)
         {
@@ -1211,14 +1260,20 @@ ostree_repo_pull (OstreeRepo               *self,
           goto out;
         }
       
-      metalink = _ostree_metalink_new (pull_data->fetcher, "summary", OSTREE_MAX_METADATA_SIZE, 
metalink_uri);
+      metalink = _ostree_metalink_new (pull_data->fetcher, "summary",
+                                       OSTREE_MAX_METADATA_SIZE, metalink_uri);
       soup_uri_free (metalink_uri);
 
-      if (!_ostree_metalink_request_sync (metalink, &pull_data->base_uri,
-                                          &metalink_data,
-                                          cancellable, error))
+      if (!request_metalink_sync (pull_data, metalink, &target_uri, &metalink_data,
+                                  cancellable, error))
         goto out;
 
+      {
+        gs_free char *repo_base = g_path_get_dirname (soup_uri_get_path (target_uri));
+        pull_data->base_uri = soup_uri_copy (target_uri);
+        soup_uri_set_path (pull_data->base_uri, repo_base);
+      }
+
       if (!ot_util_variant_map (metalink_data, OSTREE_SUMMARY_GVARIANT_FORMAT, FALSE,
                                 &pull_data->summary, error))
         goto out;
@@ -1286,6 +1341,7 @@ ostree_repo_pull (OstreeRepo               *self,
         }
     }
 
+  g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch);
   while (g_hash_table_iter_next (&hash_iter, &key, &value))
     {
       const char *branch = key;
diff --git a/src/libotutil/ot-checksum-utils.c b/src/libotutil/ot-checksum-utils.c
index bf9b89f..b728992 100644
--- a/src/libotutil/ot-checksum-utils.c
+++ b/src/libotutil/ot-checksum-utils.c
@@ -146,7 +146,7 @@ ot_checksum_file (GFile          *file,
                   GError        **error)
 {
   GChecksum *checksum = NULL;
-  gs_free gchar *ret = NULL;
+  char *ret = NULL;
   gs_unref_object GInputStream *in = NULL;
 
   in = (GInputStream*)g_file_read (file, cancellable, error);
diff --git a/tests/test-pull-metalink.sh b/tests/test-pull-metalink.sh
index ccd17b7..69bfc2c 100755
--- a/tests/test-pull-metalink.sh
+++ b/tests/test-pull-metalink.sh
@@ -50,7 +50,8 @@ cat > ${test_tmpdir}/metalink-data/metalink.xml <<EOF
         <hash type="sha512">$(sha512sum ${summary_path} | cut -f 1 -d ' ')</hash>
       </verification>
       <resources maxconnections="1">
-        <url protocol="http" type="http" location="US" preference="100" >$(cat httpd-address)</url>
+        <url protocol="http" type="http" location="US" preference="100" >$(cat 
httpd-address)/ostree/gnomerepo/enoent</url>
+        <url protocol="http" type="http" location="US" preference="99" >$(cat 
httpd-address)/ostree/gnomerepo/summary</url>
       </resources>
     </file>
   </files>
@@ -65,3 +66,52 @@ ${CMD_PREFIX} ostree --repo=repo pull origin:main
 ${CMD_PREFIX} ostree --repo=repo rev-parse origin:main
 ${CMD_PREFIX} ostree --repo=repo fsck
 echo "ok pull via metalink"
+
+cp metalink-data/metalink.xml{,.orig}
+cp ostree-srv/gnomerepo/summary{,.orig}
+
+test_metalink_pull_error() {
+    msg=$1
+    rm repo -rf
+    mkdir repo
+    ${CMD_PREFIX} ostree --repo=repo init
+    ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin metalink=$(cat 
metalink-httpd-address)/metalink.xml
+    if ${CMD_PREFIX} ostree --repo=repo pull origin:main 2>err.txt; then
+       assert_not_reached "pull unexpectedly succeeded"
+    fi
+    cat err.txt
+    assert_file_has_content err.txt "${msg}"
+}
+
+cd ${test_tmpdir}
+sed -e 's,<hash type="sha512">.*</hash>,<hash type="sha512">bacon</hash>,' < metalink-data/metalink.xml.orig 

metalink-data/metalink.xml

+test_metalink_pull_error "Invalid hash digest for sha512"
+echo "ok metalink err hash format"
+
+cd ${test_tmpdir}
+sed -e 's,<hash type="sha512">.*</hash>,<hash type="sha512">'$( (echo -n dummy; cat ${summary_path}) | 
sha512sum | cut -f 1 -d ' ')'</hash>,' < metalink-data/metalink.xml.orig > metalink-data/metalink.xml
+test_metalink_pull_error "Expected checksum is .* but actual is"
+echo "ok metalink err hash sha512"
+
+cd ${test_tmpdir}
+cp metalink-data/metalink.xml.orig metalink-data/metalink.xml
+echo -n moo > ostree-srv/gnomerepo/summary
+test_metalink_pull_error "Expected size is .* bytes but content is 3 bytes"
+echo "ok metalink err size"
+cp ostree-srv/gnomerepo/summary{.orig,}
+
+cd ${test_tmpdir}
+grep -v sha256 < metalink-data/metalink.xml.orig |grep -v sha512 > metalink-data/metalink.xml
+test_metalink_pull_error "No.*verification.*with known.*hash"
+echo "ok metalink err no verify"
+
+cd ${test_tmpdir}
+grep -v '<url protocol' < metalink-data/metalink.xml.orig > metalink-data/metalink.xml
+test_metalink_pull_error "No.*url.*method.*elements"
+echo "ok metalink err no url"
+
+cd ${test_tmpdir}
+echo bacon > metalink-data/metalink.xml
+test_metalink_pull_error "Document must begin with an element"
+echo "ok metalink err malformed"
+
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]