[ostree] Add repository "summary" file and metalink support
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree] Add repository "summary" file and metalink support
- Date: Sun, 7 Sep 2014 16:58:45 +0000 (UTC)
commit f8f5da219edd2279322bba916879fd53c2b65350
Author: Colin Walters <walters verbum org>
Date: Thu Jul 31 18:50:19 2014 -0400
Add repository "summary" file and metalink support
For Fedora and potentially other distributions which use globally
distributed mirrors, metalink is a popular solution to redirect
clients to a dynamic set of mirrors.
In order to make metalink work though, it needs *one* file which can
be checksummed. (Well, potentially we could explode all refs into the
metalink.xml, but that would be a lot more invasive, and a bit weird
as we'd end up checksumming the checksum file).
This commit adds a new command:
$ ostree summary -u
To regenerate the summary file. Can only be run by one process at a
time.
After that's done, the metalink can be generated based on it, and the
client fetch code will parse and load it.
https://bugzilla.gnome.org/show_bug.cgi?id=729585
Makefile-libostree.am | 2 +
Makefile-ostree.am | 1 +
Makefile-tests.am | 1 +
src/libostree/ostree-core.h | 9 +
src/libostree/ostree-metalink.c | 710 +++++++++++++++++++++++++++++++++++++
src/libostree/ostree-metalink.h | 66 ++++
src/libostree/ostree-repo-pull.c | 211 ++++++++++--
src/libostree/ostree-repo.c | 86 +++++-
src/libostree/ostree-repo.h | 6 +
src/libotutil/ot-checksum-utils.c | 26 ++
src/libotutil/ot-checksum-utils.h | 5 +
src/libotutil/ot-variant-utils.c | 64 ++++
src/libotutil/ot-variant-utils.h | 7 +
src/ostree/main.c | 1 +
src/ostree/ot-builtin-summary.c | 63 ++++
src/ostree/ot-builtins.h | 1 +
tests/test-pull-metalink.sh | 120 +++++++
17 files changed, 1352 insertions(+), 27 deletions(-)
---
diff --git a/Makefile-libostree.am b/Makefile-libostree.am
index 1b75600..e7b141a 100644
--- a/Makefile-libostree.am
+++ b/Makefile-libostree.am
@@ -112,6 +112,8 @@ if USE_LIBSOUP
libostree_1_la_SOURCES += \
src/libostree/ostree-fetcher.h \
src/libostree/ostree-fetcher.c \
+ src/libostree/ostree-metalink.h \
+ src/libostree/ostree-metalink.c \
src/libostree/ostree-repo-pull.c \
$(NULL)
libostree_1_la_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS)
diff --git a/Makefile-ostree.am b/Makefile-ostree.am
index f1381ac..76df368 100644
--- a/Makefile-ostree.am
+++ b/Makefile-ostree.am
@@ -40,6 +40,7 @@ ostree_SOURCES = src/ostree/main.c \
src/ostree/ot-builtin-remote.c \
src/ostree/ot-builtin-reset.c \
src/ostree/ot-builtin-rev-parse.c \
+ src/ostree/ot-builtin-summary.c \
src/ostree/ot-builtin-show.c \
src/ostree/ot-builtin-static-delta.c \
src/ostree/ot-main.h \
diff --git a/Makefile-tests.am b/Makefile-tests.am
index c36b1b8..03a2ff6 100644
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -31,6 +31,7 @@ testfiles = test-basic \
test-pull-archive-z \
test-pull-corruption \
test-pull-large-metadata \
+ test-pull-metalink \
test-pull-resume \
test-gpg-signed-commit \
test-admin-deploy-syslinux \
diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h
index a867dbe..e54e45d 100644
--- a/src/libostree/ostree-core.h
+++ b/src/libostree/ostree-core.h
@@ -118,6 +118,15 @@ typedef enum {
#define OSTREE_COMMIT_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_COMMIT_GVARIANT_STRING)
/**
+ * OSTREE_SUMMARY_GVARIANT_FORMAT:
+ *
+ * refs: a(s(taya{sv})) - Map of ref name -> (latest commit size, latest commit checksum, additional
metadata), sorted by ref name
+ * extensions: a{sv} - Additional metadata, none defined at the current time
+ */
+#define OSTREE_SUMMARY_GVARIANT_STRING "(a(s(taya{sv}))a{sv})"
+#define OSTREE_SUMMARY_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_SUMMARY_GVARIANT_STRING)
+
+/**
* OstreeRepoMode:
* @OSTREE_REPO_MODE_BARE: Files are stored as themselves; can only be written as root
* @OSTREE_REPO_MODE_ARCHIVE_Z2: Files are compressed, should be owned by non-root. Can be served via HTTP
diff --git a/src/libostree/ostree-metalink.c b/src/libostree/ostree-metalink.c
new file mode 100644
index 0000000..00cfb2c
--- /dev/null
+++ b/src/libostree/ostree-metalink.c
@@ -0,0 +1,710 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2014 Colin Walters <walters verbum org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include "ostree-metalink.h"
+
+#include "otutil.h"
+#include "libgsystem.h"
+
+typedef enum {
+ OSTREE_METALINK_STATE_INITIAL,
+ OSTREE_METALINK_STATE_METALINK,
+ OSTREE_METALINK_STATE_FILES,
+ OSTREE_METALINK_STATE_FILE,
+ OSTREE_METALINK_STATE_SIZE,
+ OSTREE_METALINK_STATE_VERIFICATION,
+ OSTREE_METALINK_STATE_HASH,
+ OSTREE_METALINK_STATE_RESOURCES,
+ OSTREE_METALINK_STATE_URL,
+
+ OSTREE_METALINK_STATE_PASSTHROUGH /* Ignoring unknown elements */
+} OstreeMetalinkState;
+
+struct OstreeMetalink
+{
+ GObject parent_instance;
+
+ SoupURI *uri;
+
+ OstreeFetcher *fetcher;
+ char *requested_file;
+ guint64 max_size;
+};
+
+G_DEFINE_TYPE (OstreeMetalink, _ostree_metalink, G_TYPE_OBJECT)
+
+typedef struct
+{
+ OstreeMetalink *metalink;
+
+ GTask *task;
+ GMarkupParseContext *parser;
+
+ guint passthrough_depth;
+ OstreeMetalinkState passthrough_previous;
+
+ guint found_a_file_element : 1;
+ guint found_our_file_element : 1;
+ guint verification_known : 1;
+
+ GChecksumType in_verification_type;
+
+ guint64 size;
+ char *verification_sha256;
+ char *verification_sha512;
+
+ GFile *result;
+
+ char *last_metalink_error;
+ guint current_url_index;
+ GPtrArray *urls;
+
+ OstreeMetalinkState state;
+} OstreeMetalinkRequest;
+
+static void
+state_transition (OstreeMetalinkRequest *self,
+ OstreeMetalinkState new_state)
+{
+ g_assert (self->state != new_state);
+ self->state = new_state;
+}
+
+static void
+unknown_element (OstreeMetalinkRequest *self,
+ const char *element_name,
+ GError **error)
+{
+ state_transition (self, OSTREE_METALINK_STATE_PASSTHROUGH);
+ g_assert (self->passthrough_depth == 0);
+}
+
+static void
+metalink_parser_start (GMarkupParseContext *context,
+ const gchar *element_name,
+ const gchar **attribute_names,
+ const gchar **attribute_values,
+ gpointer user_data,
+ GError **error)
+{
+ GTask *task = user_data;
+ OstreeMetalinkRequest *self = g_task_get_task_data (task);
+
+ switch (self->state)
+ {
+ case OSTREE_METALINK_STATE_INITIAL:
+ if (strcmp (element_name, "metalink") == 0)
+ state_transition (self, OSTREE_METALINK_STATE_METALINK);
+ else
+ unknown_element (self, element_name, error);
+ break;
+ case OSTREE_METALINK_STATE_METALINK:
+ if (strcmp (element_name, "files") == 0)
+ state_transition (self, OSTREE_METALINK_STATE_FILES);
+ else
+ unknown_element (self, element_name, error);
+ break;
+ case OSTREE_METALINK_STATE_FILES:
+ /* If we've already processed a <file> element we're OK with, just
+ * ignore the others.
+ */
+ if (self->urls->len > 0)
+ {
+ state_transition (self, OSTREE_METALINK_STATE_PASSTHROUGH);
+ }
+ else if (strcmp (element_name, "file") == 0)
+ {
+ const char *file_name;
+
+ if (!g_markup_collect_attributes (element_name,
+ attribute_names,
+ attribute_values,
+ error,
+ G_MARKUP_COLLECT_STRING,
+ "name",
+ &file_name,
+ G_MARKUP_COLLECT_INVALID))
+ goto out;
+
+ self->found_a_file_element = TRUE;
+
+ if (strcmp (file_name, self->metalink->requested_file) != 0)
+ {
+ state_transition (self, OSTREE_METALINK_STATE_PASSTHROUGH);
+ g_assert (self->passthrough_depth == 0);
+ }
+ else
+ {
+ self->found_our_file_element = TRUE;
+ state_transition (self, OSTREE_METALINK_STATE_FILE);
+ }
+ }
+ else
+ unknown_element (self, element_name, error);
+ break;
+ case OSTREE_METALINK_STATE_FILE:
+ if (strcmp (element_name, "size") == 0)
+ state_transition (self, OSTREE_METALINK_STATE_SIZE);
+ else if (strcmp (element_name, "verification") == 0)
+ state_transition (self, OSTREE_METALINK_STATE_VERIFICATION);
+ else if (strcmp (element_name, "resources") == 0)
+ state_transition (self, OSTREE_METALINK_STATE_RESOURCES);
+ else
+ unknown_element (self, element_name, error);
+ break;
+ case OSTREE_METALINK_STATE_SIZE:
+ unknown_element (self, element_name, error);
+ break;
+ case OSTREE_METALINK_STATE_VERIFICATION:
+ if (strcmp (element_name, "hash") == 0)
+ {
+ char *verification_type_str = NULL;
+
+ state_transition (self, OSTREE_METALINK_STATE_HASH);
+ if (!g_markup_collect_attributes (element_name,
+ attribute_names,
+ attribute_values,
+ error,
+ G_MARKUP_COLLECT_STRING,
+ "type",
+ &verification_type_str,
+ G_MARKUP_COLLECT_INVALID))
+ goto out;
+
+ /* Only accept sha256/sha512 */
+ self->verification_known = TRUE;
+ if (strcmp (verification_type_str, "sha256") == 0)
+ self->in_verification_type = G_CHECKSUM_SHA256;
+ else if (strcmp (verification_type_str, "sha512") == 0)
+ self->in_verification_type = G_CHECKSUM_SHA512;
+ else
+ self->verification_known = FALSE;
+ }
+ else
+ unknown_element (self, element_name, error);
+ break;
+ case OSTREE_METALINK_STATE_HASH:
+ unknown_element (self, element_name, error);
+ break;
+ case OSTREE_METALINK_STATE_RESOURCES:
+ if (self->size == 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "No <size> element found or it is zero");
+ goto out;
+ }
+ if (!self->verification_known)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "No <verification> element with known <hash type=> found");
+ goto out;
+ }
+
+ if (strcmp (element_name, "url") == 0)
+ {
+ const char *protocol;
+
+ if (!g_markup_collect_attributes (element_name,
+ attribute_names,
+ attribute_values,
+ error,
+ G_MARKUP_COLLECT_STRING,
+ "protocol",
+ &protocol,
+ G_MARKUP_COLLECT_STRING,
+ "type",
+ NULL,
+ G_MARKUP_COLLECT_STRING,
+ "location",
+ NULL,
+ G_MARKUP_COLLECT_STRING,
+ "preference",
+ NULL,
+ G_MARKUP_COLLECT_INVALID))
+ goto out;
+
+ /* Ignore non-HTTP resources */
+ if (!(strcmp (protocol, "http") == 0 || strcmp (protocol, "https") == 0))
+ state_transition (self, OSTREE_METALINK_STATE_PASSTHROUGH);
+ else
+ state_transition (self, OSTREE_METALINK_STATE_URL);
+ }
+ else
+ unknown_element (self, element_name, error);
+ break;
+ case OSTREE_METALINK_STATE_URL:
+ unknown_element (self, element_name, error);
+ break;
+ case OSTREE_METALINK_STATE_PASSTHROUGH:
+ self->passthrough_depth++;
+ break;
+ }
+
+ out:
+ return;
+}
+
+static void
+metalink_parser_end (GMarkupParseContext *context,
+ const gchar *element_name,
+ gpointer user_data,
+ GError **error)
+{
+ GTask *task = user_data;
+ OstreeMetalinkRequest *self = g_task_get_task_data (task);
+
+ switch (self->state)
+ {
+ case OSTREE_METALINK_STATE_INITIAL:
+ break;
+ case OSTREE_METALINK_STATE_METALINK:
+ state_transition (self, OSTREE_METALINK_STATE_INITIAL);
+ break;
+ case OSTREE_METALINK_STATE_FILES:
+ state_transition (self, OSTREE_METALINK_STATE_METALINK);
+ break;
+ case OSTREE_METALINK_STATE_FILE:
+ state_transition (self, OSTREE_METALINK_STATE_FILES);
+ break;
+ case OSTREE_METALINK_STATE_SIZE:
+ case OSTREE_METALINK_STATE_VERIFICATION:
+ case OSTREE_METALINK_STATE_RESOURCES:
+ state_transition (self, OSTREE_METALINK_STATE_FILE);
+ break;
+ case OSTREE_METALINK_STATE_HASH:
+ state_transition (self, OSTREE_METALINK_STATE_VERIFICATION);
+ break;
+ case OSTREE_METALINK_STATE_URL:
+ state_transition (self, OSTREE_METALINK_STATE_RESOURCES);
+ break;
+ case OSTREE_METALINK_STATE_PASSTHROUGH:
+ g_assert_cmpint (self->passthrough_depth, >, 0);
+ self->passthrough_depth--;
+ if (self->passthrough_depth == 0)
+ state_transition (self, self->passthrough_previous);
+ break;
+ }
+}
+
+static void
+metalink_parser_text (GMarkupParseContext *context,
+ const gchar *text,
+ gsize text_len,
+ gpointer user_data,
+ GError **error)
+{
+ GTask *task = user_data;
+ OstreeMetalinkRequest *self = g_task_get_task_data (task);
+
+ switch (self->state)
+ {
+ case OSTREE_METALINK_STATE_INITIAL:
+ break;
+ case OSTREE_METALINK_STATE_METALINK:
+ break;
+ case OSTREE_METALINK_STATE_FILES:
+ break;
+ case OSTREE_METALINK_STATE_FILE:
+ break;
+ case OSTREE_METALINK_STATE_SIZE:
+ {
+ gs_free char *duped = g_strndup (text, text_len);
+ self->size = g_ascii_strtoull (duped, NULL, 10);
+ }
+ break;
+ case OSTREE_METALINK_STATE_VERIFICATION:
+ break;
+ case OSTREE_METALINK_STATE_HASH:
+ if (self->verification_known)
+ {
+ switch (self->in_verification_type)
+ {
+ case G_CHECKSUM_SHA256:
+ g_free (self->verification_sha256);
+ self->verification_sha256 = g_strndup (text, text_len);
+ break;
+ case G_CHECKSUM_SHA512:
+ g_free (self->verification_sha512);
+ self->verification_sha512 = g_strndup (text, text_len);
+ break;
+ default:
+ g_assert_not_reached ();
+ }
+ }
+ break;
+ case OSTREE_METALINK_STATE_RESOURCES:
+ break;
+ case OSTREE_METALINK_STATE_URL:
+ {
+ gs_free char *uri_text = g_strndup (text, text_len);
+ SoupURI *uri = soup_uri_new (uri_text);
+ if (uri != NULL)
+ g_ptr_array_add (self->urls, uri);
+ }
+ break;
+ case OSTREE_METALINK_STATE_PASSTHROUGH:
+ break;
+ }
+
+}
+
+static void
+_ostree_metalink_finalize (GObject *object)
+{
+ OstreeMetalink *self;
+
+ self = OSTREE_METALINK (object);
+
+ g_object_unref (self->fetcher);
+ g_free (self->requested_file);
+ soup_uri_free (self->uri);
+
+ G_OBJECT_CLASS (_ostree_metalink_parent_class)->finalize (object);
+}
+
+static void
+_ostree_metalink_class_init (OstreeMetalinkClass *klass)
+{
+ GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
+
+ gobject_class->finalize = _ostree_metalink_finalize;
+}
+
+static void
+_ostree_metalink_init (OstreeMetalink *self)
+{
+}
+
+OstreeMetalink *
+_ostree_metalink_new (OstreeFetcher *fetcher,
+ const char *requested_file,
+ guint64 max_size,
+ SoupURI *uri)
+{
+ OstreeMetalink *self = (OstreeMetalink*)g_object_new (OSTREE_TYPE_METALINK, NULL);
+
+ self->fetcher = g_object_ref (fetcher);
+ self->requested_file = g_strdup (requested_file);
+ self->max_size = max_size;
+ self->uri = soup_uri_copy (uri);
+
+ return self;
+}
+
+static void
+try_next_url (OstreeMetalinkRequest *self);
+
+static gboolean
+valid_hex_checksum (const char *s, gsize expected_len)
+{
+ gsize len = strspn (s, "01234567890abcdef");
+
+ return len == expected_len && s[len] == '\0';
+}
+
+static void
+on_fetched_url (GObject *src,
+ GAsyncResult *res,
+ gpointer user_data)
+{
+ GTask *task = user_data;
+ OstreeMetalinkRequest *self = g_task_get_task_data (task);
+ GError *local_error = NULL;
+ gs_unref_object GFile *result = NULL;
+ gs_unref_object GFileInfo *finfo = NULL;
+
+ result = _ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)src, res, &local_error);
+ if (!result)
+ goto out;
+
+ finfo = g_file_query_info (result, OSTREE_GIO_FAST_QUERYINFO, 0,
+ g_task_get_cancellable (task), &local_error);
+ if (!finfo)
+ goto out;
+
+ if (g_file_info_get_size (finfo) != self->size)
+ {
+ g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Expected size is %" G_GUINT64_FORMAT " bytes but content is %" G_GUINT64_FORMAT " bytes",
+ self->size, g_file_info_get_size (finfo));
+ goto out;
+ }
+
+ if (self->verification_sha512)
+ {
+ gs_free char *actual = ot_checksum_file (result, G_CHECKSUM_SHA512,
+ g_task_get_cancellable (task),
+ &local_error);
+
+ if (!actual)
+ goto out;
+
+ if (strcmp (self->verification_sha512, actual) != 0)
+ {
+ g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Expected checksum is %s but actual is %s",
+ self->verification_sha512, actual);
+ goto out;
+ }
+ }
+
+ if (self->verification_sha256)
+ {
+ gs_free char *actual = ot_checksum_file (result, G_CHECKSUM_SHA256,
+ g_task_get_cancellable (task),
+ &local_error);
+
+ if (!actual)
+ goto out;
+
+ if (strcmp (self->verification_sha256, actual) != 0)
+ {
+ g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Expected checksum is %s but actual is %s",
+ self->verification_sha256, actual);
+ goto out;
+ }
+ }
+
+ out:
+ if (local_error)
+ {
+ g_free (self->last_metalink_error);
+ self->last_metalink_error = g_strdup (local_error->message);
+ g_clear_error (&local_error);
+
+ /* And here we iterate on the next one if we hit an error */
+ self->current_url_index++;
+ try_next_url (self);
+ }
+ else
+ {
+ self->result = g_object_ref (result);
+ g_task_return_boolean (self->task, TRUE);
+ }
+}
+
+static void
+try_next_url (OstreeMetalinkRequest *self)
+{
+ if (self->current_url_index >= self->urls->len)
+ {
+ g_task_return_new_error (self->task, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Exhausted %u metalink targets, last error: %s",
+ self->urls->len, self->last_metalink_error);
+ }
+ else
+ {
+ SoupURI *next = self->urls->pdata[self->current_url_index];
+
+ _ostree_fetcher_request_uri_with_partial_async (self->metalink->fetcher, next,
+ self->metalink->max_size,
+ g_task_get_cancellable (self->task),
+ on_fetched_url, self->task);
+ }
+}
+
+static gboolean
+start_target_request_phase (OstreeMetalinkRequest *self,
+ GError **error)
+{
+ gboolean ret = FALSE;
+
+ if (!self->found_a_file_element)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "No <file> element found");
+ goto out;
+ }
+
+ if (!self->found_our_file_element)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "No <file name='%s'> found", self->metalink->requested_file);
+ goto out;
+ }
+
+ if (!(self->verification_sha256 || self->verification_sha512))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "No <verification> hash for sha256 or sha512 found");
+ goto out;
+ }
+
+ if (self->verification_sha256 && !valid_hex_checksum (self->verification_sha256, 64))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid hash digest for sha256");
+ goto out;
+ }
+
+ if (self->verification_sha512 && !valid_hex_checksum (self->verification_sha512, 128))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid hash digest for sha512");
+ goto out;
+ }
+
+ if (self->urls->len == 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "No <url method='http'> elements found");
+ goto out;
+ }
+
+ try_next_url (self);
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+static void
+on_metalink_bytes_read (GObject *src,
+ GAsyncResult *result,
+ gpointer user_data)
+{
+ GError *local_error = NULL;
+ GTask *task = user_data;
+ OstreeMetalinkRequest *self = g_task_get_task_data (task);
+ gs_unref_bytes GBytes *bytes = NULL;
+ gsize len;
+ const guint8 *data;
+
+ bytes = g_input_stream_read_bytes_finish ((GInputStream*)src,
+ result, &local_error);
+ if (!bytes)
+ goto out;
+
+ data = g_bytes_get_data (bytes, &len);
+
+ if (len == 0)
+ {
+ if (!start_target_request_phase (self, &local_error))
+ goto out;
+ }
+ else
+ {
+ if (!g_markup_parse_context_parse (self->parser, (const char*)data, len, &local_error))
+ goto out;
+
+ g_input_stream_read_bytes_async ((GInputStream*)src, 8192, G_PRIORITY_DEFAULT,
+ g_task_get_cancellable (task),
+ on_metalink_bytes_read, task);
+ }
+
+ out:
+ if (local_error)
+ g_task_return_error (task, local_error);
+}
+
+static void
+on_retrieved_metalink (GObject *src,
+ GAsyncResult *result,
+ gpointer user_data)
+{
+ GError *local_error = NULL;
+ GTask *task = user_data;
+ gs_unref_object GInputStream *metalink_stream = NULL;
+
+ metalink_stream = _ostree_fetcher_stream_uri_finish ((OstreeFetcher*)src, result, &local_error);
+ if (!metalink_stream)
+ goto out;
+
+ g_input_stream_read_bytes_async (metalink_stream, 8192, G_PRIORITY_DEFAULT,
+ g_task_get_cancellable (task),
+ on_metalink_bytes_read, task);
+
+ out:
+ if (local_error)
+ g_task_return_error (task, local_error);
+}
+
+static void
+ostree_metalink_request_unref (gpointer data)
+{
+ OstreeMetalinkRequest *request = data;
+ g_object_unref (request->metalink);
+ g_clear_object (&request->result);
+ g_free (request->last_metalink_error);
+ g_ptr_array_unref (request->urls);
+ g_free (request);
+}
+
+static const GMarkupParser metalink_parser = {
+ metalink_parser_start,
+ metalink_parser_end,
+ metalink_parser_text,
+ NULL,
+ NULL
+};
+
+void
+_ostree_metalink_request_async (OstreeMetalink *self,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data)
+{
+ GTask *task = g_task_new (self, cancellable, callback, user_data);
+ OstreeMetalinkRequest *request = g_new0 (OstreeMetalinkRequest, 1);
+
+ request->metalink = g_object_ref (self);
+ request->urls = g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free);
+ request->task = task; /* Unowned */
+
+ request->parser = g_markup_parse_context_new (&metalink_parser, G_MARKUP_PREFIX_ERROR_POSITION, task,
NULL);
+
+ g_task_set_task_data (task, request, ostree_metalink_request_unref);
+ _ostree_fetcher_stream_uri_async (self->fetcher, self->uri,
+ self->max_size, cancellable,
+ on_retrieved_metalink, task);
+}
+
+gboolean
+_ostree_metalink_request_finish (OstreeMetalink *self,
+ GAsyncResult *result,
+ SoupURI **out_target_uri,
+ GFile **out_data,
+ GError **error)
+{
+ OstreeMetalinkRequest *request;
+
+ g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
+
+ request = g_task_get_task_data ((GTask*)result);
+
+ if (g_task_propagate_boolean ((GTask*)result, error))
+ {
+ g_assert_cmpint (request->current_url_index, <, request->urls->len);
+ *out_target_uri = request->urls->pdata[request->current_url_index];
+ *out_data = g_object_ref (request->result);
+ return TRUE;
+ }
+ else
+ return FALSE;
+}
+
+SoupURI *
+_ostree_metalink_get_uri (OstreeMetalink *self)
+{
+ return self->uri;
+}
diff --git a/src/libostree/ostree-metalink.h b/src/libostree/ostree-metalink.h
new file mode 100644
index 0000000..0c26ade
--- /dev/null
+++ b/src/libostree/ostree-metalink.h
@@ -0,0 +1,66 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2014 Colin Walters <walters verbum org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#pragma once
+
+#ifndef __GI_SCANNER__
+
+#include "ostree-fetcher.h"
+
+G_BEGIN_DECLS
+
+#define OSTREE_TYPE_METALINK (_ostree_metalink_get_type ())
+#define OSTREE_METALINK(o) (G_TYPE_CHECK_INSTANCE_CAST ((o), OSTREE_TYPE_METALINK, OstreeMetalink))
+#define OSTREE_METALINK_CLASS(k) (G_TYPE_CHECK_CLASS_CAST((k), OSTREE_TYPE_METALINK,
OstreeMetalinkClass))
+#define OSTREE_IS_METALINK(o) (G_TYPE_CHECK_INSTANCE_TYPE ((o), OSTREE_TYPE_METALINK))
+#define OSTREE_IS_METALINK_CLASS(k) (G_TYPE_CHECK_CLASS_TYPE ((k), OSTREE_TYPE_METALINK))
+#define OSTREE_METALINK_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS ((o), OSTREE_TYPE_METALINK,
OstreeMetalinkClass))
+
+typedef struct OstreeMetalinkClass OstreeMetalinkClass;
+typedef struct OstreeMetalink OstreeMetalink;
+
+struct OstreeMetalinkClass
+{
+ GObjectClass parent_class;
+};
+
+GType _ostree_metalink_get_type (void) G_GNUC_CONST;
+
+OstreeMetalink *_ostree_metalink_new (OstreeFetcher *fetcher,
+ const char *requested_file,
+ guint64 max_size,
+ SoupURI *uri);
+
+SoupURI *_ostree_metalink_get_uri (OstreeMetalink *self);
+
+void _ostree_metalink_request_async (OstreeMetalink *self,
+ GCancellable *cancellable,
+ GAsyncReadyCallback callback,
+ gpointer user_data);
+
+gboolean _ostree_metalink_request_finish (OstreeMetalink *self,
+ GAsyncResult *result,
+ SoupURI **out_target_uri,
+ GFile **out_data,
+ GError **error);
+
+G_END_DECLS
+
+#endif
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index 0ad9157..273c963 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -26,7 +26,7 @@
#include "ostree-core-private.h"
#include "ostree-repo-private.h"
#include "ostree-repo-static-delta-private.h"
-#include "ostree-fetcher.h"
+#include "ostree-metalink.h"
#include "otutil.h"
typedef struct {
@@ -52,7 +52,9 @@ typedef struct {
gboolean gpg_verify;
+ GVariant *summary;
GPtrArray *static_delta_metas;
+ GHashTable *expected_commit_sizes; /* Maps commit checksum to known size */
GHashTable *scanned_metadata; /* Maps object name to itself */
GHashTable *requested_metadata; /* Maps object name to itself */
GHashTable *requested_content; /* Maps object name to itself */
@@ -368,6 +370,49 @@ fetch_uri_contents_utf8_sync (OtPullData *pull_data,
return ret;
}
+typedef struct
+{
+ OtPullData *pull_data;
+ SoupURI **out_target_uri;
+ GFile **out_data;
+ gboolean success;
+} FetchMetalinkSyncData;
+
+static void
+on_metalink_fetched (GObject *src,
+ GAsyncResult *result,
+ gpointer user_data)
+{
+ FetchMetalinkSyncData *data = user_data;
+
+ data->success = _ostree_metalink_request_finish ((OstreeMetalink*)src, result,
+ data->out_target_uri, data->out_data,
+ data->pull_data->async_error);
+ g_main_loop_quit (data->pull_data->loop);
+}
+
+static gboolean
+request_metalink_sync (OtPullData *pull_data,
+ OstreeMetalink *metalink,
+ SoupURI **out_target_uri,
+ GFile **out_data,
+ GCancellable *cancellable,
+ GError **error)
+{
+ FetchMetalinkSyncData data = { 0, };
+
+ data.pull_data = pull_data;
+ data.out_target_uri = out_target_uri;
+ data.out_data = out_data;
+
+ pull_data->fetching_sync_uri = _ostree_metalink_get_uri (metalink);
+ _ostree_metalink_request_async (metalink, cancellable, on_metalink_fetched, &data);
+
+ run_mainloop_monitor_fetcher (pull_data);
+
+ return data.success;
+}
+
static void
enqueue_one_object_request (OtPullData *pull_data,
const char *checksum,
@@ -519,6 +564,45 @@ fetch_ref_contents (OtPullData *pull_data,
return ret;
}
+static gboolean
+lookup_commit_checksum_from_summary (OtPullData *pull_data,
+ const char *ref,
+ char **out_checksum,
+ gsize *out_size,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_unref_variant GVariant *refs = g_variant_get_child_value (pull_data->summary, 0);
+ gs_unref_variant GVariant *refdata = NULL;
+ gs_unref_variant GVariant *reftargetdata = NULL;
+ gs_unref_variant GVariant *commit_data = NULL;
+ guint64 commit_size;
+ gs_unref_variant GVariant *commit_csum_v = NULL;
+ gs_unref_bytes GBytes *commit_bytes = NULL;
+ int i;
+
+ if (!ot_variant_bsearch_str (refs, ref, &i))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "No such branch '%s' in repository summary",
+ ref);
+ goto out;
+ }
+
+ refdata = g_variant_get_child_value (refs, i);
+ reftargetdata = g_variant_get_child_value (refdata, 1);
+ g_variant_get (reftargetdata, "(t ay@a{sv})", &commit_size, &commit_csum_v, NULL);
+
+ if (!ostree_validate_structureof_csum_v (commit_csum_v, error))
+ goto out;
+
+ ret = TRUE;
+ *out_checksum = ostree_checksum_from_bytes_v (commit_csum_v);
+ *out_size = commit_size;
+ out:
+ return ret;
+}
+
static void
content_fetch_on_write_complete (GObject *object,
GAsyncResult *result,
@@ -690,7 +774,8 @@ meta_fetch_on_complete (GObject *object,
GError **error = &local_error;
ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype);
- g_debug ("fetch of %s complete", ostree_object_to_string (checksum, objtype));
+ g_debug ("fetch of %s%s complete", ostree_object_to_string (checksum, objtype),
+ fetch_data->is_detached_meta ? " (detached)" : "");
temp_path = _ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)object, result, error);
if (!temp_path)
@@ -926,9 +1011,12 @@ enqueue_one_object_request (OtPullData *pull_data,
gboolean is_meta;
FetchObjectData *fetch_data;
gs_free char *objpath = NULL;
+ guint64 *expected_max_size_p;
+ guint64 expected_max_size;
- g_debug ("queuing fetch of %s.%s", checksum,
- ostree_object_type_to_string (objtype));
+ g_debug ("queuing fetch of %s.%s%s", checksum,
+ ostree_object_type_to_string (objtype),
+ is_detached_meta ? " (detached)" : "");
if (is_detached_meta)
{
@@ -958,10 +1046,19 @@ enqueue_one_object_request (OtPullData *pull_data,
fetch_data->pull_data = pull_data;
fetch_data->object = ostree_object_name_serialize (checksum, objtype);
fetch_data->is_detached_meta = is_detached_meta;
+
+ expected_max_size_p = g_hash_table_lookup (pull_data->expected_commit_sizes, checksum);
+ if (expected_max_size_p)
+ expected_max_size = *expected_max_size_p;
+ else if (is_meta)
+ expected_max_size = OSTREE_MAX_METADATA_SIZE;
+ else
+ expected_max_size = 0;
+
_ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, obj_uri,
- is_meta ? OSTREE_MAX_METADATA_SIZE : 0,
- pull_data->cancellable,
- is_meta ? meta_fetch_on_complete :
content_fetch_on_complete, fetch_data);
+ expected_max_size,
+ pull_data->cancellable,
+ is_meta ? meta_fetch_on_complete :
content_fetch_on_complete, fetch_data);
soup_uri_free (obj_uri);
}
@@ -1124,9 +1221,11 @@ ostree_repo_pull_one_dir (OstreeRepo *self,
gs_free char *remote_key = NULL;
gs_free char *path = NULL;
gs_free char *baseurl = NULL;
+ gs_free char *metalink_url_str = NULL;
gs_unref_hashtable GHashTable *requested_refs_to_fetch = NULL;
gs_unref_hashtable GHashTable *commits_to_fetch = NULL;
gs_free char *remote_mode_str = NULL;
+ gs_unref_object OstreeMetalink *metalink = NULL;
OtPullData pull_data_real = { 0, };
OtPullData *pull_data = &pull_data_real;
GKeyFile *config = NULL;
@@ -1146,6 +1245,9 @@ ostree_repo_pull_one_dir (OstreeRepo *self,
pull_data->repo = self;
pull_data->progress = progress;
+ pull_data->expected_commit_sizes = g_hash_table_new_full (g_str_hash, g_str_equal,
+ (GDestroyNotify)g_free,
+ (GDestroyNotify)g_free);
pull_data->scanned_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
(GDestroyNotify)g_variant_unref, NULL);
pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal,
@@ -1167,10 +1269,6 @@ ostree_repo_pull_one_dir (OstreeRepo *self,
remote_key);
goto out;
}
- if (!repo_get_string_key_inherit (self, remote_key, "url", &baseurl, error))
- goto out;
-
- pull_data->base_uri = soup_uri_new (baseurl);
#ifdef HAVE_GPGME
if (!ot_keyfile_get_boolean_with_default (config, remote_key, "gpg-verify",
@@ -1256,11 +1354,54 @@ ostree_repo_pull_one_dir (OstreeRepo *self,
_ostree_fetcher_set_proxy (pull_data->fetcher, http_proxy);
}
- if (!pull_data->base_uri)
+ if (!ot_keyfile_get_value_with_default (config, remote_key, "metalink",
+ NULL, &metalink_url_str, error))
+ goto out;
+
+ if (!metalink_url_str)
{
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Failed to parse url '%s'", baseurl);
- goto out;
+ if (!repo_get_string_key_inherit (self, remote_key, "url", &baseurl, error))
+ goto out;
+
+ pull_data->base_uri = soup_uri_new (baseurl);
+
+ if (!pull_data->base_uri)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Failed to parse url '%s'", baseurl);
+ goto out;
+ }
+ }
+ else
+ {
+ gs_unref_object GFile *metalink_data = NULL;
+ SoupURI *metalink_uri = soup_uri_new (metalink_url_str);
+ SoupURI *target_uri = NULL;
+
+ if (!metalink_uri)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid metalink URL: %s", metalink_url_str);
+ goto out;
+ }
+
+ metalink = _ostree_metalink_new (pull_data->fetcher, "summary",
+ OSTREE_MAX_METADATA_SIZE, metalink_uri);
+ soup_uri_free (metalink_uri);
+
+ if (!request_metalink_sync (pull_data, metalink, &target_uri, &metalink_data,
+ cancellable, error))
+ goto out;
+
+ {
+ gs_free char *repo_base = g_path_get_dirname (soup_uri_get_path (target_uri));
+ pull_data->base_uri = soup_uri_copy (target_uri);
+ soup_uri_set_path (pull_data->base_uri, repo_base);
+ }
+
+ if (!ot_util_variant_map (metalink_data, OSTREE_SUMMARY_GVARIANT_FORMAT, FALSE,
+ &pull_data->summary, error))
+ goto out;
}
if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error))
@@ -1292,7 +1433,6 @@ ostree_repo_pull_one_dir (OstreeRepo *self,
for (strviter = refs_to_fetch; *strviter; strviter++)
{
const char *branch = *strviter;
- char *contents;
if (ostree_validate_checksum_string (branch, NULL))
{
@@ -1301,11 +1441,7 @@ ostree_repo_pull_one_dir (OstreeRepo *self,
}
else
{
- if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
- goto out;
-
- /* Transfer ownership of contents */
- g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents);
+ g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL);
}
}
}
@@ -1325,14 +1461,37 @@ ostree_repo_pull_one_dir (OstreeRepo *self,
for (;branches_iter && *branches_iter; branches_iter++)
{
const char *branch = *branches_iter;
- char *contents;
- if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
+ g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL);
+ }
+ }
+
+ g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ const char *branch = key;
+ char *contents;
+
+ if (pull_data->summary)
+ {
+ guint64 commit_size;
+ guint64 *malloced_size;
+
+ if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, &commit_size, error))
goto out;
- /* Transfer ownership of contents */
- g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents);
+ malloced_size = g_new0 (guint64, 1);
+ *malloced_size = commit_size;
+ g_hash_table_insert (pull_data->expected_commit_sizes, contents, malloced_size);
+ }
+ else
+ {
+ if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
+ goto out;
}
+
+ /* Transfer ownership of contents */
+ g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents);
}
/* Create the state directory here - it's new with the commitpartial code,
@@ -1460,7 +1619,9 @@ ostree_repo_pull_one_dir (OstreeRepo *self,
g_free (pull_data->remote_name);
if (pull_data->base_uri)
soup_uri_free (pull_data->base_uri);
+ g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref);
g_clear_pointer (&pull_data->static_delta_metas, (GDestroyNotify) g_ptr_array_unref);
+ g_clear_pointer (&pull_data->expected_commit_sizes, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref);
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 8a85ef2..daa532e 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -381,7 +381,7 @@ GS_DEFINE_CLEANUP_FUNCTION0(GKeyFile*, local_keyfile_unref, g_key_file_unref)
* ostree_repo_remote_add:
* @self: Repo
* @name: Name of remote
- * @url: URL for remote
+ * @url: URL for remote (if URL begins with metalink=, it will be used as such)
* @options: (allow-none): GVariant of type a{sv}
* @cancellable: Cancellable
* @error: Error
@@ -446,7 +446,11 @@ ostree_repo_remote_add (OstreeRepo *self,
target_keyfile = ostree_repo_copy_config (self);
}
- g_key_file_set_string (target_keyfile, section, "url", url);
+ if (g_str_has_prefix (url, "metalink="))
+ g_key_file_set_string (target_keyfile, section, "metalink", url + strlen ("metalink="));
+ else
+ g_key_file_set_string (target_keyfile, section, "url", url);
+
if (options)
keyfile_set_from_vardict (target_keyfile, section, options);
@@ -2264,3 +2268,81 @@ out:
(void) gs_file_unlink (commit_tmp_path, NULL, NULL);
return ret;
}
+
+/**
+ * ostree_repo_regenerate_summary:
+ * @self: Repo
+ * @additional_metadata: (allow-none): A GVariant of type a{sv}, or %NULL
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * An OSTree repository can contain a high level "summary" file that
+ * describes the available branches and other metadata.
+ *
+ * It is not regenerated automatically when commits are created; this
+ * API is available to atomically regenerate the summary after
+ * multiple commits. It should only be invoked by one process at a
+ * time.
+ */
+gboolean
+ostree_repo_regenerate_summary (OstreeRepo *self,
+ GVariant *additional_metadata,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_unref_object GFile *summary_path = NULL;
+ gs_unref_hashtable GHashTable *refs = NULL;
+ gs_unref_variant_builder GVariantBuilder *refs_builder = NULL;
+ gs_unref_variant GVariant *summary = NULL;
+ GList *ordered_keys = NULL;
+ GList *iter = NULL;
+
+ if (!ostree_repo_list_refs (self, NULL, &refs, cancellable, error))
+ goto out;
+
+ refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(s(taya{sv}))"));
+
+ ordered_keys = g_hash_table_get_keys (refs);
+ ordered_keys = g_list_sort (ordered_keys, (GCompareFunc)strcmp);
+
+ for (iter = ordered_keys; iter; iter = iter->next)
+ {
+ const char *ref = iter->data;
+ const char *commit = g_hash_table_lookup (refs, ref);
+ gs_unref_variant GVariant *commit_obj = NULL;
+
+ g_assert (commit);
+
+ if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_obj, error))
+ goto out;
+
+ g_variant_builder_add_value (refs_builder,
+ g_variant_new ("(s(t ay@a{sv}))", ref,
+ g_variant_get_size (commit_obj),
+ ostree_checksum_to_bytes_v (commit),
+ ot_gvariant_new_empty_string_dict ()));
+ }
+
+ {
+ gs_unref_variant_builder GVariantBuilder *summary_builder =
+ g_variant_builder_new (OSTREE_SUMMARY_GVARIANT_FORMAT);
+
+ g_variant_builder_add_value (summary_builder, g_variant_builder_end (refs_builder));
+ g_variant_builder_add_value (summary_builder, additional_metadata ? additional_metadata :
ot_gvariant_new_empty_string_dict ());
+ summary = g_variant_builder_end (summary_builder);
+ g_variant_ref_sink (summary);
+ }
+
+ summary_path = g_file_get_child (self->repodir, "summary");
+
+ if (!ot_util_variant_save (summary_path, summary, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ if (ordered_keys)
+ g_list_free (ordered_keys);
+ return ret;
+}
+
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index cace5e5..de38ce4 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -568,5 +568,11 @@ gboolean ostree_repo_verify_commit (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
+gboolean ostree_repo_regenerate_summary (OstreeRepo *self,
+ GVariant *additional_metadata,
+ GCancellable *cancellable,
+ GError **error);
+
+
G_END_DECLS
diff --git a/src/libotutil/ot-checksum-utils.c b/src/libotutil/ot-checksum-utils.c
index 6a45c01..b728992 100644
--- a/src/libotutil/ot-checksum-utils.c
+++ b/src/libotutil/ot-checksum-utils.c
@@ -139,6 +139,32 @@ ot_gio_checksum_stream (GInputStream *in,
return ot_gio_splice_get_checksum (NULL, in, out_csum, cancellable, error);
}
+char *
+ot_checksum_file (GFile *file,
+ GChecksumType checksum_type,
+ GCancellable *cancellable,
+ GError **error)
+{
+ GChecksum *checksum = NULL;
+ char *ret = NULL;
+ gs_unref_object GInputStream *in = NULL;
+
+ in = (GInputStream*)g_file_read (file, cancellable, error);
+ if (!in)
+ goto out;
+
+ checksum = g_checksum_new (checksum_type);
+
+ if (!ot_gio_splice_update_checksum (NULL, in, checksum, cancellable, error))
+ goto out;
+
+ ret = g_strdup (g_checksum_get_string (checksum));
+ out:
+ g_clear_pointer (&checksum, (GDestroyNotify) g_checksum_free);
+ return ret;
+
+}
+
static void
checksum_stream_thread (GSimpleAsyncResult *result,
GObject *object,
diff --git a/src/libotutil/ot-checksum-utils.h b/src/libotutil/ot-checksum-utils.h
index 7778ed0..eb8bbc0 100644
--- a/src/libotutil/ot-checksum-utils.h
+++ b/src/libotutil/ot-checksum-utils.h
@@ -53,6 +53,11 @@ gboolean ot_gio_checksum_stream (GInputStream *in,
GCancellable *cancellable,
GError **error);
+char * ot_checksum_file (GFile *file,
+ GChecksumType checksum_type,
+ GCancellable *cancellable,
+ GError **error);
+
void ot_gio_checksum_stream_async (GInputStream *in,
int io_priority,
GCancellable *cancellable,
diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c
index ffc2def..aa3dff5 100644
--- a/src/libotutil/ot-variant-utils.c
+++ b/src/libotutil/ot-variant-utils.c
@@ -31,6 +31,12 @@
#include "otutil.h"
GVariant *
+ot_gvariant_new_empty_string_dict (void)
+{
+ return g_variant_builder_end (g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")));
+}
+
+GVariant *
ot_gvariant_new_bytearray (const guchar *data,
gsize len)
{
@@ -282,3 +288,61 @@ ot_variant_new_from_bytes (const GVariantType *type,
(GDestroyNotify)g_bytes_unref, bytes);
#endif
}
+
+/**
+ * ot_variant_bsearch_str:
+ * @array: A GVariant array whose first element must be a string
+ * @str: Search for this string
+ * @out_pos: Output position
+ *
+ *
+ * Binary search in a GVariant array, which must be of the form 'a(s...)',
+ * where '...' may be anything. The array elements must be sorted.
+ *
+ * Returns: %TRUE if found, %FALSE otherwise
+ */
+gboolean
+ot_variant_bsearch_str (GVariant *array,
+ const char *str,
+ int *out_pos)
+{
+ gsize imax, imin;
+ gsize imid;
+ gsize n;
+
+ n = g_variant_n_children (array);
+ if (n == 0)
+ return FALSE;
+
+ imax = n - 1;
+ imin = 0;
+ while (imax >= imin)
+ {
+ gs_unref_variant GVariant *child = NULL;
+ const char *cur;
+ int cmp;
+
+ imid = (imin + imax) / 2;
+
+ child = g_variant_get_child_value (array, imid);
+ g_variant_get_child (child, 0, "&s", &cur, NULL);
+
+ cmp = strcmp (cur, str);
+ if (cmp < 0)
+ imin = imid + 1;
+ else if (cmp > 0)
+ {
+ if (imid == 0)
+ break;
+ imax = imid - 1;
+ }
+ else
+ {
+ *out_pos = imid;
+ return TRUE;
+ }
+ }
+
+ *out_pos = imid;
+ return FALSE;
+}
diff --git a/src/libotutil/ot-variant-utils.h b/src/libotutil/ot-variant-utils.h
index 218e543..422a803 100644
--- a/src/libotutil/ot-variant-utils.h
+++ b/src/libotutil/ot-variant-utils.h
@@ -31,6 +31,8 @@ GVariant *ot_gvariant_new_bytearray (const guchar *data,
GVariant *ot_gvariant_new_ay_bytes (GBytes *bytes);
+GVariant *ot_gvariant_new_empty_string_dict (void);
+
GHashTable *ot_util_variant_asv_to_hash_table (GVariant *variant);
GVariant * ot_util_variant_take_ref (GVariant *variant);
@@ -70,5 +72,10 @@ ot_variant_new_from_bytes (const GVariantType *type,
GBytes *bytes,
gboolean trusted);
+gboolean
+ot_variant_bsearch_str (GVariant *array,
+ const char *str,
+ int *out_pos);
+
G_END_DECLS
diff --git a/src/ostree/main.c b/src/ostree/main.c
index b16d8c1..e114690 100644
--- a/src/ostree/main.c
+++ b/src/ostree/main.c
@@ -55,6 +55,7 @@ static OstreeCommand commands[] = {
{ "rev-parse", ostree_builtin_rev_parse, 0 },
{ "show", ostree_builtin_show, 0 },
{ "static-delta", ostree_builtin_static_delta, 0 },
+ { "summary", ostree_builtin_summary, 0 },
#ifdef HAVE_LIBSOUP
{ "trivial-httpd", ostree_builtin_trivial_httpd, OSTREE_BUILTIN_FLAG_NO_REPO },
#endif
diff --git a/src/ostree/ot-builtin-summary.c b/src/ostree/ot-builtin-summary.c
new file mode 100644
index 0000000..2f9cae5
--- /dev/null
+++ b/src/ostree/ot-builtin-summary.c
@@ -0,0 +1,63 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2014 Colin Walters <walters verbum org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include "ot-builtins.h"
+#include "ostree.h"
+#include "otutil.h"
+
+static gboolean opt_update;
+
+static GOptionEntry options[] = {
+ { "update", 'u', 0, G_OPTION_ARG_NONE, &opt_update, "Update the summary", NULL },
+ { NULL }
+};
+
+gboolean
+ostree_builtin_summary (int argc, char **argv, OstreeRepo *repo, GCancellable *cancellable, GError **error)
+{
+ gboolean ret = FALSE;
+ GOptionContext *context;
+
+ context = g_option_context_new ("Manage summary metadata");
+ g_option_context_add_main_entries (context, options, NULL);
+
+ if (!g_option_context_parse (context, &argc, &argv, error))
+ goto out;
+
+ if (opt_update)
+ {
+ if (!ostree_repo_regenerate_summary (repo, NULL, cancellable, error))
+ goto out;
+ }
+ else
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "No option specified; use -u to update summary");
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ if (context)
+ g_option_context_free (context);
+ return ret;
+}
diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h
index fee66f2..b8b6507 100644
--- a/src/ostree/ot-builtins.h
+++ b/src/ostree/ot-builtins.h
@@ -46,6 +46,7 @@ BUILTINPROTO(reset);
BUILTINPROTO(fsck);
BUILTINPROTO(show);
BUILTINPROTO(static_delta);
+BUILTINPROTO(summary);
BUILTINPROTO(rev_parse);
BUILTINPROTO(remote);
BUILTINPROTO(write_refs);
diff --git a/tests/test-pull-metalink.sh b/tests/test-pull-metalink.sh
new file mode 100755
index 0000000..3d68578
--- /dev/null
+++ b/tests/test-pull-metalink.sh
@@ -0,0 +1,120 @@
+#!/bin/bash
+#
+# Copyright (C) 2014 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+
+. $(dirname $0)/libtest.sh
+
+setup_fake_remote_repo1 "archive-z2"
+
+# And another web server acting as the metalink server
+cd ${test_tmpdir}
+mkdir metalink-data
+cd metalink-data
+ostree trivial-httpd --daemonize -p ${test_tmpdir}/metalink-httpd-port
+metalink_port=$(cat ${test_tmpdir}/metalink-httpd-port)
+echo "http://127.0.0.1:${metalink_port}"; > ${test_tmpdir}/metalink-httpd-address
+
+ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u
+
+summary_path=${test_tmpdir}/ostree-srv/gnomerepo/summary
+
+echo -n broken > ${summary_path}.bad
+
+echo '1..1'
+cd ${test_tmpdir}
+
+cat > ${test_tmpdir}/metalink-data/metalink.xml <<EOF
+<?xml version="1.0" encoding="utf-8"?>
+<metalink version="3.0" xmlns="http://www.metalinker.org/";>
+ <files>
+ <file name="summary">
+ <size>$(stat -c '%s' ${summary_path})</size>
+ <verification>
+ <hash type="md5">$(md5sum ${summary_path} | cut -f 1 -d ' ')</hash>
+ <hash type="sha256">$(sha256sum ${summary_path} | cut -f 1 -d ' ')</hash>
+ <hash type="sha512">$(sha512sum ${summary_path} | cut -f 1 -d ' ')</hash>
+ </verification>
+ <resources maxconnections="1">
+ <url protocol="http" type="http" location="US" preference="100" >$(cat
httpd-address)/ostree/gnomerepo/summary.bad</url>
+ <url protocol="http" type="http" location="US" preference="99" >$(cat
httpd-address)/ostree/gnomerepo/nosuchfile</url>
+ <url protocol="http" type="http" location="US" preference="98" >$(cat
httpd-address)/ostree/gnomerepo/summary</url>
+ </resources>
+ </file>
+ </files>
+</metalink>
+EOF
+
+cd ${test_tmpdir}
+mkdir repo
+${CMD_PREFIX} ostree --repo=repo init
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin metalink=$(cat
metalink-httpd-address)/metalink.xml
+${CMD_PREFIX} ostree --repo=repo pull origin:main
+${CMD_PREFIX} ostree --repo=repo rev-parse origin:main
+${CMD_PREFIX} ostree --repo=repo fsck
+echo "ok pull via metalink"
+
+cp metalink-data/metalink.xml{,.orig}
+cp ostree-srv/gnomerepo/summary{,.orig}
+
+test_metalink_pull_error() {
+ msg=$1
+ rm repo -rf
+ mkdir repo
+ ${CMD_PREFIX} ostree --repo=repo init
+ ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin metalink=$(cat
metalink-httpd-address)/metalink.xml
+ if ${CMD_PREFIX} ostree --repo=repo pull origin:main 2>err.txt; then
+ assert_not_reached "pull unexpectedly succeeded"
+ fi
+ cat err.txt
+ assert_file_has_content err.txt "${msg}"
+}
+
+cd ${test_tmpdir}
+sed -e 's,<hash type="sha512">.*</hash>,<hash type="sha512">bacon</hash>,' < metalink-data/metalink.xml.orig
metalink-data/metalink.xml
+test_metalink_pull_error "Invalid hash digest for sha512"
+echo "ok metalink err hash format"
+
+cd ${test_tmpdir}
+sed -e 's,<hash type="sha512">.*</hash>,<hash type="sha512">'$( (echo -n dummy; cat ${summary_path}) |
sha512sum | cut -f 1 -d ' ')'</hash>,' < metalink-data/metalink.xml.orig > metalink-data/metalink.xml
+test_metalink_pull_error "Expected checksum is .* but actual is"
+echo "ok metalink err hash sha512"
+
+cd ${test_tmpdir}
+cp metalink-data/metalink.xml.orig metalink-data/metalink.xml
+echo -n moo > ostree-srv/gnomerepo/summary
+test_metalink_pull_error "Expected size is .* bytes but content is 3 bytes"
+echo "ok metalink err size"
+cp ostree-srv/gnomerepo/summary{.orig,}
+
+cd ${test_tmpdir}
+grep -v sha256 < metalink-data/metalink.xml.orig |grep -v sha512 > metalink-data/metalink.xml
+test_metalink_pull_error "No.*verification.*with known.*hash"
+echo "ok metalink err no verify"
+
+cd ${test_tmpdir}
+grep -v '<url protocol' < metalink-data/metalink.xml.orig > metalink-data/metalink.xml
+test_metalink_pull_error "No.*url.*method.*elements"
+echo "ok metalink err no url"
+
+cd ${test_tmpdir}
+echo bacon > metalink-data/metalink.xml
+test_metalink_pull_error "Document must begin with an element"
+echo "ok metalink err malformed"
+
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
