[ostree/wip/packfile-rebase2] lots more pull work
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree/wip/packfile-rebase2] lots more pull work
- Date: Sat, 31 Mar 2012 14:19:18 +0000 (UTC)
commit 898278caea783202456886c631fb9a39b871cd8e
Author: Colin Walters <walters verbum org>
Date: Fri Mar 30 16:17:46 2012 -0400
lots more pull work
src/libostree/ostree-core.c | 220 ++++++++++-
src/libostree/ostree-core.h | 22 +-
src/libostree/ostree-repo.c | 525 ++++++++++++++++++++------
src/libostree/ostree-repo.h | 44 ++-
src/libotutil/ot-gio-utils.c | 26 ++
src/libotutil/ot-gio-utils.h | 2 +
src/libotutil/ot-variant-utils.c | 45 +++
src/libotutil/ot-variant-utils.h | 19 +
src/ostree/ostree-pull.c | 776 +++++++++++++++++++++++++++-----------
src/ostree/ot-builtin-fsck.c | 18 +
src/ostree/ot-builtin-init.c | 5 +
11 files changed, 1350 insertions(+), 352 deletions(-)
---
diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c
index bdf4340..9ca8f83 100644
--- a/src/libostree/ostree-core.c
+++ b/src/libostree/ostree-core.c
@@ -652,6 +652,9 @@ ostree_checksum_to_bytes (const char *sha256)
big = g_ascii_xdigit_value (sha256[j]);
little = g_ascii_xdigit_value (sha256[j+1]);
+ g_assert (big != -1);
+ g_assert (little != -1);
+
result[i] = (big << 4) | little;
}
@@ -1311,7 +1314,6 @@ ostree_read_pack_entry_as_stream (GVariant *pack_entry)
return ret_input;
}
-
gboolean
ostree_read_pack_entry_variant (GVariant *pack_entry,
OstreeObjectType expected_objtype,
@@ -1322,26 +1324,17 @@ ostree_read_pack_entry_variant (GVariant *pack_entry,
{
gboolean ret = FALSE;
GInputStream *stream = NULL;
- GMemoryOutputStream *data_stream = NULL;
GVariant *container_variant = NULL;
GVariant *ret_variant = NULL;
guint32 actual_type;
stream = ostree_read_pack_entry_as_stream (pack_entry);
- data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
-
- if (!g_output_stream_splice ((GOutputStream*)data_stream, stream,
- G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
- cancellable, error))
+ if (!ot_util_variant_from_stream (stream, OSTREE_SERIALIZED_VARIANT_FORMAT,
+ trusted, &container_variant, cancellable, error))
goto out;
- container_variant = g_variant_new_from_data (OSTREE_SERIALIZED_VARIANT_FORMAT,
- g_memory_output_stream_get_data (data_stream),
- g_memory_output_stream_get_data_size (data_stream),
- trusted, (GDestroyNotify) g_object_unref, data_stream);
g_variant_ref_sink (container_variant);
- data_stream = NULL; /* Transfer ownership */
g_variant_get (container_variant, "(uv)",
&actual_type, &ret_variant);
@@ -1359,10 +1352,211 @@ ostree_read_pack_entry_variant (GVariant *pack_entry,
ot_transfer_out_value (out_variant, &ret_variant);
out:
g_clear_object (&stream);
- g_clear_object (&data_stream);
ot_clear_gvariant (&ret_variant);
ot_clear_gvariant (&container_variant);
return ret;
}
+gboolean
+ostree_pack_index_search (GVariant *index,
+ GVariant *csum_bytes,
+ OstreeObjectType objtype,
+ guint64 *out_offset)
+{
+ gboolean ret = FALSE;
+ GVariant *index_contents;
+ gsize imax, imin;
+ gsize n;
+ guint32 target_objtype;
+
+ index_contents = g_variant_get_child_value (index, 2);
+
+ target_objtype = (guint32) objtype;
+
+ n = g_variant_n_children (index_contents);
+
+ if (n == 0)
+ goto out;
+
+ imax = n - 1;
+ imin = 0;
+ while (imax >= imin)
+ {
+ GVariant *cur_csum_bytes;
+ guint32 cur_objtype;
+ guint64 cur_offset;
+ gsize imid;
+ int c;
+
+ imid = (imin + imax) / 2;
+
+ g_variant_get_child (index_contents, imid, "(u ayt)", &cur_objtype,
+ &cur_csum_bytes, &cur_offset);
+ cur_objtype = GUINT32_FROM_BE (cur_objtype);
+ cur_offset = GUINT64_FROM_BE (cur_offset);
+
+ c = ostree_cmp_checksum_bytes (cur_csum_bytes, csum_bytes);
+ if (c == 0)
+ {
+ if (cur_objtype < target_objtype)
+ c = -1;
+ else if (cur_objtype > target_objtype)
+ c = 1;
+ }
+ g_variant_unref (cur_csum_bytes);
+
+ if (c < 0)
+ imin = imid + 1;
+ else if (c > 0)
+ {
+ if (imid == 0)
+ goto out;
+ imax = imid - 1;
+ }
+ else
+ {
+ *out_offset = cur_offset;
+ break;
+ }
+ }
+
+ ret = TRUE;
+ out:
+ ot_clear_gvariant (&index_contents);
+ return ret;
+}
+
+gboolean
+ostree_validate_structureof_objtype (guint32 objtype,
+ GError **error)
+{
+ objtype = GUINT32_FROM_BE (objtype);
+ if (objtype < OSTREE_OBJECT_TYPE_RAW_FILE
+ || objtype > OSTREE_OBJECT_TYPE_COMMIT)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid object type '%u'", objtype);
+ return FALSE;
+ }
+ return TRUE;
+}
+
+gboolean
+ostree_validate_structureof_checksum (GVariant *checksum,
+ GError **error)
+{
+ gsize n_children = g_variant_n_children (checksum);
+ if (n_children != 32)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid checksum of length %" G_GUINT64_FORMAT
+ " expected 32", (guint64) n_children);
+ return FALSE;
+ }
+ return TRUE;
+}
+static gboolean
+validate_variant (GVariant *variant,
+ const GVariantType *variant_type,
+ GError **error)
+{
+ if (!g_variant_is_normal_form (variant))
+ {
+ g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Not normal form");
+ return FALSE;
+ }
+ if (!g_variant_is_of_type (variant, variant_type))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Doesn't match variant type '%s'",
+ (char*)variant_type);
+ return FALSE;
+ }
+ return TRUE;
+}
+
+gboolean
+ostree_validate_structureof_pack_index (GVariant *index,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ const char *header;
+ GVariantIter *content_iter = NULL;
+ guint32 objtype;
+ GVariant *csum_bytes = NULL;
+ guint64 offset;
+
+ if (!validate_variant (index, OSTREE_PACK_INDEX_VARIANT_FORMAT, error))
+ goto out;
+
+ g_variant_get_child (index, 0, "&s", &header);
+
+ if (strcmp (header, "OSTv0PACKINDEX") != 0)
+ {
+ g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid pack index; doesn't match header");
+ goto out;
+ }
+
+ g_variant_get_child (index, 2, "a(uayt)", &content_iter);
+
+ while (g_variant_iter_loop (content_iter, "(u ayt)",
+ &objtype, &csum_bytes, &offset))
+ {
+ if (!ostree_validate_structureof_objtype (objtype, error))
+ goto out;
+ if (!ostree_validate_structureof_checksum (csum_bytes, error))
+ goto out;
+ }
+ csum_bytes = NULL;
+
+ ret = TRUE;
+ out:
+ if (content_iter)
+ g_variant_iter_free (content_iter);
+ ot_clear_gvariant (&csum_bytes);
+ return ret;
+}
+
+gboolean
+ostree_validate_structureof_pack_superindex (GVariant *superindex,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ const char *header;
+ GVariant *csum_bytes = NULL;
+ GVariant *bloom = NULL;
+ GVariantIter *content_iter = NULL;
+
+ if (!validate_variant (superindex, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, error))
+ goto out;
+
+ g_variant_get_child (superindex, 0, "&s", &header);
+
+ if (strcmp (header, "OSTv0PACKSUPERINDEX") != 0)
+ {
+ g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid pack superindex; doesn't match header");
+ goto out;
+ }
+
+ g_variant_get_child (superindex, 2, "a(ayay)", &content_iter);
+
+ while (g_variant_iter_loop (content_iter, "(@ay ay)",
+ &csum_bytes, &bloom))
+ {
+ if (!ostree_validate_structureof_checksum (csum_bytes, error))
+ goto out;
+ }
+ csum_bytes = NULL;
+
+ ret = TRUE;
+ out:
+ if (content_iter)
+ g_variant_iter_free (content_iter);
+ ot_clear_gvariant (&csum_bytes);
+ ot_clear_gvariant (&bloom);
+ return ret;
+}
diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h
index 0e4d154..ad08c50 100644
--- a/src/libostree/ostree-core.h
+++ b/src/libostree/ostree-core.h
@@ -98,7 +98,7 @@ typedef enum {
#define OSTREE_ARCHIVED_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(uuuuusa(ayay))")
/* Pack super index
- * s - OSTSUPERPACKINDEX
+ * s - OSTv0SUPERPACKINDEX
* a{sv} - Metadata
* a(say) - (pack file checksum, bloom filter)
*/
@@ -160,6 +160,7 @@ void ostree_object_name_deserialize_v2_hex (GVariant *variant,
char **out_checksum,
OstreeObjectType *out_objtype);
+
void ostree_object_name_deserialize_v2_bytes (GVariant *variant,
const guchar **out_checksum,
OstreeObjectType *out_objtype);
@@ -281,4 +282,23 @@ gboolean ostree_read_pack_entry_variant (GVariant *pack_entry,
GCancellable *cancellable,
GError **error);
+gboolean ostree_pack_index_search (GVariant *index,
+ GVariant *csum_bytes,
+ OstreeObjectType objtype,
+ guint64 *out_offset);
+
+/** VALIDATION **/
+
+gboolean ostree_validate_structureof_objtype (guint32 objtype,
+ GError **error);
+
+gboolean ostree_validate_structureof_checksum (GVariant *checksum,
+ GError **error);
+
+gboolean ostree_validate_structureof_pack_index (GVariant *index,
+ GError **error);
+
+gboolean ostree_validate_structureof_pack_superindex (GVariant *superindex,
+ GError **error);
+
#endif /* _OSTREE_REPO */
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index b7fe0d6..b48bd34 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -61,6 +61,7 @@ struct _OstreeRepoPrivate {
GFile *remote_heads_dir;
GFile *objects_dir;
GFile *pack_dir;
+ GFile *remote_cache_dir;
GFile *config_file;
gboolean inited;
@@ -88,6 +89,7 @@ ostree_repo_finalize (GObject *object)
g_clear_object (&priv->remote_heads_dir);
g_clear_object (&priv->objects_dir);
g_clear_object (&priv->pack_dir);
+ g_clear_object (&priv->remote_cache_dir);
g_clear_object (&priv->config_file);
g_hash_table_destroy (priv->pack_index_mappings);
g_hash_table_destroy (priv->pack_data_mappings);
@@ -162,6 +164,7 @@ ostree_repo_constructor (GType gtype,
priv->objects_dir = g_file_get_child (priv->repodir, "objects");
priv->pack_dir = g_file_get_child (priv->objects_dir, "pack");
+ priv->remote_cache_dir = g_file_get_child (priv->objects_dir, "remote-cache");
priv->config_file = g_file_get_child (priv->repodir, "config");
return object;
@@ -1602,6 +1605,40 @@ list_files_in_dir_matching (GFile *dir,
return ret;
}
+static gboolean
+map_variant_file_check_header_string (GFile *path,
+ const GVariantType *variant_type,
+ const char *expected_header,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GVariant *ret_variant = NULL;
+ const char *header;
+
+ if (!ot_util_variant_map (path, variant_type, &ret_variant, error))
+ goto out;
+
+ g_variant_get_child (ret_variant, 0, "&s", &header);
+
+ if (strcmp (header, expected_header) != 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid variant file '%s', expected header '%s'",
+ ot_gfile_get_path_cached (path),
+ expected_header);
+ goto out;
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_variant, &ret_variant);
+ out:
+ ot_clear_gvariant (&ret_variant);
+ return ret;
+}
+
+
static char *
get_checksum_from_pack_name (const char *name)
{
@@ -1654,6 +1691,51 @@ list_pack_indexes_from_dir (OstreeRepo *self,
return ret;
}
+static gboolean
+list_pack_checksums_from_superindex_file (GFile *superindex_path,
+ GPtrArray **out_indexes,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GPtrArray *ret_indexes = NULL;
+ GVariant *superindex_variant = NULL;
+ GVariantIter *variant_iter = NULL;
+ const char *magic;
+ GVariant *checksum;
+ GVariant *bloom;
+
+ if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT,
+ &superindex_variant, error))
+ goto out;
+
+ g_variant_get (superindex_variant, "(&s a{sv}a(ayay))",
+ &magic, NULL, &variant_iter);
+
+ if (strcmp (magic, "OSTv0SUPERPACKINDEX") != 0)
+ {
+ g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid header in super pack index");
+ goto out;
+ }
+
+ ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
+
+ while (g_variant_iter_loop (variant_iter, "(@ay ay)",
+ &checksum, &bloom))
+ g_ptr_array_add (ret_indexes, ostree_checksum_from_bytes (checksum));
+
+ ret = TRUE;
+ ot_transfer_out_value (out_indexes, &ret_indexes);
+ out:
+ ot_clear_gvariant (&superindex_variant);
+ if (variant_iter)
+ g_variant_iter_free (variant_iter);
+ if (ret_indexes)
+ g_ptr_array_unref (ret_indexes);
+ return ret;
+}
+
gboolean
ostree_repo_list_pack_indexes (OstreeRepo *self,
GPtrArray **out_indexes,
@@ -1664,44 +1746,23 @@ ostree_repo_list_pack_indexes (OstreeRepo *self,
OstreeRepoPrivate *priv = GET_PRIVATE (self);
GFile *superindex_path = NULL;
GPtrArray *ret_indexes = NULL;
- GVariant *superindex_variant = NULL;
- GVariantIter *variant_iter = NULL;
- const char *magic;
- GVariant *checksum;
- GVariant *bloom;
superindex_path = g_file_get_child (priv->pack_dir, "index");
- ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
-
if (g_file_query_exists (superindex_path, cancellable))
{
- if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT,
- &superindex_variant, error))
+ if (!list_pack_checksums_from_superindex_file (superindex_path, &ret_indexes, cancellable, error))
goto out;
-
- g_variant_get (superindex_variant, "(&s a{sv}a(ayay))",
- &magic, NULL, &variant_iter);
-
- if (strcmp (magic, "OSTv0SUPERPACKINDEX") != 0)
- {
- g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Invalid header in super pack index");
- goto out;
- }
-
- while (g_variant_iter_loop (variant_iter, "(@ay ay)",
- &checksum, &bloom))
- g_ptr_array_add (ret_indexes, ostree_checksum_from_bytes (checksum));
+ }
+ else
+ {
+ ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
}
ret = TRUE;
ot_transfer_out_value (out_indexes, &ret_indexes);
out:
- ot_clear_gvariant (&superindex_variant);
g_clear_object (&superindex_path);
- if (variant_iter)
- g_variant_iter_free (variant_iter);
if (ret_indexes)
g_ptr_array_unref (ret_indexes);
return ret;
@@ -1717,7 +1778,7 @@ create_index_bloom (OstreeRepo *self,
gboolean ret = FALSE;
GVariant *ret_bloom;
- /* TODO - compute bloom filter */
+ /* TODO - define and compute bloom filter */
ret_bloom = g_variant_new_fixed_array (G_VARIANT_TYPE ("y"), NULL, 0, 1);
g_variant_ref_sink (ret_bloom);
@@ -1729,6 +1790,10 @@ create_index_bloom (OstreeRepo *self,
return ret;
}
+/**
+ * Regenerate the pack superindex file based on the set of pack
+ * indexes currently in the filesystem.
+ */
gboolean
ostree_repo_regenerate_pack_index (OstreeRepo *self,
GCancellable *cancellable,
@@ -1786,6 +1851,19 @@ ostree_repo_regenerate_pack_index (OstreeRepo *self,
return ret;
}
+
+static GFile *
+get_pack_index_name_from_checksum (GFile *parent, const char *pack_checksum)
+{
+ return ot_gfile_get_child_strconcat (parent, "ostpack-", pack_checksum, ".index", NULL);
+}
+
+static GFile *
+get_pack_data_name_from_checksum (GFile *parent, const char *pack_checksum)
+{
+ return ot_gfile_get_child_strconcat (parent, "ostpack-", pack_checksum, ".data", NULL);
+}
+
gboolean
ostree_repo_add_pack_file (OstreeRepo *self,
const char *pack_checksum,
@@ -1796,24 +1874,17 @@ ostree_repo_add_pack_file (OstreeRepo *self,
{
gboolean ret = FALSE;
OstreeRepoPrivate *priv = GET_PRIVATE (self);
- char *dest_name = NULL;
GFile *pack_index_path = NULL;
GFile *pack_data_path = NULL;
if (!ot_gfile_ensure_directory (priv->pack_dir, FALSE, error))
goto out;
- g_free (dest_name);
- dest_name = g_strconcat ("ostpack-", pack_checksum, ".data", NULL);
- pack_data_path = g_file_get_child (priv->pack_dir, dest_name);
-
+ pack_data_path = get_pack_data_name_from_checksum (priv->pack_dir, pack_checksum);
if (!ot_gfile_rename (data_path, pack_data_path, cancellable, error))
goto out;
- g_free (dest_name);
- dest_name = g_strconcat ("ostpack-", pack_checksum, ".index", NULL);
- pack_index_path = g_file_get_child (priv->pack_dir, dest_name);
-
+ pack_index_path = get_pack_index_name_from_checksum (priv->pack_dir, pack_checksum);
if (!ot_gfile_rename (index_path, pack_index_path, cancellable, error))
goto out;
@@ -1821,7 +1892,303 @@ ostree_repo_add_pack_file (OstreeRepo *self,
out:
g_clear_object (&pack_index_path);
g_clear_object (&pack_data_path);
- g_free (dest_name);
+ return ret;
+}
+
+static gboolean
+ensure_remote_cache_dir (OstreeRepo *self,
+ const char *remote_name,
+ GFile **out_cache_dir,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ GFile *path = NULL;
+
+ path = g_file_get_child (priv->remote_cache_dir, remote_name);
+
+ if (!ot_gfile_ensure_directory (path, FALSE, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ g_clear_object (&path);
+ return ret;
+}
+
+/**
+ * Take a pack superindex file @superindex_path, and clean up any
+ * no-longer-referenced pack files in the lookaside cache for
+ * @remote_name. The updated index file will also be saved into the
+ * cache.
+ *
+ * Upon successful return, @out_cached_indexes will hold checksum
+ * strings for indexes which are already in the cache, and
+ * @out_uncached_indexes will hold strings for those which are not.
+ */
+gboolean
+ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
+ const char *remote_name,
+ GFile *superindex_path,
+ GPtrArray **out_cached_indexes,
+ GPtrArray **out_uncached_indexes,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GVariant *superindex_variant = NULL;
+ GVariantIter *superindex_contents_iter = NULL;
+ GFile *cache_path = NULL;
+ GFile *superindex_cache_path = NULL;
+ GPtrArray *index_files = NULL;
+ GHashTable *new_pack_indexes = NULL;
+ GHashTableIter hash_iter;
+ gpointer key, value;
+ GPtrArray *ret_cached_indexes = NULL;
+ GPtrArray *ret_uncached_indexes = NULL;
+ guint i;
+ GVariant *csum_bytes = NULL;
+ GVariant *bloom = NULL;
+ char *pack_checksum = NULL;
+
+ if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error))
+ goto out;
+
+ ret_cached_indexes = g_ptr_array_new_with_free_func (g_free);
+ ret_uncached_indexes = g_ptr_array_new_with_free_func (g_free);
+
+ if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT,
+ &superindex_variant, error))
+ goto out;
+
+ if (!ostree_validate_structureof_pack_superindex (superindex_variant, error))
+ goto out;
+
+ new_pack_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+ g_variant_get_child (superindex_variant, 2, "a(ayay)",
+ &superindex_contents_iter);
+
+ while (g_variant_iter_loop (superindex_contents_iter,
+ "(@ay ay)", &csum_bytes, &bloom))
+ {
+ pack_checksum = ostree_checksum_from_bytes (csum_bytes);
+ g_hash_table_insert (new_pack_indexes, pack_checksum, pack_checksum);
+ pack_checksum = NULL; /* transfer ownership */
+ }
+
+ if (!list_files_in_dir_matching (cache_path,
+ "ostpack-", ".index",
+ &index_files,
+ cancellable, error))
+ goto out;
+
+ for (i = 0; i < index_files->len; i++)
+ {
+ GFile *index_file = index_files->pdata[i];
+
+ g_free (pack_checksum);
+ pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (index_file));
+
+ if (!g_hash_table_lookup (new_pack_indexes, pack_checksum))
+ {
+ if (!ot_gfile_unlink (index_file, cancellable, error))
+ goto out;
+ }
+
+ g_ptr_array_add (ret_cached_indexes, pack_checksum);
+ pack_checksum = NULL; /* transfer ownership */
+ }
+
+ g_hash_table_iter_init (&hash_iter, new_pack_indexes);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ const char *cur_pack_checksum = key;
+ gboolean found = FALSE;
+
+ for (i = 0; i < ret_cached_indexes->len; i++)
+ {
+ if (strcmp (cur_pack_checksum, ret_cached_indexes->pdata[i]) == 0)
+ {
+ found = TRUE;
+ break;
+ }
+ }
+
+ if (!found)
+ g_ptr_array_add (ret_uncached_indexes, g_strdup (cur_pack_checksum));
+ }
+
+ superindex_cache_path = g_file_get_child (cache_path, "index");
+ if (!ot_util_variant_save (superindex_cache_path, superindex_variant, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ ot_transfer_out_value (out_cached_indexes, &ret_cached_indexes);
+ ot_transfer_out_value (out_uncached_indexes, &ret_uncached_indexes);
+ out:
+ g_free (pack_checksum);
+ g_clear_object (&cache_path);
+ g_clear_object (&superindex_cache_path);
+ if (superindex_contents_iter)
+ g_variant_iter_free (superindex_contents_iter);
+ ot_clear_ptrarray (&ret_cached_indexes);
+ ot_clear_ptrarray (&ret_uncached_indexes);
+ ot_clear_hashtable (&new_pack_indexes);
+ ot_clear_ptrarray (&index_files);
+ return ret;
+}
+
+/**
+ * Load the index for pack @pack_checksum from cache directory for
+ * @remote_name.
+ */
+gboolean
+ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GVariant *ret_variant = NULL;
+ GFile *cache_dir = NULL;
+ GFile *cached_pack_path = NULL;
+
+ if (!ensure_remote_cache_dir (self, remote_name, &cache_dir,
+ cancellable, error))
+ goto out;
+
+ cached_pack_path = get_pack_index_name_from_checksum (cache_dir, pack_checksum);
+ if (!ot_util_variant_map (cached_pack_path, OSTREE_PACK_INDEX_VARIANT_FORMAT,
+ &ret_variant, error))
+ goto out;
+
+ ret = TRUE;
+ ot_transfer_out_value (out_variant, &ret_variant);
+ out:
+ g_clear_object (&cache_dir);
+ g_clear_object (&cached_pack_path);
+ ot_clear_gvariant (&ret_variant);
+ return ret;
+}
+
+/**
+ * The variable @cached_path should refer to a file containing a pack
+ * index. It will be validated and added to the cache directory for
+ * @remote_name.
+ */
+gboolean
+ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GFile *cached_path,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *cachedir = NULL;
+ GFile *target_path = NULL;
+ GVariant *input_index_variant = NULL;
+ GVariant *output_index_variant = NULL;
+
+ if (!map_variant_file_check_header_string (cached_path,
+ OSTREE_PACK_INDEX_VARIANT_FORMAT,
+ "OSTv0PACKINDEX",
+ &input_index_variant,
+ cancellable, error))
+ goto out;
+
+ if (!ostree_validate_structureof_pack_index (input_index_variant, error))
+ goto out;
+
+ output_index_variant = g_variant_get_normal_form (input_index_variant);
+
+ if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error))
+ goto out;
+
+ cached_path = get_pack_index_name_from_checksum (cachedir, pack_checksum);
+ if (!ot_util_variant_save (target_path, output_index_variant, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ g_clear_object (&cachedir);
+ g_clear_object (&target_path);
+ ot_clear_gvariant (&input_index_variant);
+ ot_clear_gvariant (&output_index_variant);
+ return ret;
+}
+
+/**
+ * Check for availability of the pack index pointing to @pack_checksum
+ * in the lookaside cache for @remote_name. If not found, then the
+ * output parameter @out_cached_path will be %NULL.
+ */
+gboolean
+ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GFile **out_cached_path,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *cache_dir = NULL;
+ GFile *cached_pack_path = NULL;
+ GFile *ret_cached_path = NULL;
+
+ if (!ensure_remote_cache_dir (self, remote_name, &cache_dir,
+ cancellable, error))
+ goto out;
+
+ cached_pack_path = get_pack_data_name_from_checksum (cache_dir, pack_checksum);
+ if (g_file_query_exists (cached_pack_path, cancellable))
+ {
+ ret_cached_path = cached_pack_path;
+ cached_pack_path = NULL;
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_cached_path, &ret_cached_path);
+ out:
+ g_clear_object (&cache_dir);
+ g_clear_object (&cached_pack_path);
+ return ret;
+}
+
+/**
+ * Add file @cached_path into the cache for given @remote_name.
+ *
+ * <note>
+ * This unlinks @cached_path.
+ * </note>
+ */
+gboolean
+ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GFile *cached_path,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *cachedir = NULL;
+ GFile *target_path = NULL;
+
+ if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error))
+ goto out;
+
+ target_path = get_pack_data_name_from_checksum (cachedir, pack_checksum);
+
+ if (!ot_gfile_rename (cached_path, target_path, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ g_clear_object (&cachedir);
+ g_clear_object (&target_path);
return ret;
}
@@ -2735,7 +3102,6 @@ ostree_repo_load_pack_index (OstreeRepo *self,
OstreeRepoPrivate *priv = GET_PRIVATE (self);
GVariant *ret_variant = NULL;
GFile *path = NULL;
- const char *header;
ret_variant = g_hash_table_lookup (priv->pack_index_mappings, sha256);
if (ret_variant)
@@ -2745,23 +3111,16 @@ ostree_repo_load_pack_index (OstreeRepo *self,
else
{
path = ostree_repo_get_pack_index_path (self, sha256);
- if (!ot_util_variant_map (path,
- OSTREE_PACK_INDEX_VARIANT_FORMAT,
- &ret_variant, error))
+ if (!map_variant_file_check_header_string (path,
+ OSTREE_PACK_INDEX_VARIANT_FORMAT,
+ "OSTv0PACKINDEX",
+ &ret_variant,
+ cancellable, error))
goto out;
g_hash_table_insert (priv->pack_index_mappings, g_strdup (sha256),
g_variant_ref (ret_variant));
}
- g_variant_get_child (ret_variant, 0, "&s", &header);
-
- if (strcmp (header, "OSTv0PACKINDEX") != 0)
- {
- g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Invalid pack version, expected 'OSTv0PACKINDEX'");
- goto out;
- }
-
ret = TRUE;
ot_transfer_out_value (out_variant, &ret_variant);
out:
@@ -2822,65 +3181,6 @@ ostree_repo_map_pack_file (OstreeRepo *self,
return ret;
}
-static gboolean
-bsearch_in_pack_index (GVariant *index_contents,
- GVariant *csum_bytes,
- OstreeObjectType objtype,
- guint64 *out_offset)
-{
- gsize imax, imin;
- gsize n;
- guint32 target_objtype = (guint32) objtype;
-
- n = g_variant_n_children (index_contents);
-
- if (n == 0)
- return FALSE;
-
- imax = n - 1;
- imin = 0;
- while (imax >= imin)
- {
- GVariant *cur_csum_bytes;
- guint32 cur_objtype;
- guint64 cur_offset;
- gsize imid;
- int c;
-
- imid = (imin + imax) / 2;
-
- g_variant_get_child (index_contents, imid, "(u ayt)", &cur_objtype,
- &cur_csum_bytes, &cur_offset);
- cur_objtype = GUINT32_FROM_BE (cur_objtype);
- cur_offset = GUINT64_FROM_BE (cur_offset);
-
- c = ostree_cmp_checksum_bytes (cur_csum_bytes, csum_bytes);
- if (c == 0)
- {
- if (cur_objtype < target_objtype)
- c = -1;
- else if (cur_objtype > target_objtype)
- c = 1;
- }
- g_variant_unref (cur_csum_bytes);
-
- if (c < 0)
- imin = imid + 1;
- else if (c > 0)
- {
- if (imid == 0)
- break;
- imax = imid - 1;
- }
- else
- {
- *out_offset = cur_offset;
- return TRUE;
- }
- }
-
- return FALSE;
-}
gboolean
ostree_repo_load_file (OstreeRepo *self,
@@ -3119,7 +3419,6 @@ find_object_in_packs (OstreeRepo *self,
GFile *index_path = NULL;
GVariant *csum_bytes = NULL;
GVariant *index_variant = NULL;
- GVariant *index_contents = NULL;
guint i;
csum_bytes = ostree_checksum_to_bytes (checksum);
@@ -3139,10 +3438,7 @@ find_object_in_packs (OstreeRepo *self,
if (!ostree_repo_load_pack_index (self, pack_checksum, &index_variant, cancellable, error))
goto out;
- ot_clear_gvariant (&index_contents);
- index_contents = g_variant_get_child_value (index_variant, 2);
-
- if (!bsearch_in_pack_index (index_contents, csum_bytes, objtype, &offset))
+ if (!ostree_pack_index_search (index_variant, csum_bytes, objtype, &offset))
continue;
ret_pack_checksum = g_strdup (pack_checksum);
@@ -3161,7 +3457,6 @@ find_object_in_packs (OstreeRepo *self,
g_clear_object (&index_path);
ot_clear_gvariant (&index_variant);
ot_clear_gvariant (&csum_bytes);
- ot_clear_gvariant (&index_contents);
return ret;
}
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index e293cfb..55c918e 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -230,11 +230,9 @@ gboolean ostree_repo_stage_commit (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
-gboolean
-ostree_repo_regenerate_pack_index (OstreeRepo *self,
- GCancellable *cancellable,
- GError **error);
-
+gboolean ostree_repo_regenerate_pack_index (OstreeRepo *self,
+ GCancellable *cancellable,
+ GError **error);
gboolean ostree_repo_add_pack_file (OstreeRepo *self,
const char *checksum,
@@ -243,6 +241,42 @@ gboolean ostree_repo_add_pack_file (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
+gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
+ const char *remote_name,
+ GFile *superindex_path,
+ GPtrArray **out_cached_indexes,
+ GPtrArray **out_uncached_indexes,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GFile *cached_path,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GFile **out_cached_path,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GFile *cached_path,
+ GCancellable *cancellable,
+ GError **error);
+
typedef enum {
OSTREE_REPO_CHECKOUT_MODE_NONE = 0,
OSTREE_REPO_CHECKOUT_MODE_USER = 1
diff --git a/src/libotutil/ot-gio-utils.c b/src/libotutil/ot-gio-utils.c
index 6d2ca73..15dddc7 100644
--- a/src/libotutil/ot-gio-utils.c
+++ b/src/libotutil/ot-gio-utils.c
@@ -77,6 +77,32 @@ ot_gfile_ensure_directory (GFile *dir,
return ret;
}
+GFile *
+ot_gfile_get_child_strconcat (GFile *parent,
+ const char *first,
+ ...)
+{
+ va_list args;
+ GFile *ret;
+ GString *buf;
+ const char *arg;
+
+ g_return_val_if_fail (first != NULL, NULL);
+
+ va_start (args, first);
+
+ buf = g_string_new (first);
+
+ while ((arg = va_arg (args, const char *)) != NULL)
+ g_string_append (buf, arg);
+
+ ret = g_file_get_child (parent, buf->str);
+
+ g_string_free (buf, TRUE);
+
+ return ret;
+}
+
/**
* ot_gfile_unlink:
* @path: Path to file
diff --git a/src/libotutil/ot-gio-utils.h b/src/libotutil/ot-gio-utils.h
index c5831d3..5fda5da 100644
--- a/src/libotutil/ot-gio-utils.h
+++ b/src/libotutil/ot-gio-utils.h
@@ -36,6 +36,8 @@ G_BEGIN_DECLS
GFileType ot_gfile_type_for_mode (guint32 mode);
+GFile *ot_gfile_get_child_strconcat (GFile *parent, const char *first, ...) G_GNUC_NULL_TERMINATED;
+
GFile *ot_gfile_new_for_path (const char *path);
const char *ot_gfile_get_path_cached (GFile *file);
diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c
index cb616f3..73cadca 100644
--- a/src/libotutil/ot-variant-utils.c
+++ b/src/libotutil/ot-variant-utils.c
@@ -88,6 +88,12 @@ ot_util_variant_take_ref (GVariant *variant)
#endif
}
+/**
+ * Return in @out_variant the result of memory-mapping the entire
+ * contents of file @src.
+ *
+ * Note the returned @out_variant is not floating.
+ */
gboolean
ot_util_variant_map (GFile *src,
const GVariantType *type,
@@ -121,3 +127,42 @@ ot_util_variant_map (GFile *src,
g_mapped_file_unref (mfile);
return ret;
}
+
+/**
+ * Read all input from @src, allocating a new #GVariant from it into
+ * output variable @out_variant. @src will be closed as a result.
+ *
+ * Note the returned @out_variant is not floating.
+ */
+gboolean
+ot_util_variant_from_stream (GInputStream *src,
+ const GVariantType *type,
+ gboolean trusted,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GMemoryOutputStream *data_stream = NULL;
+ GVariant *ret_variant = NULL;
+
+ data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
+
+ if (!g_output_stream_splice ((GOutputStream*)data_stream, src,
+ G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
+ cancellable, error))
+ goto out;
+
+ ret_variant = g_variant_new_from_data (type, g_memory_output_stream_get_data (data_stream),
+ g_memory_output_stream_get_data_size (data_stream),
+ trusted, (GDestroyNotify) g_object_unref, data_stream);
+ data_stream = NULL; /* Transfer ownership */
+ g_variant_ref_sink (ret_variant);
+
+ ret = TRUE;
+ ot_transfer_out_value (out_variant, &ret_variant);
+ out:
+ g_clear_object (&data_stream);
+ ot_clear_gvariant (&ret_variant);
+ return ret;
+}
diff --git a/src/libotutil/ot-variant-utils.h b/src/libotutil/ot-variant-utils.h
index 69f3111..e443a74 100644
--- a/src/libotutil/ot-variant-utils.h
+++ b/src/libotutil/ot-variant-utils.h
@@ -33,6 +33,18 @@ G_BEGIN_DECLS
*a_v = NULL; \
} while (0);
+#define ot_clear_ptrarray(a_v) do { \
+ if (*a_v) \
+ g_ptr_array_unref (*a_v); \
+ *a_v = NULL; \
+ } while (0);
+
+#define ot_clear_hashtable(a_v) do { \
+ if (*a_v) \
+ g_hash_table_unref (*a_v); \
+ *a_v = NULL; \
+ } while (0);
+
GHashTable *ot_util_variant_asv_to_hash_table (GVariant *variant);
GVariant * ot_util_variant_take_ref (GVariant *variant);
@@ -47,6 +59,13 @@ gboolean ot_util_variant_map (GFile *src,
GVariant **out_variant,
GError **error);
+gboolean ot_util_variant_from_stream (GInputStream *src,
+ const GVariantType *type,
+ gboolean trusted,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error);
+
G_END_DECLS
#endif
diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c
index b9a76a5..7c32737 100644
--- a/src/ostree/ostree-pull.c
+++ b/src/ostree/ostree-pull.c
@@ -55,6 +55,50 @@ log_verbose (const char *fmt,
}
typedef struct {
+ OstreeRepo *repo;
+ char *remote_name;
+ SoupSession *session;
+ SoupURI *base_uri;
+
+ gboolean fetched_packs;
+ GPtrArray *cached_pack_indexes;
+} OtPullData;
+
+static SoupURI *
+suburi_new (SoupURI *base,
+ const char *first,
+ ...) G_GNUC_NULL_TERMINATED;
+
+static SoupURI *
+suburi_new (SoupURI *base,
+ const char *first,
+ ...)
+{
+ va_list args;
+ GPtrArray *arg_array;
+ const char *arg;
+ char *subpath;
+ SoupURI *ret;
+
+ arg_array = g_ptr_array_new ();
+ g_ptr_array_add (arg_array, (char*)soup_uri_get_path (base));
+ g_ptr_array_add (arg_array, (char*)first);
+
+ while ((arg = va_arg (args, const char *)) != NULL)
+ g_ptr_array_add (arg_array, (char*)arg);
+
+ subpath = g_build_filenamev ((char**)arg_array->pdata);
+ g_ptr_array_unref (arg_array);
+
+ ret = soup_uri_copy (base);
+ soup_uri_set_path (ret, subpath);
+ g_free (subpath);
+
+ return ret;
+}
+
+
+typedef struct {
SoupSession *session;
GOutputStream *stream;
gboolean had_error;
@@ -78,8 +122,7 @@ on_got_chunk (SoupMessage *msg,
}
static gboolean
-fetch_uri (OstreeRepo *repo,
- SoupSession *soup,
+fetch_uri (OtPullData *pull_data,
SoupURI *uri,
const char *tmp_prefix,
GFile **out_temp_filename,
@@ -94,14 +137,14 @@ fetch_uri (OstreeRepo *repo,
GOutputStream *output_stream = NULL;
OstreeSoupChunkData chunkdata;
- if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (repo),
+ if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (pull_data->repo),
tmp_prefix, NULL,
&ret_temp_filename,
&output_stream,
NULL, error))
goto out;
- chunkdata.session = soup;
+ chunkdata.session = pull_data->session;
chunkdata.stream = output_stream;
chunkdata.had_error = FALSE;
chunkdata.error = error;
@@ -114,7 +157,7 @@ fetch_uri (OstreeRepo *repo,
g_signal_connect (msg, "got-chunk", G_CALLBACK (on_got_chunk), &chunkdata);
- response = soup_session_send_message (soup, msg);
+ response = soup_session_send_message (pull_data->session, msg);
if (response != 200)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
@@ -138,8 +181,7 @@ fetch_uri (OstreeRepo *repo,
}
static gboolean
-fetch_uri_contents_utf8 (OstreeRepo *repo,
- SoupSession *soup,
+fetch_uri_contents_utf8 (OtPullData *pull_data,
SoupURI *uri,
char **out_contents,
GCancellable *cancellable,
@@ -150,7 +192,7 @@ fetch_uri_contents_utf8 (OstreeRepo *repo,
char *ret_contents = NULL;
gsize len;
- if (!fetch_uri (repo, soup, uri, "tmp-", &tmpf, cancellable, error))
+ if (!fetch_uri (pull_data, uri, "tmp-", &tmpf, cancellable, error))
goto out;
if (!g_file_load_contents (tmpf, cancellable, &ret_contents, &len, NULL, error))
@@ -173,29 +215,209 @@ fetch_uri_contents_utf8 (OstreeRepo *repo,
return ret;
}
+static gboolean
+fetch_one_pack_file (OtPullData *pull_data,
+ const char *pack_checksum,
+ GFile **out_cached_path,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *ret_cached_path = NULL;
+ GFile *tmp_path = NULL;
+ char *pack_name = NULL;
+ SoupURI *pack_uri = NULL;
+
+ if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
+ pack_checksum, &ret_cached_path,
+ cancellable, error))
+ goto out;
+
+ if (ret_cached_path == NULL)
+ {
+ pack_name = g_strconcat ("ostpack-", pack_checksum, ".data", NULL);
+ pack_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_name, NULL);
+
+ if (!fetch_uri (pull_data, pack_uri, "packdata-", &tmp_path, cancellable, error))
+ goto out;
+
+ if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
+ pack_checksum, tmp_path,
+ cancellable, error))
+ goto out;
+ }
+
+ if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
+ pack_checksum, &ret_cached_path,
+ cancellable, error))
+ goto out;
+
+ g_assert (ret_cached_path != NULL);
+
+ ret = TRUE;
+ ot_transfer_out_value (out_cached_path, &ret_cached_path);
+ out:
+ g_clear_object (&ret_cached_path);
+ g_clear_object (&tmp_path);
+ g_free (pack_name);
+ if (pack_uri)
+ soup_uri_free (pack_uri);
+ return ret;
+}
+
+static gboolean
+find_object_in_remote_packs (OtPullData *pull_data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ char **out_pack_checksum,
+ guint64 *out_offset,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GVariant *mapped_pack = NULL;
+ GVariant *csum_bytes = NULL;
+ char *ret_pack_checksum = NULL;
+ guint64 offset;
+ guint i;
+
+ csum_bytes = ostree_checksum_to_bytes (checksum);
+
+ for (i = 0; i < pull_data->cached_pack_indexes->len; i++)
+ {
+ const char *pack_checksum = pull_data->cached_pack_indexes->pdata[i];
+
+ ot_clear_gvariant (&mapped_pack);
+ if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
+ pack_checksum, &mapped_pack,
+ cancellable, error))
+ goto out;
+
+ if (ostree_pack_index_search (mapped_pack, csum_bytes, objtype, &offset))
+ {
+ ret_pack_checksum = g_strdup (pack_checksum);
+ break;
+ }
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum);
+ if (out_offset)
+ *out_offset = offset;
+ out:
+ ot_clear_gvariant (&mapped_pack);
+ g_free (ret_pack_checksum);
+ ot_clear_gvariant (&csum_bytes);
+ return ret;
+}
+
+static gboolean
+fetch_one_cache_index (OtPullData *pull_data,
+ const char *pack_checksum,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ SoupURI *index_uri = NULL;
+ GFile *tmp_path = NULL;
+ char *pack_index_name = NULL;
+
+ pack_index_name = g_strconcat ("ostpack-", pack_checksum, ".index", NULL);
+ index_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_index_name, NULL);
+
+ if (!fetch_uri (pull_data, index_uri, "packindex-", &tmp_path,
+ cancellable, error))
+ goto out;
+
+ if (!ostree_repo_add_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
+ pack_checksum, tmp_path,
+ cancellable, error))
+ goto out;
+
+ if (!ot_gfile_unlink (tmp_path, cancellable, error))
+ goto out;
+
+ g_clear_object (&tmp_path);
+
+ ret = TRUE;
+ out:
+ if (tmp_path != NULL)
+ (void) ot_gfile_unlink (tmp_path, NULL, NULL);
+ g_clear_object (&tmp_path);
+ g_free (pack_index_name);
+ if (index_uri)
+ soup_uri_free (index_uri);
+ return ret;
+}
static gboolean
-fetch_object (OstreeRepo *repo,
- SoupSession *soup,
- SoupURI *baseuri,
- const char *checksum,
- OstreeObjectType objtype,
- GFile **out_temp_path,
- GCancellable *cancellable,
- GError **error)
+fetch_and_cache_pack_indexes (OtPullData *pull_data,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ SoupURI *superindex_uri = NULL;
+ GFile *superindex_tmppath = NULL;
+ GPtrArray *cached_indexes = NULL;
+ GPtrArray *uncached_indexes = NULL;
+ GVariant *superindex_variant = NULL;
+ GVariantIter *contents_iter = NULL;
+ guint i;
+
+ superindex_uri = suburi_new (pull_data->base_uri, "objects", "pack", "index", NULL);
+
+ if (!fetch_uri (pull_data, superindex_uri, "index-",
+ &superindex_tmppath, cancellable, error))
+ goto out;
+
+ if (!ostree_repo_resync_cached_remote_pack_indexes (pull_data->repo, pull_data->remote_name,
+ superindex_tmppath,
+ &cached_indexes, &uncached_indexes,
+ cancellable, error))
+ goto out;
+
+ for (i = 0; i < cached_indexes->len; i++)
+ g_ptr_array_add (pull_data->cached_pack_indexes,
+ g_strdup (cached_indexes->pdata[i]));
+
+ for (i = 0; i < uncached_indexes->len; i++)
+ {
+ const char *pack_checksum = uncached_indexes->pdata[i];
+
+ if (!fetch_one_cache_index (pull_data, pack_checksum, cancellable, error))
+ goto out;
+
+ g_ptr_array_add (pull_data->cached_pack_indexes, g_strdup (pack_checksum));
+ }
+
+ ret = TRUE;
+ out:
+ if (superindex_uri)
+ soup_uri_free (superindex_uri);
+ g_clear_object (&superindex_tmppath);
+ ot_clear_gvariant (&superindex_variant);
+ if (contents_iter)
+ g_variant_iter_free (contents_iter);
+ return ret;
+}
+
+static gboolean
+fetch_loose_object (OtPullData *pull_data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ GFile **out_temp_path,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
char *objpath = NULL;
- char *relpath = NULL;
SoupURI *obj_uri = NULL;
GFile *ret_temp_path = NULL;
objpath = ostree_get_relative_object_path (checksum, objtype);
- obj_uri = soup_uri_copy (baseuri);
- relpath = g_build_filename (soup_uri_get_path (obj_uri), objpath, NULL);
- soup_uri_set_path (obj_uri, relpath);
+ obj_uri = suburi_new (pull_data->base_uri, objpath, NULL);
- if (!fetch_uri (repo, soup, obj_uri, ostree_object_type_to_string (objtype), &ret_temp_path,
+ if (!fetch_uri (pull_data, obj_uri, ostree_object_type_to_string (objtype), &ret_temp_path,
cancellable, error))
goto out;
@@ -206,97 +428,309 @@ fetch_object (OstreeRepo *repo,
soup_uri_free (obj_uri);
g_clear_object (&ret_temp_path);
g_free (objpath);
- g_free (relpath);
return ret;
}
static gboolean
-fetch_and_store_object (OstreeRepo *repo,
- SoupSession *soup,
- SoupURI *baseuri,
- const char *checksum,
- OstreeObjectType objtype,
- gboolean *out_is_pending,
- GVariant **out_metadata,
- GCancellable *cancellable,
- GError **error)
+find_object (OtPullData *pull_data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ gboolean *out_is_stored,
+ gboolean *out_is_pending,
+ char **out_remote_pack_checksum,
+ guint64 *out_offset,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
- GFileInfo *file_info = NULL;
- GInputStream *input = NULL;
+ gboolean ret_is_stored;
+ gboolean ret_is_pending;
GFile *stored_path = NULL;
GFile *pending_path = NULL;
- char *pack_checksum = NULL;
- GFile *temp_path = NULL;
- GVariant *ret_metadata = NULL;
- gboolean ret_is_pending;
+ char *local_pack_checksum = NULL;
+ char *ret_remote_pack_checksum = NULL;
+ guint64 offset;
- g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE);
-
- if (!ostree_repo_find_object (repo, objtype, checksum,
+ if (!ostree_repo_find_object (pull_data->repo, objtype, checksum,
&stored_path, &pending_path,
- &pack_checksum, NULL,
+ &local_pack_checksum, NULL,
cancellable, error))
goto out;
-
- if (!(stored_path || pending_path || pack_checksum))
+
+ ret_is_stored = (stored_path != NULL || local_pack_checksum != NULL);
+ ret_is_pending = pending_path != NULL;
+
+ if (!(ret_is_stored || ret_is_pending))
{
- if (!fetch_object (repo, soup, baseuri, checksum, objtype, &temp_path, cancellable, error))
+ if (!find_object_in_remote_packs (pull_data, checksum, objtype,
+ &ret_remote_pack_checksum, &offset,
+ cancellable, error))
goto out;
}
- if (temp_path)
+ ret = TRUE;
+ if (out_is_stored)
+ *out_is_stored = ret_is_stored;
+ if (out_is_pending)
+ *out_is_pending = ret_is_pending;
+ ot_transfer_out_value (out_remote_pack_checksum, &ret_remote_pack_checksum);
+ out:
+ g_free (local_pack_checksum);
+ g_free (ret_remote_pack_checksum);
+ g_clear_object (&stored_path);
+ return ret;
+}
+
+static void
+unlink_file_on_unref (GFile *f)
+{
+ (void) ot_gfile_unlink (f, NULL, NULL);
+ g_object_unref (f);
+}
+
+static gboolean
+fetch_object_if_not_stored (OtPullData *pull_data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ gboolean *out_is_stored,
+ gboolean *out_is_pending,
+ GInputStream **out_input,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gboolean ret_is_stored = FALSE;
+ gboolean ret_is_pending = FALSE;
+ GInputStream *ret_input = NULL;
+ GFile *temp_path = NULL;
+ GFile *pack_path = NULL;
+ GMappedFile *pack_map = NULL;
+ char *remote_pack_checksum = NULL;
+ guint64 pack_offset;
+ GVariant *pack_entry = NULL;
+
+ if (!find_object (pull_data, checksum, objtype, &ret_is_stored,
+ &ret_is_pending, &remote_pack_checksum,
+ &pack_offset, cancellable, error))
+ goto out;
+
+ if (remote_pack_checksum != NULL)
{
- file_info = g_file_query_info (temp_path, OSTREE_GIO_FAST_QUERYINFO,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error);
- if (!file_info)
+ g_assert (!(ret_is_stored || ret_is_pending));
+
+ if (!fetch_one_pack_file (pull_data, remote_pack_checksum, &pack_path,
+ cancellable, error))
+ goto out;
+
+ pack_map = g_mapped_file_new (ot_gfile_get_path_cached (pack_path), FALSE, error);
+ if (!pack_map)
+ goto out;
+
+ if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map),
+ g_mapped_file_get_length (pack_map),
+ pack_offset, FALSE, &pack_entry,
+ cancellable, error))
+ goto out;
+
+ ret_input = ostree_read_pack_entry_as_stream (pack_entry);
+ }
+ else if (!(ret_is_stored || ret_is_pending))
+ {
+ if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error))
goto out;
- input = (GInputStream*)g_file_read (temp_path, cancellable, error);
- if (!input)
+ ret_input = (GInputStream*)g_file_read (temp_path, cancellable, error);
+ if (!ret_input)
goto out;
+ g_object_set_data_full ((GObject*)ret_input, "ostree-tmpfile-unlink",
+ g_object_ref (temp_path),
+ (GDestroyNotify)unlink_file_on_unref);
}
-
- if (pending_path || temp_path)
+
+ ret = TRUE;
+ ot_transfer_out_value (out_input, &ret_input);
+ if (out_is_stored)
+ *out_is_stored = ret_is_stored;
+ if (out_is_pending)
+ *out_is_pending = ret_is_pending;
+ out:
+ g_clear_object (&temp_path);
+ g_clear_object (&pack_path);
+ if (pack_map)
+ g_mapped_file_unref (pack_map);
+ ot_clear_gvariant (&pack_entry);
+ g_clear_object (&pack_path);
+ g_clear_object (&ret_input);
+ return ret;
+}
+
+static gboolean
+fetch_and_store_object (OtPullData *pull_data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ gboolean *out_is_pending,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFileInfo *file_info = NULL;
+ GInputStream *input = NULL;
+ GFile *stored_path = NULL;
+ GFile *pending_path = NULL;
+ char *pack_checksum = NULL;
+ gboolean is_stored;
+ gboolean ret_is_pending;
+
+ g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE);
+
+ if (!fetch_object_if_not_stored (pull_data, checksum, objtype,
+ &is_stored, &ret_is_pending, &input,
+ cancellable, error))
+ goto out;
+
+ if (ret_is_pending || input)
{
- if (!ostree_repo_stage_object (repo, objtype, checksum, file_info, NULL, input, cancellable, error))
+ if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, NULL, NULL,
+ input, cancellable, error))
goto out;
log_verbose ("Staged object: %s.%s", checksum, ostree_object_type_to_string (objtype));
+ }
- ret_is_pending = TRUE;
- if (out_metadata)
+ ret = TRUE;
+ if (out_is_pending)
+ *out_is_pending = ret_is_pending;
+ out:
+ g_clear_object (&file_info);
+ g_clear_object (&input);
+ g_clear_object (&stored_path);
+ g_clear_object (&pending_path);
+ g_free (pack_checksum);
+ return ret;
+}
+
+static gboolean
+fetch_and_store_metadata (OtPullData *pull_data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ gboolean *out_is_pending,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gboolean ret_is_pending;
+ GVariant *ret_variant = NULL;
+
+ if (!fetch_and_store_object (pull_data, checksum, objtype,
+ &ret_is_pending, cancellable, error))
+ goto out;
+
+ if (!ostree_repo_load_variant (pull_data->repo, objtype, checksum,
+ &ret_variant, error))
+ goto out;
+
+ ret = TRUE;
+ ot_transfer_out_value (out_variant, &ret_variant);
+ if (out_is_pending)
+ *out_is_pending = ret_is_pending;
+ out:
+ ot_clear_gvariant (&ret_variant);
+ return ret;
+}
+
+static gboolean
+fetch_and_store_file (OtPullData *pull_data,
+ const char *checksum,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GInputStream *input = NULL;
+ GFile *stored_path = NULL;
+ GFile *pending_path = NULL;
+ char *pack_checksum = NULL;
+ GVariant *archive_metadata = NULL;
+ GFileInfo *archive_file_info = NULL;
+ GVariant *archive_xattrs = NULL;
+ gboolean skip_archive_fetch;
+
+ /* If we're fetching from an archive into a bare repository, we need
+ * to explicitly check for raw file types locally.
+ */
+ if (ostree_repo_get_mode (pull_data->repo) == OSTREE_REPO_MODE_BARE)
+ {
+ if (!ostree_repo_find_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE,
+ checksum, &stored_path, &pending_path, &pack_checksum,
+ NULL, cancellable, error))
+ goto out;
+
+ if (stored_path || pack_checksum)
+ skip_archive_fetch = TRUE;
+ else if (pending_path != NULL)
{
- if (!ostree_map_metadata_file (pending_path ? pending_path : temp_path, objtype, &ret_metadata, error))
+ skip_archive_fetch = TRUE;
+ if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE,
+ checksum, NULL, NULL, NULL, cancellable, error))
goto out;
}
+ else
+ skip_archive_fetch = FALSE;
+
+ g_clear_object (&stored_path);
}
else
{
- ret_is_pending = FALSE;
+ skip_archive_fetch = FALSE;
}
+ if (!skip_archive_fetch)
+ {
+ if (!fetch_object_if_not_stored (pull_data, checksum,
+ OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
+ NULL, NULL, &input, cancellable, error))
+ goto out;
+
+ if (input != NULL)
+ {
+ if (!ot_util_variant_from_stream (input, OSTREE_SERIALIZED_VARIANT_FORMAT,
+ FALSE, &archive_metadata, cancellable, error))
+ goto out;
+
+ if (!ostree_parse_archived_file_meta (archive_metadata, &archive_file_info,
+ &archive_xattrs, error))
+ goto out;
+
+ g_clear_object (&input);
+ if (g_file_info_get_file_type (archive_file_info) == G_FILE_TYPE_REGULAR)
+ {
+ if (!fetch_object_if_not_stored (pull_data, checksum,
+ OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
+ NULL, NULL, &input,
+ cancellable, error))
+ goto out;
+ }
+
+ if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE, checksum,
+ archive_file_info, archive_xattrs, input,
+ cancellable, error))
+ goto out;
+ }
+ }
+
ret = TRUE;
- if (out_is_pending)
- *out_is_pending = ret_is_pending;
- ot_transfer_out_value (out_metadata, &ret_metadata);
out:
- if (temp_path)
- (void) unlink (ot_gfile_get_path_cached (temp_path));
- ot_clear_gvariant (&ret_metadata);
- g_clear_object (&temp_path);
- g_clear_object (&file_info);
- g_clear_object (&input);
+ g_free (pack_checksum);
g_clear_object (&stored_path);
g_clear_object (&pending_path);
- g_free (pack_checksum);
+ g_clear_object (&input);
+ ot_clear_gvariant (&archive_xattrs);
+ g_clear_object (&archive_file_info);
return ret;
}
static gboolean
-fetch_and_store_tree_recurse (OstreeRepo *repo,
- SoupSession *soup,
- SoupURI *base_uri,
+fetch_and_store_tree_recurse (OtPullData *pull_data,
const char *rev,
GCancellable *cancellable,
GError **error)
@@ -307,18 +741,12 @@ fetch_and_store_tree_recurse (OstreeRepo *repo,
GVariant *dirs_variant = NULL;
gboolean is_pending;
int i, n;
- GVariant *archive_metadata = NULL;
- GFileInfo *archive_file_info = NULL;
- GVariant *archive_xattrs = NULL;
- GFile *meta_temp_path = NULL;
- GFile *content_temp_path = NULL;
GFile *stored_path = NULL;
GFile *pending_path = NULL;
char *pack_checksum = NULL;
- GInputStream *input = NULL;
- if (!fetch_and_store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_DIR_TREE,
- &is_pending, &tree, cancellable, error))
+ if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_DIR_TREE,
+ &is_pending, &tree, cancellable, error))
goto out;
if (!is_pending)
@@ -342,85 +770,8 @@ fetch_and_store_tree_recurse (OstreeRepo *repo,
if (!ostree_validate_checksum_string (checksum, error))
goto out;
- g_clear_object (&stored_path);
- g_clear_object (&pending_path);
- g_free (pack_checksum);
- pack_checksum = NULL;
- /* If we're fetching from an archive into a bare repository, we need
- * to explicitly check for raw file types locally.
- */
- if (ostree_repo_get_mode (repo) == OSTREE_REPO_MODE_BARE)
- {
- if (!ostree_repo_find_object (repo, OSTREE_OBJECT_TYPE_RAW_FILE, checksum,
- &stored_path, &pending_path, &pack_checksum, NULL,
- cancellable, error))
- goto out;
- }
- else
- {
- if (!ostree_repo_find_object (repo, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, checksum,
- &stored_path, &pending_path, &pack_checksum, NULL,
- cancellable, error))
- goto out;
- }
-
- g_clear_object (&input);
- g_clear_object (&archive_file_info);
- ot_clear_gvariant (&archive_xattrs);
- if (!(stored_path || pending_path))
- {
- g_clear_object (&meta_temp_path);
- if (!fetch_object (repo, soup, base_uri, checksum,
- OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
- &meta_temp_path,
- cancellable,
- error))
- goto out;
-
- if (!ostree_map_metadata_file (meta_temp_path,
- OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
- &archive_metadata, error))
- goto out;
-
- if (!ostree_parse_archived_file_meta (archive_metadata, &archive_file_info, &archive_xattrs, error))
- goto out;
-
- if (g_file_info_get_file_type (archive_file_info) == G_FILE_TYPE_REGULAR)
- {
- if (!fetch_object (repo, soup, base_uri, checksum,
- OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
- &content_temp_path,
- cancellable,
- error))
- goto out;
-
- input = (GInputStream*)g_file_read (content_temp_path, cancellable, error);
- if (!input)
- goto out;
- }
- }
-
- if (!stored_path)
- {
- log_verbose ("Staged file object: %s", checksum);
-
- if (!ostree_repo_stage_object (repo, OSTREE_OBJECT_TYPE_RAW_FILE,
- checksum,
- archive_file_info, archive_xattrs, input,
- cancellable, error))
- goto out;
- }
-
- if (meta_temp_path)
- {
- (void) unlink (ot_gfile_get_path_cached (meta_temp_path));
- g_clear_object (&meta_temp_path);
- }
- if (content_temp_path)
- {
- (void) unlink (ot_gfile_get_path_cached (content_temp_path));
- g_clear_object (&content_temp_path);
- }
+ if (!fetch_and_store_file (pull_data, checksum, cancellable, error))
+ goto out;
}
n = g_variant_n_children (dirs_variant);
@@ -440,11 +791,11 @@ fetch_and_store_tree_recurse (OstreeRepo *repo,
if (!ostree_validate_checksum_string (meta_checksum, error))
goto out;
- if (!fetch_and_store_object (repo, soup, base_uri, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META,
- NULL, NULL, cancellable, error))
+ if (!fetch_and_store_object (pull_data, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META,
+ NULL, cancellable, error))
goto out;
- if (!fetch_and_store_tree_recurse (repo, soup, base_uri, tree_checksum, cancellable, error))
+ if (!fetch_and_store_tree_recurse (pull_data, tree_checksum, cancellable, error))
goto out;
}
}
@@ -454,30 +805,14 @@ fetch_and_store_tree_recurse (OstreeRepo *repo,
ot_clear_gvariant (&tree);
ot_clear_gvariant (&files_variant);
ot_clear_gvariant (&dirs_variant);
- ot_clear_gvariant (&archive_metadata);
- ot_clear_gvariant (&archive_xattrs);
- g_clear_object (&archive_file_info);
- g_clear_object (&input);
g_clear_object (&stored_path);
g_clear_object (&pending_path);
g_free (pack_checksum);
- if (content_temp_path)
- {
- (void) unlink (ot_gfile_get_path_cached (content_temp_path));
- g_clear_object (&content_temp_path);
- }
- if (meta_temp_path)
- {
- (void) unlink (ot_gfile_get_path_cached (meta_temp_path));
- g_clear_object (&meta_temp_path);
- }
return ret;
}
static gboolean
-fetch_and_store_commit_recurse (OstreeRepo *repo,
- SoupSession *soup,
- SoupURI *base_uri,
+fetch_and_store_commit_recurse (OtPullData *pull_data,
const char *rev,
GCancellable *cancellable,
GError **error)
@@ -488,8 +823,8 @@ fetch_and_store_commit_recurse (OstreeRepo *repo,
const char *tree_meta_checksum;
gboolean is_pending;
- if (!fetch_and_store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_COMMIT,
- &is_pending, &commit, cancellable, error))
+ if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_COMMIT,
+ &is_pending, &commit, cancellable, error))
goto out;
if (!is_pending)
@@ -500,11 +835,11 @@ fetch_and_store_commit_recurse (OstreeRepo *repo,
g_variant_get_child (commit, 6, "&s", &tree_contents_checksum);
g_variant_get_child (commit, 7, "&s", &tree_meta_checksum);
- if (!fetch_and_store_object (repo, soup, base_uri, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META,
- NULL, NULL, cancellable, error))
+ if (!fetch_and_store_object (pull_data, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META,
+ NULL, cancellable, error))
goto out;
- if (!fetch_and_store_tree_recurse (repo, soup, base_uri, tree_contents_checksum,
+ if (!fetch_and_store_tree_recurse (pull_data, tree_contents_checksum,
cancellable, error))
goto out;
}
@@ -516,9 +851,7 @@ fetch_and_store_commit_recurse (OstreeRepo *repo,
}
static gboolean
-fetch_ref_contents (OstreeRepo *repo,
- SoupSession *soup,
- SoupURI *base_uri,
+fetch_ref_contents (OtPullData *pull_data,
const char *ref,
char **out_contents,
GCancellable *cancellable,
@@ -526,14 +859,11 @@ fetch_ref_contents (OstreeRepo *repo,
{
gboolean ret = FALSE;
char *ret_contents = NULL;
- char *refpath = NULL;
SoupURI *target_uri = NULL;
- target_uri = soup_uri_copy (base_uri);
- refpath = g_build_filename (soup_uri_get_path (target_uri), "refs", "heads", ref, NULL);
- soup_uri_set_path (target_uri, refpath);
+ target_uri = suburi_new (pull_data->base_uri, "refs", "heads", ref, NULL);
- if (!fetch_uri_contents_utf8 (repo, soup, target_uri, &ret_contents, cancellable, error))
+ if (!fetch_uri_contents_utf8 (pull_data, target_uri, &ret_contents, cancellable, error))
goto out;
g_strchomp (ret_contents);
@@ -544,7 +874,6 @@ fetch_ref_contents (OstreeRepo *repo,
ret = TRUE;
ot_transfer_out_value (out_contents, &ret_contents);
out:
- g_free (refpath);
g_free (ret_contents);
if (target_uri)
soup_uri_free (target_uri);
@@ -552,12 +881,9 @@ fetch_ref_contents (OstreeRepo *repo,
}
static gboolean
-pull_one_commit (OstreeRepo *repo,
- const char *remote,
+pull_one_commit (OtPullData *pull_data,
const char *branch,
const char *rev,
- SoupSession *soup,
- SoupURI *base_uri,
GCancellable *cancellable,
GError **error)
{
@@ -567,9 +893,9 @@ pull_one_commit (OstreeRepo *repo,
char *baseurl = NULL;
char *original_rev = NULL;
- remote_ref = g_strdup_printf ("%s/%s", remote, branch);
+ remote_ref = g_strdup_printf ("%s/%s", pull_data->remote_name, branch);
- if (!ostree_repo_resolve_rev (repo, remote_ref, TRUE, &original_rev, error))
+ if (!ostree_repo_resolve_rev (pull_data->repo, remote_ref, TRUE, &original_rev, error))
goto out;
if (original_rev && strcmp (rev, original_rev) == 0)
@@ -581,16 +907,27 @@ pull_one_commit (OstreeRepo *repo,
if (!ostree_validate_checksum_string (rev, error))
goto out;
- if (!ostree_repo_prepare_transaction (repo, NULL, error))
+ if (!pull_data->fetched_packs)
+ {
+ pull_data->fetched_packs = TRUE;
+ pull_data->cached_pack_indexes = g_ptr_array_new_with_free_func (g_free);
+
+ g_print ("Fetching packs\n");
+
+ if (!fetch_and_cache_pack_indexes (pull_data, cancellable, error))
+ goto out;
+ }
+
+ if (!ostree_repo_prepare_transaction (pull_data->repo, NULL, error))
goto out;
- if (!fetch_and_store_commit_recurse (repo, soup, base_uri, rev, cancellable, error))
+ if (!fetch_and_store_commit_recurse (pull_data, rev, cancellable, error))
goto out;
- if (!ostree_repo_commit_transaction (repo, cancellable, error))
+ if (!ostree_repo_commit_transaction (pull_data->repo, cancellable, error))
goto out;
- if (!ostree_repo_write_ref (repo, remote, branch, rev, error))
+ if (!ostree_repo_write_ref (pull_data->repo, pull_data->remote_name, branch, rev, error))
goto out;
g_print ("remote %s is now %s\n", remote_ref, rev);
@@ -666,9 +1003,9 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
{
GOptionContext *context;
gboolean ret = FALSE;
+ OtPullData pull_data_real;
+ OtPullData *pull_data = &pull_data_real;
OstreeRepo *repo = NULL;
- const char *remote;
- SoupSession *soup = NULL;
char *path = NULL;
char *baseurl = NULL;
char *summary_data = NULL;
@@ -692,27 +1029,29 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
if (!ostree_repo_check (repo, error))
goto out;
+ memset (pull_data, 0, sizeof (*pull_data));
+ pull_data->repo = repo;
+
if (argc < 2)
{
ot_util_usage_error (context, "REMOTE must be specified", error);
goto out;
}
- remote = argv[1];
-
- soup = soup_session_sync_new_with_options (SOUP_SESSION_USER_AGENT, "ostree ",
- SOUP_SESSION_ADD_FEATURE_BY_TYPE, SOUP_TYPE_COOKIE_JAR,
- NULL);
+ pull_data->remote_name = g_strdup (argv[1]);
+ pull_data->session = soup_session_sync_new_with_options (SOUP_SESSION_USER_AGENT, "ostree ",
+ SOUP_SESSION_ADD_FEATURE_BY_TYPE, SOUP_TYPE_COOKIE_JAR,
+ NULL);
config = ostree_repo_get_config (repo);
- key = g_strdup_printf ("remote \"%s\"", remote);
+ key = g_strdup_printf ("remote \"%s\"", pull_data->remote_name);
baseurl = g_key_file_get_string (config, key, "url", error);
if (!baseurl)
goto out;
- base_uri = soup_uri_new (baseurl);
+ pull_data->base_uri = soup_uri_new (baseurl);
- if (!base_uri)
+ if (!pull_data->base_uri)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse url '%s'", baseurl);
@@ -727,7 +1066,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
const char *branch = argv[i];
char *contents;
- if (!fetch_ref_contents (repo, soup, base_uri, branch, &contents, cancellable, error))
+ if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
goto out;
/* Transfer ownership of contents */
@@ -740,7 +1079,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
path = g_build_filename (soup_uri_get_path (summary_uri), "refs", "summary", NULL);
soup_uri_set_path (summary_uri, path);
- if (!fetch_uri_contents_utf8 (repo, soup, summary_uri, &summary_data, cancellable, error))
+ if (!fetch_uri_contents_utf8 (pull_data, summary_uri, &summary_data, cancellable, error))
goto out;
if (!parse_ref_summary (summary_data, &refs_to_fetch, error))
@@ -754,7 +1093,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
const char *ref = key;
const char *sha256 = value;
- if (!pull_one_commit (repo, remote, ref, sha256, soup, base_uri, cancellable, error))
+ if (!pull_one_commit (pull_data, ref, sha256, cancellable, error))
goto out;
}
@@ -768,13 +1107,14 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
g_free (branch_rev);
if (context)
g_option_context_free (context);
- g_clear_object (&soup);
- if (base_uri)
- soup_uri_free (base_uri);
+ g_clear_object (&pull_data->session);
+ if (pull_data->base_uri)
+ soup_uri_free (pull_data->base_uri);
+ if (pull_data->cached_pack_indexes)
+ g_ptr_array_unref (pull_data->cached_pack_indexes);
if (summary_uri)
soup_uri_free (summary_uri);
g_clear_object (&repo);
- g_clear_object (&soup);
return ret;
}
diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c
index 06a305c..41876c4 100644
--- a/src/ostree/ot-builtin-fsck.c
+++ b/src/ostree/ot-builtin-fsck.c
@@ -204,6 +204,8 @@ fsck_pack_files (OtFsckData *data,
{
gboolean ret = FALSE;
GPtrArray *pack_indexes = NULL;
+ GVariant *index_variant = NULL;
+ GFile *pack_index_path = NULL;
GFile *pack_data_path = NULL;
GInputStream *input = NULL;
GChecksum *pack_content_checksum = NULL;
@@ -216,6 +218,22 @@ fsck_pack_files (OtFsckData *data,
{
const char *checksum = pack_indexes->pdata[i];
+ g_clear_object (&pack_index_path);
+ pack_index_path = ostree_repo_get_pack_index_path (data->repo, checksum);
+
+ ot_clear_gvariant (&index_variant);
+ if (!ot_util_variant_map (pack_index_path,
+ OSTREE_PACK_INDEX_VARIANT_FORMAT,
+ &index_variant, error))
+ goto out;
+
+ if (!ostree_validate_structureof_pack_index (index_variant, error))
+ {
+ g_prefix_error (error, "Corrupted pack index '%s': ",
+ ot_gfile_get_path_cached (pack_index_path));
+ goto out;
+ }
+
g_clear_object (&pack_data_path);
pack_data_path = ostree_repo_get_pack_data_path (data->repo, checksum);
diff --git a/src/ostree/ot-builtin-init.c b/src/ostree/ot-builtin-init.c
index 1bc5392..b194097 100644
--- a/src/ostree/ot-builtin-init.c
+++ b/src/ostree/ot-builtin-init.c
@@ -99,6 +99,11 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error)
if (!g_file_make_directory (child, NULL, error))
goto out;
+ g_clear_object (&child);
+ child = g_file_get_child (repo_path, "remote-cache");
+ if (!g_file_make_directory (child, NULL, error))
+ goto out;
+
ret = TRUE;
out:
if (context)
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
