[ostree] core: Add pack files
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree] core: Add pack files
- Date: Sat, 31 Mar 2012 15:52:41 +0000 (UTC)
commit 80bdfd7f42266bcac1dc6a71d98bbcb54c671673
Author: Colin Walters <walters verbum org>
Date: Sat Mar 31 10:37:51 2012 -0400
core: Add pack files
This concept is also directly inspired by git. At present, our
implementation is quite similar, except we don't have delta
compression.
Makefile-ostree.am | 2 +
src/libostree/README.md | 78 +
src/libostree/ostree-core.c | 528 +++++++-
src/libostree/ostree-core.h | 100 ++
src/libostree/ostree-repo-file.c | 76 +-
src/libostree/ostree-repo.c | 2773 +++++++++++++++++++++++++----------
src/libostree/ostree-repo.h | 118 ++-
src/ostree/main.c | 2 +
src/ostree/ostree-pull.c | 797 ++++++++---
src/ostree/ot-builtin-fsck.c | 185 ++-
src/ostree/ot-builtin-init.c | 30 +-
src/ostree/ot-builtin-local-clone.c | 77 +-
src/ostree/ot-builtin-pack.c | 920 ++++++++++++
src/ostree/ot-builtin-prune.c | 60 +-
src/ostree/ot-builtin-unpack.c | 305 ++++
src/ostree/ot-builtins.h | 2 +
tests/t0001-archive.sh | 35 +-
tests/t0010-pull.sh | 21 +-
18 files changed, 4965 insertions(+), 1144 deletions(-)
---
diff --git a/Makefile-ostree.am b/Makefile-ostree.am
index 83333eb..0eedd14 100644
--- a/Makefile-ostree.am
+++ b/Makefile-ostree.am
@@ -34,6 +34,8 @@ ostree_SOURCES = src/ostree/main.c \
src/ostree/ot-builtin-ls.c \
src/ostree/ot-builtin-prune.c \
src/ostree/ot-builtin-remote.c \
+ src/ostree/ot-builtin-pack.c \
+ src/ostree/ot-builtin-unpack.c \
src/ostree/ot-builtin-rev-parse.c \
src/ostree/ot-builtin-show.c \
src/ostree/ot-main.h \
diff --git a/src/libostree/README.md b/src/libostree/README.md
new file mode 100644
index 0000000..83d9f5f
--- /dev/null
+++ b/src/libostree/README.md
@@ -0,0 +1,78 @@
+Repository design
+-----------------
+
+At the heart of OSTree is the repository. It's very similar to git,
+with the idea of content-addressed storage. However, OSTree is
+designed to store operating system binaries, not source code. There
+are several consequences to this. The key difference as compared to
+git is that the OSTree definition of "content" includes key Unix
+metadata such as owner uid/gid, as well as all extended attributes.
+
+Essentially OSTree is designed so that if two files have the same
+OSTree checksum, it's safe to replace them with a hard link. This
+fundamental design means that an OSTree repository imposes negligible
+overhead. In contrast, a git repository stores copies of
+zlib-compressed data.
+
+Key differences versus git
+--------------------------
+
+ * As mentioned above, extended attributes and owner uid/gid are versioned
+ * Optimized for Unix hardlinks between repository and checkout
+ * SHA256 instead of SHA1
+ * Support for empty directories
+
+Binary files
+------------
+
+While this is still in planning, I plan to heavily optimize OSTree for
+versioning ELF operating systems. In industry jargon, this would be
+"content-aware storage".
+
+Trimming history
+----------------
+
+OSTree will also be optimized to trim intermediate history; in theory
+one can regenerate binaries from corresponding (git) source code, so
+we don't need to keep all possible builds over time.
+
+MILESTONE 1
+-----------
+* Basic pack files (like git)
+
+MILESTONE 2
+-----------
+* Store checksums as ay
+* Drop version/metadata from tree/dirmeta objects
+* Split pack files into metadata/data
+* Restructure repository so that links can be generated as a cache;
+ i.e. objects/raw, pack files are now the canonical
+* For files, checksum combination of metadata variant + raw data
+ - i.e. there is only OSTREE_OBJECT_TYPE_FILE (again)
+
+MILESTONE 3
+-----------
+
+* Drop archive/raw distinction - archive repositories always generate
+ packfiles per commit
+* Include git packv4 ideas:
+ - metadata packfiles have string dictionary (tree filenames and checksums)
+ - data packfiles match up similar objects
+* Rolling checksums for partitioning large files? Kernel debuginfo
+* Improved pack clustering
+ - file fingerprinting?
+* ELF-x86 aware deltas
+
+Related work in storage
+-----------------------
+
+git: http://git-scm.com/
+Venti: http://plan9.bell-labs.com/magic/man2html/6/venti
+Elephant FS: http://www.hpl.hp.com/personal/Alistair_Veitch/papers/elephant-hotos/index.html
+
+Compression
+-----------
+
+xdelta: http://xdelta.org/
+Bsdiff: http://www.daemonology.net/bsdiff/
+xz: http://tukaani.org/xz/
diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c
index 263988c..1e3a9e1 100644
--- a/src/libostree/ostree-core.c
+++ b/src/libostree/ostree-core.c
@@ -28,6 +28,9 @@
#include <sys/types.h>
#include <attr/xattr.h>
+#define ALIGN_VALUE(this, boundary) \
+ (( ((unsigned long)(this)) + (((unsigned long)(boundary)) -1)) & (~(((unsigned long)(boundary))-1)))
+
gboolean
ostree_validate_checksum_string (const char *sha256,
GError **error)
@@ -487,6 +490,34 @@ ostree_set_xattrs (GFile *f,
}
gboolean
+ostree_unwrap_metadata (GVariant *container,
+ OstreeObjectType expected_type,
+ GVariant **out_variant,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GVariant *ret_variant = NULL;
+ guint32 actual_type;
+
+ g_variant_get (container, "(uv)",
+ &actual_type, &ret_variant);
+ actual_type = GUINT32_FROM_BE (actual_type);
+ if (actual_type != expected_type)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Corrupted metadata object; found type %u, expected %u",
+ actual_type, (guint32)expected_type);
+ goto out;
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_variant, &ret_variant);
+ out:
+ ot_clear_gvariant (&ret_variant);
+ return ret;
+}
+
+gboolean
ostree_map_metadata_file (GFile *file,
OstreeObjectType expected_type,
GVariant **out_variant,
@@ -495,22 +526,16 @@ ostree_map_metadata_file (GFile *file,
gboolean ret = FALSE;
GVariant *ret_variant = NULL;
GVariant *container = NULL;
- guint32 actual_type;
if (!ot_util_variant_map (file, OSTREE_SERIALIZED_VARIANT_FORMAT,
&container, error))
goto out;
- g_variant_get (container, "(uv)",
- &actual_type, &ret_variant);
- ot_util_variant_take_ref (ret_variant);
- actual_type = GUINT32_FROM_BE (actual_type);
- if (actual_type != expected_type)
+ if (!ostree_unwrap_metadata (container, expected_type, &ret_variant,
+ error))
{
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Corrupted metadata object '%s'; found type %u, expected %u",
- ot_gfile_get_path_cached (file),
- actual_type, (guint32)expected_type);
+ g_prefix_error (error, "While parsing '%s': ",
+ ot_gfile_get_path_cached (file));
goto out;
}
@@ -584,6 +609,142 @@ ostree_object_from_string (const char *str,
*out_objtype = ostree_object_type_from_string (dot + 1);
}
+guint
+ostree_hash_object_name (gconstpointer a)
+{
+ GVariant *variant = (gpointer)a;
+ const char *checksum;
+ OstreeObjectType objtype;
+ gint objtype_int;
+
+ ostree_object_name_deserialize (variant, &checksum, &objtype);
+ objtype_int = (gint) objtype;
+ return g_str_hash (checksum) + g_int_hash (&objtype_int);
+}
+
+int
+ostree_cmp_checksum_bytes (GVariant *a,
+ GVariant *b)
+{
+ gconstpointer a_data;
+ gconstpointer b_data;
+ gsize a_n_elts;
+ gsize b_n_elts;
+
+ a_data = g_variant_get_fixed_array (a, &a_n_elts, 1);
+ g_assert (a_n_elts == 32);
+ b_data = g_variant_get_fixed_array (b, &b_n_elts, 1);
+ g_assert (b_n_elts == 32);
+
+ return memcmp (a_data, b_data, 32);
+}
+
+
+GVariant *
+ostree_object_name_serialize (const char *checksum,
+ OstreeObjectType objtype)
+{
+ return g_variant_new ("(su)", checksum, (guint32)objtype);
+}
+
+void
+ostree_object_name_deserialize (GVariant *variant,
+ const char **out_checksum,
+ OstreeObjectType *out_objtype)
+{
+ guint32 objtype_u32;
+ g_variant_get (variant, "(&su)", out_checksum, &objtype_u32);
+ *out_objtype = (OstreeObjectType)objtype_u32;
+}
+
+GVariant *
+ostree_checksum_to_bytes (const char *sha256)
+{
+ guchar result[32];
+ guint i;
+ guint j;
+
+ for (i = 0, j = 0; i < 32; i += 1, j += 2)
+ {
+ gint big, little;
+
+ g_assert (sha256[j]);
+ g_assert (sha256[j+1]);
+
+ big = g_ascii_xdigit_value (sha256[j]);
+ little = g_ascii_xdigit_value (sha256[j+1]);
+
+ g_assert (big != -1);
+ g_assert (little != -1);
+
+ result[i] = (big << 4) | little;
+ }
+
+ return g_variant_new_fixed_array (G_VARIANT_TYPE ("y"),
+ (guchar*)result, 32, 1);
+}
+
+char *
+ostree_checksum_from_bytes (GVariant *csum_bytes)
+{
+ static const gchar hexchars[] = "0123456789abcdef";
+ char *ret;
+ const guchar *bytes;
+ gsize n_elts;
+ guint i, j;
+
+ bytes = g_variant_get_fixed_array (csum_bytes, &n_elts, 1);
+ g_assert (n_elts == 32);
+
+ ret = g_malloc (65);
+
+ for (i = 0, j = 0; i < 32; i++, j += 2)
+ {
+ guchar byte = bytes[i];
+ ret[j] = hexchars[byte >> 4];
+ ret[j+1] = hexchars[byte & 0xF];
+ }
+ ret[j] = '\0';
+
+ return ret;
+}
+
+GVariant *
+ostree_object_name_serialize_v2 (const char *checksum,
+ OstreeObjectType objtype)
+{
+ return g_variant_new ("(u ay)", (guint32)objtype, ostree_checksum_to_bytes (checksum));
+}
+
+void
+ostree_object_name_deserialize_v2_hex (GVariant *variant,
+ char **out_checksum,
+ OstreeObjectType *out_objtype)
+{
+ GVariant *csum_bytes;
+ guint32 objtype_u32;
+
+ g_variant_get (variant, "(u ay)", &objtype_u32, &csum_bytes);
+ g_variant_ref_sink (csum_bytes);
+ *out_checksum = ostree_checksum_from_bytes (csum_bytes);
+ g_variant_unref (csum_bytes);
+ *out_objtype = (OstreeObjectType)objtype_u32;
+}
+
+void
+ostree_object_name_deserialize_v2_bytes (GVariant *variant,
+ const guchar **out_checksum,
+ OstreeObjectType *out_objtype)
+{
+ GVariant *csum_bytes;
+ guint32 objtype_u32;
+ gsize n_elts;
+
+ g_variant_get (variant, "(u ay)", &objtype_u32, &csum_bytes);
+ *out_checksum = (guchar*)g_variant_get_fixed_array (csum_bytes, &n_elts, 1);
+ *out_objtype = (OstreeObjectType)objtype_u32;
+}
+
char *
ostree_get_relative_object_path (const char *checksum,
OstreeObjectType type)
@@ -1074,3 +1235,350 @@ ostree_create_temp_hardlink (GFile *dir,
g_clear_object (&possible_file);
return ret;
}
+
+gboolean
+ostree_read_pack_entry_raw (guchar *pack_data,
+ guint64 pack_len,
+ guint64 offset,
+ gboolean trusted,
+ GVariant **out_entry,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GVariant *ret_entry = NULL;
+ guint64 entry_start;
+ guint64 entry_end;
+ guint32 entry_len;
+
+ if (G_UNLIKELY (!(offset <= pack_len)))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Corrupted pack index; out of range offset %" G_GUINT64_FORMAT,
+ offset);
+ goto out;
+ }
+ if (G_UNLIKELY (!((offset & 0x3) == 0)))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Corrupted pack index; unaligned offset %" G_GUINT64_FORMAT,
+ offset);
+ goto out;
+ }
+
+ entry_start = ALIGN_VALUE (offset + 4, 8);
+ if (G_UNLIKELY (!(entry_start <= pack_len)))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Corrupted pack index; out of range data offset %" G_GUINT64_FORMAT,
+ entry_start);
+ goto out;
+ }
+
+ g_assert ((((guint64)pack_data+offset) & 0x3) == 0);
+ entry_len = GUINT32_FROM_BE (*((guint32*)(pack_data+offset)));
+
+ entry_end = entry_start + entry_len;
+ if (G_UNLIKELY (!(entry_end <= pack_len)))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Corrupted pack index; out of range entry length %u",
+ entry_len);
+ goto out;
+ }
+
+ ret_entry = g_variant_new_from_data (OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT,
+ pack_data+entry_start, entry_len,
+ trusted, NULL, NULL);
+ ret = TRUE;
+ ot_transfer_out_value (out_entry, &ret_entry);
+ out:
+ ot_clear_gvariant (&ret_entry);
+ return ret;
+}
+
+GInputStream *
+ostree_read_pack_entry_as_stream (GVariant *pack_entry)
+{
+ GInputStream *memory_input;
+ GInputStream *ret_input = NULL;
+ GVariant *pack_data = NULL;
+ guchar entry_flags;
+ gconstpointer data_ptr;
+ gsize data_len;
+
+ g_variant_get_child (pack_entry, 1, "y", &entry_flags);
+ g_variant_get_child (pack_entry, 3, "@ay", &pack_data);
+
+ data_ptr = g_variant_get_fixed_array (pack_data, &data_len, 1);
+ memory_input = g_memory_input_stream_new_from_data (data_ptr, data_len, NULL);
+ g_object_set_data_full ((GObject*)memory_input, "ostree-mem-gvariant",
+ pack_data, (GDestroyNotify) g_variant_unref);
+
+ if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
+ {
+ GConverter *decompressor;
+
+ decompressor = (GConverter*)g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP);
+ ret_input = (GInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
+ "converter", decompressor,
+ "base-stream", memory_input,
+ "close-base-stream", TRUE,
+ NULL);
+ g_object_unref (decompressor);
+ }
+ else
+ {
+ ret_input = memory_input;
+ memory_input = NULL;
+ }
+
+ return ret_input;
+}
+
+gboolean
+ostree_read_pack_entry_variant (GVariant *pack_entry,
+ OstreeObjectType expected_objtype,
+ gboolean trusted,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GInputStream *stream = NULL;
+ GVariant *container_variant = NULL;
+ GVariant *ret_variant = NULL;
+ guint32 actual_type;
+
+ stream = ostree_read_pack_entry_as_stream (pack_entry);
+
+ if (!ot_util_variant_from_stream (stream, OSTREE_SERIALIZED_VARIANT_FORMAT,
+ trusted, &container_variant, cancellable, error))
+ goto out;
+
+ g_variant_ref_sink (container_variant);
+
+ g_variant_get (container_variant, "(uv)",
+ &actual_type, &ret_variant);
+ actual_type = GUINT32_FROM_BE (actual_type);
+
+ if (actual_type != expected_objtype)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Corrupted metadata object in pack file; found type %u, expected %u",
+ actual_type, (guint32)expected_objtype);
+ goto out;
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_variant, &ret_variant);
+ out:
+ g_clear_object (&stream);
+ ot_clear_gvariant (&ret_variant);
+ ot_clear_gvariant (&container_variant);
+ return ret;
+}
+
+gboolean
+ostree_pack_index_search (GVariant *index,
+ GVariant *csum_bytes,
+ OstreeObjectType objtype,
+ guint64 *out_offset)
+{
+ gboolean ret = FALSE;
+ GVariant *index_contents;
+ gsize imax, imin;
+ gsize n;
+ guint32 target_objtype;
+
+ index_contents = g_variant_get_child_value (index, 2);
+
+ target_objtype = (guint32) objtype;
+
+ n = g_variant_n_children (index_contents);
+
+ if (n == 0)
+ goto out;
+
+ imax = n - 1;
+ imin = 0;
+ while (imax >= imin)
+ {
+ GVariant *cur_csum_bytes;
+ guint32 cur_objtype;
+ guint64 cur_offset;
+ gsize imid;
+ int c;
+
+ imid = (imin + imax) / 2;
+
+ g_variant_get_child (index_contents, imid, "(u ayt)", &cur_objtype,
+ &cur_csum_bytes, &cur_offset);
+ cur_objtype = GUINT32_FROM_BE (cur_objtype);
+
+ c = ostree_cmp_checksum_bytes (cur_csum_bytes, csum_bytes);
+ if (c == 0)
+ {
+ if (cur_objtype < target_objtype)
+ c = -1;
+ else if (cur_objtype > target_objtype)
+ c = 1;
+ }
+ g_variant_unref (cur_csum_bytes);
+
+ if (c < 0)
+ imin = imid + 1;
+ else if (c > 0)
+ {
+ if (imid == 0)
+ goto out;
+ imax = imid - 1;
+ }
+ else
+ {
+ if (out_offset)
+ *out_offset = GUINT64_FROM_BE (cur_offset);
+ ret = TRUE;
+ goto out;
+ }
+ }
+
+ out:
+ ot_clear_gvariant (&index_contents);
+ return ret;
+}
+
+gboolean
+ostree_validate_structureof_objtype (guint32 objtype,
+ GError **error)
+{
+ objtype = GUINT32_FROM_BE (objtype);
+ if (objtype < OSTREE_OBJECT_TYPE_RAW_FILE
+ || objtype > OSTREE_OBJECT_TYPE_COMMIT)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid object type '%u'", objtype);
+ return FALSE;
+ }
+ return TRUE;
+}
+
+gboolean
+ostree_validate_structureof_checksum (GVariant *checksum,
+ GError **error)
+{
+ gsize n_children = g_variant_n_children (checksum);
+ if (n_children != 32)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid checksum of length %" G_GUINT64_FORMAT
+ " expected 32", (guint64) n_children);
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static gboolean
+validate_variant (GVariant *variant,
+ const GVariantType *variant_type,
+ GError **error)
+{
+ if (!g_variant_is_normal_form (variant))
+ {
+ g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Not normal form");
+ return FALSE;
+ }
+ if (!g_variant_is_of_type (variant, variant_type))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Doesn't match variant type '%s'",
+ (char*)variant_type);
+ return FALSE;
+ }
+ return TRUE;
+}
+
+gboolean
+ostree_validate_structureof_pack_index (GVariant *index,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ const char *header;
+ GVariantIter *content_iter = NULL;
+ guint32 objtype;
+ GVariant *csum_bytes = NULL;
+ guint64 offset;
+
+ if (!validate_variant (index, OSTREE_PACK_INDEX_VARIANT_FORMAT, error))
+ goto out;
+
+ g_variant_get_child (index, 0, "&s", &header);
+
+ if (strcmp (header, "OSTv0PACKINDEX") != 0)
+ {
+ g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid pack index; doesn't match header");
+ goto out;
+ }
+
+ g_variant_get_child (index, 2, "a(uayt)", &content_iter);
+
+ while (g_variant_iter_loop (content_iter, "(u ayt)",
+ &objtype, &csum_bytes, &offset))
+ {
+ if (!ostree_validate_structureof_objtype (objtype, error))
+ goto out;
+ if (!ostree_validate_structureof_checksum (csum_bytes, error))
+ goto out;
+ }
+ csum_bytes = NULL;
+
+ ret = TRUE;
+ out:
+ if (content_iter)
+ g_variant_iter_free (content_iter);
+ ot_clear_gvariant (&csum_bytes);
+ return ret;
+}
+
+gboolean
+ostree_validate_structureof_pack_superindex (GVariant *superindex,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ const char *header;
+ GVariant *csum_bytes = NULL;
+ GVariant *bloom = NULL;
+ GVariantIter *content_iter = NULL;
+
+ if (!validate_variant (superindex, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT, error))
+ goto out;
+
+ g_variant_get_child (superindex, 0, "&s", &header);
+
+ if (strcmp (header, "OSTv0SUPERPACKINDEX") != 0)
+ {
+ g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid pack superindex; doesn't match header");
+ goto out;
+ }
+
+ g_variant_get_child (superindex, 2, "a(ayay)", &content_iter);
+
+ while (g_variant_iter_loop (content_iter, "(@ay ay)",
+ &csum_bytes, &bloom))
+ {
+ if (!ostree_validate_structureof_checksum (csum_bytes, error))
+ goto out;
+ }
+ csum_bytes = NULL;
+
+ ret = TRUE;
+ out:
+ if (content_iter)
+ g_variant_iter_free (content_iter);
+ ot_clear_gvariant (&csum_bytes);
+ ot_clear_gvariant (&bloom);
+ return ret;
+}
diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h
index a3abb3f..e733f51 100644
--- a/src/libostree/ostree-core.h
+++ b/src/libostree/ostree-core.h
@@ -97,9 +97,43 @@ typedef enum {
*/
#define OSTREE_ARCHIVED_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(uuuuusa(ayay))")
+/* Pack super index
+ * s - OSTv0SUPERPACKINDEX
+ * a{sv} - Metadata
+ * a(say) - (pack file checksum, bloom filter)
+ */
+#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(ayay))")
+
+/* Pack index
+ * s - OSTv0PACKINDEX
+ * a{sv} - Metadata
+ * a(uayt) - (objtype, checksum, offset into packfile)
+ */
+#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}a(uayt))")
+
+typedef enum {
+ OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0,
+ OSTREE_PACK_FILE_ENTRY_FLAG_GZIP = (1 << 0)
+} OstreePackFileEntryFlag;
+
+/* Pack files
+ * s - OSTv0PACKFILE
+ * a{sv} - Metadata
+ * t - number of entries
+ *
+ * Repeating pair of:
+ * <padding to alignment of 8>
+ * ( uyayay ) - objtype, flags, checksum, data
+ */
+#define OSTREE_PACK_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(sa{sv}t)")
+
+#define OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT G_VARIANT_TYPE ("(uyayay)")
+
gboolean ostree_validate_checksum_string (const char *sha256,
GError **error);
+GVariant *ostree_checksum_to_bytes (const char *sha256);
+
gboolean ostree_validate_rev (const char *rev, GError **error);
void ostree_checksum_update_stat (GChecksum *checksum, guint32 uid, guint32 gid, guint32 mode);
@@ -108,6 +142,32 @@ const char * ostree_object_type_to_string (OstreeObjectType objtype);
OstreeObjectType ostree_object_type_from_string (const char *str);
+guint ostree_hash_object_name (gconstpointer a);
+
+int ostree_cmp_checksum_bytes (GVariant *a, GVariant *b);
+
+GVariant *ostree_object_name_serialize (const char *checksum,
+ OstreeObjectType objtype);
+
+void ostree_object_name_deserialize (GVariant *variant,
+ const char **out_checksum,
+ OstreeObjectType *out_objtype);
+
+GVariant *ostree_object_name_serialize_v2 (const char *checksum,
+ OstreeObjectType objtype);
+
+void ostree_object_name_deserialize_v2_hex (GVariant *variant,
+ char **out_checksum,
+ OstreeObjectType *out_objtype);
+
+
+void ostree_object_name_deserialize_v2_bytes (GVariant *variant,
+ const guchar **out_checksum,
+ OstreeObjectType *out_objtype);
+
+GVariant * ostree_checksum_to_bytes (const char *sha256);
+char * ostree_checksum_from_bytes (GVariant *bytes);
+
char * ostree_object_to_string (const char *checksum,
OstreeObjectType objtype);
@@ -123,6 +183,11 @@ GVariant *ostree_get_xattrs_for_file (GFile *f,
GVariant *ostree_wrap_metadata_variant (OstreeObjectType type, GVariant *metadata);
+gboolean ostree_unwrap_metadata (GVariant *container,
+ OstreeObjectType expected_type,
+ GVariant **out_variant,
+ GError **error);
+
gboolean ostree_set_xattrs (GFile *f, GVariant *xattrs,
GCancellable *cancellable, GError **error);
@@ -205,5 +270,40 @@ gboolean ostree_parse_archived_file_meta (GVariant *data,
GVariant **out_xattrs,
GError **error);
+gboolean ostree_read_pack_entry_raw (guchar *pack_data,
+ guint64 pack_len,
+ guint64 object_offset,
+ gboolean trusted,
+ GVariant **out_entry,
+ GCancellable *cancellable,
+ GError **error);
+
+GInputStream *ostree_read_pack_entry_as_stream (GVariant *pack_entry);
+
+gboolean ostree_read_pack_entry_variant (GVariant *pack_entry,
+ OstreeObjectType expected_objtype,
+ gboolean trusted,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_pack_index_search (GVariant *index,
+ GVariant *csum_bytes,
+ OstreeObjectType objtype,
+ guint64 *out_offset);
+
+/** VALIDATION **/
+
+gboolean ostree_validate_structureof_objtype (guint32 objtype,
+ GError **error);
+
+gboolean ostree_validate_structureof_checksum (GVariant *checksum,
+ GError **error);
+
+gboolean ostree_validate_structureof_pack_index (GVariant *index,
+ GError **error);
+
+gboolean ostree_validate_structureof_pack_superindex (GVariant *superindex,
+ GError **error);
#endif /* _OSTREE_REPO */
diff --git a/src/libostree/ostree-repo-file.c b/src/libostree/ostree-repo-file.c
index 9cc7b82..ac4b129 100644
--- a/src/libostree/ostree-repo-file.c
+++ b/src/libostree/ostree-repo-file.c
@@ -724,39 +724,46 @@ bsearch_in_file_variant (GVariant *variant,
const char *name,
int *out_pos)
{
- int i, n;
- int m;
+ gsize imax, imin;
+ gsize imid;
+ gsize n;
- i = 0;
- n = g_variant_n_children (variant) - 1;
- m = 0;
+ n = g_variant_n_children (variant);
+ if (n == 0)
+ return FALSE;
- while (i <= n)
+ imax = n - 1;
+ imin = 0;
+ while (imax >= imin)
{
GVariant *child;
const char *cur;
int cmp;
- m = i + ((n - i) / 2);
+ imid = (imin + imax) / 2;
- child = g_variant_get_child_value (variant, m);
+ child = g_variant_get_child_value (variant, imid);
g_variant_get_child (child, 0, "&s", &cur, NULL);
cmp = strcmp (cur, name);
if (cmp < 0)
- i = m + 1;
+ imin = imid + 1;
else if (cmp > 0)
- n = m - 1;
+ {
+ if (imid == 0)
+ break;
+ imax = imid - 1;
+ }
else
{
ot_clear_gvariant (&child);
- *out_pos = m;
+ *out_pos = imid;
return TRUE;
}
ot_clear_gvariant (&child);
}
- *out_pos = m;
+ *out_pos = imid;
return FALSE;
}
@@ -817,13 +824,9 @@ ostree_repo_file_tree_query_child (OstreeRepoFile *self,
const char *name = NULL;
gboolean ret = FALSE;
GFileInfo *ret_info = NULL;
- GFile *archive_data_path = NULL;
- GFileInfo *archive_data_info = NULL;
- GVariant *archive_metadata = NULL;
GVariant *files_variant = NULL;
GVariant *dirs_variant = NULL;
GVariant *tree_child_metadata = NULL;
- GFile *local_child = NULL;
GFileAttributeMatcher *matcher = NULL;
int c;
@@ -844,40 +847,9 @@ ostree_repo_file_tree_query_child (OstreeRepoFile *self,
g_variant_get_child (files_variant, n, "(&s&s)", &name, &checksum);
- local_child = ostree_repo_get_file_object_path (self->repo, checksum);
-
- if (ostree_repo_get_mode (self->repo) == OSTREE_REPO_MODE_ARCHIVE)
- {
- if (!ostree_map_metadata_file (local_child, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
- &archive_metadata, error))
- goto out;
- if (!ostree_parse_archived_file_meta (archive_metadata, &ret_info, NULL, error))
- goto out;
-
- archive_data_path = ostree_repo_get_object_path (self->repo, checksum,
- OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
- archive_data_info = g_file_query_info (archive_data_path,
- OSTREE_GIO_FAST_QUERYINFO,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
- cancellable,
- error);
- if (!archive_data_info)
- goto out;
-
- g_file_info_set_attribute_uint64 (ret_info, "standard::size",
- g_file_info_get_attribute_uint64 (archive_data_info,
- "standard::size"));
- }
- else
- {
- ret_info = g_file_query_info (local_child,
- OSTREE_GIO_FAST_QUERYINFO,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
- cancellable,
- error);
- if (!ret_info)
- goto out;
- }
+ if (!ostree_repo_load_file (self->repo, checksum, NULL, &ret_info, NULL,
+ cancellable, error))
+ goto out;
}
else
{
@@ -918,12 +890,8 @@ ostree_repo_file_tree_query_child (OstreeRepoFile *self,
ot_transfer_out_value(out_info, &ret_info);
out:
g_clear_object (&ret_info);
- g_clear_object (&local_child);
- g_clear_object (&archive_data_path);
- g_clear_object (&archive_data_info);
if (matcher)
g_file_attribute_matcher_unref (matcher);
- ot_clear_gvariant (&archive_metadata);
ot_clear_gvariant (&tree_child_metadata);
ot_clear_gvariant (&files_variant);
ot_clear_gvariant (&dirs_variant);
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 4c99d88..1461869 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -60,6 +60,8 @@ struct _OstreeRepoPrivate {
GFile *local_heads_dir;
GFile *remote_heads_dir;
GFile *objects_dir;
+ GFile *pack_dir;
+ GFile *remote_cache_dir;
GFile *config_file;
gboolean inited;
@@ -68,6 +70,9 @@ struct _OstreeRepoPrivate {
GKeyFile *config;
OstreeRepoMode mode;
+ GHashTable *pack_index_mappings;
+ GHashTable *pack_data_mappings;
+
GHashTable *pending_transaction;
};
@@ -83,7 +88,11 @@ ostree_repo_finalize (GObject *object)
g_clear_object (&priv->local_heads_dir);
g_clear_object (&priv->remote_heads_dir);
g_clear_object (&priv->objects_dir);
+ g_clear_object (&priv->pack_dir);
+ g_clear_object (&priv->remote_cache_dir);
g_clear_object (&priv->config_file);
+ g_hash_table_destroy (priv->pack_index_mappings);
+ g_hash_table_destroy (priv->pack_data_mappings);
g_hash_table_destroy (priv->pending_transaction);
if (priv->config)
g_key_file_free (priv->config);
@@ -154,6 +163,8 @@ ostree_repo_constructor (GType gtype,
priv->remote_heads_dir = g_file_resolve_relative_path (priv->repodir, "refs/remotes");
priv->objects_dir = g_file_get_child (priv->repodir, "objects");
+ priv->pack_dir = g_file_get_child (priv->objects_dir, "pack");
+ priv->remote_cache_dir = g_file_get_child (priv->repodir, "remote-cache");
priv->config_file = g_file_get_child (priv->repodir, "config");
return object;
@@ -185,6 +196,12 @@ ostree_repo_init (OstreeRepo *self)
{
OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ priv->pack_index_mappings = g_hash_table_new_full (g_str_hash, g_str_equal,
+ g_free,
+ (GDestroyNotify)g_variant_unref);
+ priv->pack_data_mappings = g_hash_table_new_full (g_str_hash, g_str_equal,
+ g_free,
+ (GDestroyNotify)g_mapped_file_unref);
priv->pending_transaction = g_hash_table_new_full (g_str_hash, g_str_equal,
g_free,
NULL);
@@ -335,7 +352,6 @@ ostree_repo_resolve_rev (OstreeRepo *self,
GFile *origindir = NULL;
GError *temp_error = NULL;
GVariant *commit = NULL;
- GPtrArray *components = NULL;
g_return_val_if_fail (rev != NULL, FALSE);
@@ -768,48 +784,6 @@ get_pending_object_path (OstreeRepo *self,
return ret;
}
-gboolean
-ostree_repo_find_object (OstreeRepo *self,
- OstreeObjectType objtype,
- const char *checksum,
- GFile **out_stored_path,
- GFile **out_pending_path,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- GFile *object_path = NULL;
- struct stat stbuf;
-
- g_return_val_if_fail (out_stored_path, FALSE);
- g_return_val_if_fail (out_pending_path, FALSE);
-
- object_path = ostree_repo_get_object_path (self, checksum, objtype);
-
- *out_stored_path = NULL;
- *out_pending_path = NULL;
- if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0)
- {
- *out_stored_path = object_path;
- object_path = NULL;
- }
- else
- {
- g_clear_object (&object_path);
- object_path = get_pending_object_path (self, checksum, objtype);
- if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0)
- {
- *out_pending_path = object_path;
- object_path = NULL;
- }
- }
-
- ret = TRUE;
- /* out: */
- g_clear_object (&object_path);
- return ret;
-}
-
static GFileInfo *
dup_file_info_owned_by_me (GFileInfo *file_info)
{
@@ -824,6 +798,7 @@ dup_file_info_owned_by_me (GFileInfo *file_info)
static gboolean
stage_object_impl (OstreeRepo *self,
OstreeObjectType objtype,
+ gboolean store_if_packed,
GFileInfo *file_info,
GVariant *xattrs,
GInputStream *input,
@@ -941,7 +916,7 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self,
g_checksum_update (ret_checksum, (guint8*)g_variant_get_data (xattrs), g_variant_get_size (xattrs));
}
- if (expected_checksum)
+ if (expected_checksum && ret_checksum)
{
if (strcmp (g_checksum_get_string (ret_checksum), expected_checksum) != 0)
{
@@ -953,6 +928,8 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self,
}
actual_checksum = expected_checksum;
}
+ else if (expected_checksum)
+ actual_checksum = expected_checksum;
else
actual_checksum = g_checksum_get_string (ret_checksum);
@@ -980,6 +957,7 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self,
static gboolean
stage_object_impl (OstreeRepo *self,
OstreeObjectType objtype,
+ gboolean store_if_packed,
GFileInfo *file_info,
GVariant *xattrs,
GInputStream *input,
@@ -995,6 +973,8 @@ stage_object_impl (OstreeRepo *self,
GFile *temp_file = NULL;
GFile *stored_path = NULL;
GFile *pending_path = NULL;
+ char *pack_checksum = NULL;
+ guint64 pack_offset;
const char *actual_checksum;
g_return_val_if_fail (priv->in_transaction, FALSE);
@@ -1006,15 +986,28 @@ stage_object_impl (OstreeRepo *self,
if (expected_checksum)
{
- if (!ostree_repo_find_object (self, objtype, expected_checksum, &stored_path, &pending_path,
- cancellable, error))
- goto out;
+ if (!store_if_packed)
+ {
+ if (!ostree_repo_find_object (self, objtype, expected_checksum,
+ &stored_path, &pending_path,
+ &pack_checksum, &pack_offset,
+ cancellable, error))
+ goto out;
+ }
+ else
+ {
+ if (!ostree_repo_find_object (self, objtype, expected_checksum,
+ &stored_path, &pending_path,
+ NULL, NULL,
+ cancellable, error))
+ goto out;
+ }
}
g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META);
- if (stored_path == NULL && pending_path == NULL)
+ if (stored_path == NULL && pending_path == NULL && pack_checksum == NULL)
{
if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE)
{
@@ -1075,7 +1068,7 @@ stage_object_impl (OstreeRepo *self,
}
else
{
- g_assert (stored_path != NULL);
+ g_assert (stored_path != NULL || pack_checksum != NULL);
/* Nothing to do */
}
@@ -1088,6 +1081,7 @@ stage_object_impl (OstreeRepo *self,
g_clear_object (&temp_info);
g_clear_object (&stored_path);
g_clear_object (&pending_path);
+ g_free (pack_checksum);
ot_clear_checksum (&ret_checksum);
return ret;
}
@@ -1219,7 +1213,7 @@ stage_gvariant_object (OstreeRepo *self,
g_variant_get_size (serialized),
NULL);
- if (!stage_object_impl (self, type,
+ if (!stage_object_impl (self, type, FALSE,
NULL, NULL, mem,
NULL, &ret_checksum, cancellable, error))
goto out;
@@ -1233,33 +1227,6 @@ stage_gvariant_object (OstreeRepo *self,
return ret;
}
-gboolean
-ostree_repo_load_variant (OstreeRepo *self,
- OstreeObjectType expected_type,
- const char *sha256,
- GVariant **out_variant,
- GError **error)
-{
- gboolean ret = FALSE;
- GFile *object_path = NULL;
- GFile *tmpfile = NULL;
- GVariant *ret_variant = NULL;
-
- g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (expected_type), FALSE);
-
- object_path = ostree_repo_get_object_path (self, sha256, expected_type);
- if (!ostree_map_metadata_file (object_path, expected_type, &ret_variant, error))
- goto out;
-
- ret = TRUE;
- ot_transfer_out_value (out_variant, &ret_variant);
- out:
- g_clear_object (&object_path);
- g_clear_object (&tmpfile);
- ot_clear_gvariant (&ret_variant);
- return ret;
-}
-
static gboolean
stage_directory_meta (OstreeRepo *self,
GFileInfo *file_info,
@@ -1309,13 +1276,14 @@ gboolean
ostree_repo_stage_object_trusted (OstreeRepo *self,
OstreeObjectType objtype,
const char *checksum,
+ gboolean store_if_packed,
GFileInfo *file_info,
GVariant *xattrs,
GInputStream *input,
GCancellable *cancellable,
GError **error)
{
- return stage_object_impl (self, objtype,
+ return stage_object_impl (self, objtype, store_if_packed,
file_info, xattrs, input,
checksum, NULL, cancellable, error);
}
@@ -1333,7 +1301,7 @@ ostree_repo_stage_object (OstreeRepo *self,
gboolean ret = FALSE;
GChecksum *actual_checksum = NULL;
- if (!stage_object_impl (self, objtype,
+ if (!stage_object_impl (self, objtype, FALSE,
file_info, xattrs, input,
expected_checksum, &actual_checksum, cancellable, error))
goto out;
@@ -1567,872 +1535,2111 @@ ostree_repo_stage_commit (OstreeRepo *self,
return ret;
}
-static GVariant *
-create_tree_variant_from_hashes (GHashTable *file_checksums,
- GHashTable *dir_contents_checksums,
- GHashTable *dir_metadata_checksums)
+static gboolean
+list_files_in_dir_matching (GFile *dir,
+ const char *prefix,
+ const char *suffix,
+ GPtrArray **out_files,
+ GCancellable *cancellable,
+ GError **error)
{
- GVariantBuilder files_builder;
- GVariantBuilder dirs_builder;
- GHashTableIter hash_iter;
- GSList *sorted_filenames = NULL;
- GSList *iter;
- gpointer key, value;
- GVariant *serialized_tree;
+ gboolean ret = FALSE;
+ GError *temp_error = NULL;
+ GFileEnumerator *enumerator = NULL;
+ GFileInfo *file_info = NULL;
+ GPtrArray *ret_files = NULL;
- g_variant_builder_init (&files_builder, G_VARIANT_TYPE ("a(ss)"));
- g_variant_builder_init (&dirs_builder, G_VARIANT_TYPE ("a(sss)"));
+ g_return_val_if_fail (prefix != NULL || suffix != NULL, FALSE);
- g_hash_table_iter_init (&hash_iter, file_checksums);
- while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ ret_files = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref);
+
+ enumerator = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO,
+ G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+ cancellable,
+ error);
+ if (!enumerator)
+ goto out;
+
+ while ((file_info = g_file_enumerator_next_file (enumerator, cancellable, &temp_error)) != NULL)
{
- const char *name = key;
- sorted_filenames = g_slist_prepend (sorted_filenames, (char*)name);
- }
+ const char *name;
+ guint32 type;
- sorted_filenames = g_slist_sort (sorted_filenames, (GCompareFunc)strcmp);
+ name = g_file_info_get_attribute_byte_string (file_info, "standard::name");
+ type = g_file_info_get_attribute_uint32 (file_info, "standard::type");
- for (iter = sorted_filenames; iter; iter = iter->next)
- {
- const char *name = iter->data;
- const char *value;
+ if (type != G_FILE_TYPE_REGULAR)
+ goto loop_next;
- value = g_hash_table_lookup (file_checksums, name);
- g_variant_builder_add (&files_builder, "(ss)", name, value);
- }
-
- g_slist_free (sorted_filenames);
- sorted_filenames = NULL;
+ if (prefix)
+ {
+ if (!g_str_has_prefix (name, prefix))
+ goto loop_next;
+ }
+ if (suffix)
+ {
+ if (!g_str_has_suffix (name, suffix))
+ goto loop_next;
+ }
- g_hash_table_iter_init (&hash_iter, dir_metadata_checksums);
- while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ g_ptr_array_add (ret_files, g_file_get_child (dir, name));
+
+ loop_next:
+ g_clear_object (&file_info);
+ }
+ if (temp_error != NULL)
{
- const char *name = key;
- sorted_filenames = g_slist_prepend (sorted_filenames, (char*)name);
+ g_propagate_error (error, temp_error);
+ goto out;
}
+ if (!g_file_enumerator_close (enumerator, cancellable, error))
+ goto out;
- sorted_filenames = g_slist_sort (sorted_filenames, (GCompareFunc)strcmp);
+ ret = TRUE;
+ ot_transfer_out_value (out_files, &ret_files);
+ out:
+ if (ret_files)
+ g_ptr_array_unref (ret_files);
+ g_clear_object (&file_info);
+ g_clear_object (&enumerator);
+ return ret;
+}
- for (iter = sorted_filenames; iter; iter = iter->next)
- {
- const char *name = iter->data;
+static gboolean
+map_variant_file_check_header_string (GFile *path,
+ const GVariantType *variant_type,
+ const char *expected_header,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GVariant *ret_variant = NULL;
+ const char *header;
- g_variant_builder_add (&dirs_builder, "(sss)",
- name,
- g_hash_table_lookup (dir_contents_checksums, name),
- g_hash_table_lookup (dir_metadata_checksums, name));
- }
+ if (!ot_util_variant_map (path, variant_type, &ret_variant, error))
+ goto out;
- g_slist_free (sorted_filenames);
- sorted_filenames = NULL;
+ g_variant_get_child (ret_variant, 0, "&s", &header);
- serialized_tree = g_variant_new ("(u a{sv}@a(ss)@a(sss))",
- GUINT32_TO_BE (0),
- create_empty_gvariant_dict (),
- g_variant_builder_end (&files_builder),
- g_variant_builder_end (&dirs_builder));
- g_variant_ref_sink (serialized_tree);
+ if (strcmp (header, expected_header) != 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid variant file '%s', expected header '%s'",
+ ot_gfile_get_path_cached (path),
+ expected_header);
+ goto out;
+ }
- return serialized_tree;
+ ret = TRUE;
+ ot_transfer_out_value (out_variant, &ret_variant);
+ out:
+ ot_clear_gvariant (&ret_variant);
+ return ret;
}
-static GFileInfo *
-create_modified_file_info (GFileInfo *info,
- OstreeRepoCommitModifier *modifier)
+
+static char *
+get_checksum_from_pack_name (const char *name)
{
- GFileInfo *ret;
+ const char *dash;
+ const char *dot;
- if (!modifier)
- return (GFileInfo*)g_object_ref (info);
+ dash = strchr (name, '-');
+ g_assert (dash);
+ dot = strrchr (name, '.');
+ g_assert (dot);
- ret = g_file_info_dup (info);
+ g_assert_cmpint (dot - (dash + 1), ==, 64);
- return ret;
+ return g_strndup (dash + 1, 64);
}
-static OstreeRepoCommitFilterResult
-apply_commit_filter (OstreeRepo *self,
- OstreeRepoCommitModifier *modifier,
- GPtrArray *path,
- GFileInfo *file_info,
- GFileInfo **out_modified_info)
+static gboolean
+list_pack_indexes_from_dir (OstreeRepo *self,
+ GPtrArray **out_indexes,
+ GCancellable *cancellable,
+ GError **error)
{
- GString *path_buf;
+ gboolean ret = FALSE;
+ GPtrArray *index_files = NULL;
+ GPtrArray *ret_indexes = NULL;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
guint i;
- OstreeRepoCommitFilterResult result;
- GFileInfo *modified_info;
-
- if (modifier == NULL || modifier->filter == NULL)
+
+ if (!list_files_in_dir_matching (priv->pack_dir,
+ "ostpack-", ".index",
+ &index_files,
+ cancellable, error))
+ goto out;
+
+ ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
+ for (i = 0; i < index_files->len; i++)
{
- *out_modified_info = g_object_ref (file_info);
- return OSTREE_REPO_COMMIT_FILTER_ALLOW;
+ GFile *index_path = index_files->pdata[i];
+ const char *basename = ot_gfile_get_basename_cached (index_path);
+ g_ptr_array_add (ret_indexes, get_checksum_from_pack_name (basename));
}
- path_buf = g_string_new ("");
+ ret = TRUE;
+ ot_transfer_out_value (out_indexes, &ret_indexes);
+ out:
+ if (index_files)
+ g_ptr_array_unref (index_files);
+ if (ret_indexes)
+ g_ptr_array_unref (ret_indexes);
+ return ret;
+}
- if (path->len == 0)
- g_string_append_c (path_buf, '/');
- else
+static gboolean
+list_pack_checksums_from_superindex_file (GFile *superindex_path,
+ GPtrArray **out_indexes,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GPtrArray *ret_indexes = NULL;
+ GVariant *superindex_variant = NULL;
+ GVariantIter *variant_iter = NULL;
+ const char *magic;
+ GVariant *checksum;
+ GVariant *bloom;
+
+ if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT,
+ &superindex_variant, error))
+ goto out;
+
+ g_variant_get (superindex_variant, "(&s a{sv}a(ayay))",
+ &magic, NULL, &variant_iter);
+
+ if (strcmp (magic, "OSTv0SUPERPACKINDEX") != 0)
{
- for (i = 0; i < path->len; i++)
- {
- const char *elt = path->pdata[i];
-
- g_string_append_c (path_buf, '/');
- g_string_append (path_buf, elt);
- }
+ g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid header in super pack index");
+ goto out;
}
- modified_info = g_file_info_dup (file_info);
- result = modifier->filter (self, path_buf->str, modified_info, modifier->user_data);
- *out_modified_info = modified_info;
+ ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
+
+ while (g_variant_iter_loop (variant_iter, "(@ay ay)",
+ &checksum, &bloom))
+ g_ptr_array_add (ret_indexes, ostree_checksum_from_bytes (checksum));
- g_string_free (path_buf, TRUE);
- return result;
+ ret = TRUE;
+ ot_transfer_out_value (out_indexes, &ret_indexes);
+ out:
+ ot_clear_gvariant (&superindex_variant);
+ if (variant_iter)
+ g_variant_iter_free (variant_iter);
+ if (ret_indexes)
+ g_ptr_array_unref (ret_indexes);
+ return ret;
}
-static gboolean
-stage_directory_to_mtree_internal (OstreeRepo *self,
- GFile *dir,
- OstreeMutableTree *mtree,
- OstreeRepoCommitModifier *modifier,
- GPtrArray *path,
- GCancellable *cancellable,
- GError **error)
+gboolean
+ostree_repo_list_pack_indexes (OstreeRepo *self,
+ GPtrArray **out_indexes,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
- OstreeRepoFile *repo_dir = NULL;
- GError *temp_error = NULL;
- GFileInfo *child_info = NULL;
- OstreeMutableTree *child_mtree = NULL;
- GFileEnumerator *dir_enum = NULL;
- GFileInfo *modified_info = NULL;
- GFile *child = NULL;
- GChecksum *child_file_checksum = NULL;
- GVariant *xattrs = NULL;
- GInputStream *file_input = NULL;
- gboolean repo_dir_was_empty = FALSE;
- OstreeRepoCommitFilterResult filter_result;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ GFile *superindex_path = NULL;
+ GPtrArray *ret_indexes = NULL;
- /* We can only reuse checksums directly if there's no modifier */
- if (OSTREE_IS_REPO_FILE (dir) && modifier == NULL)
- repo_dir = (OstreeRepoFile*)dir;
+ superindex_path = g_file_get_child (priv->pack_dir, "index");
- if (repo_dir)
+ if (g_file_query_exists (superindex_path, cancellable))
{
- if (!ostree_repo_file_ensure_resolved (repo_dir, error))
+ if (!list_pack_checksums_from_superindex_file (superindex_path, &ret_indexes, cancellable, error))
goto out;
-
- ostree_mutable_tree_set_metadata_checksum (mtree, ostree_repo_file_get_checksum (repo_dir));
- repo_dir_was_empty =
- g_hash_table_size (ostree_mutable_tree_get_files (mtree)) == 0
- && g_hash_table_size (ostree_mutable_tree_get_subdirs (mtree)) == 0;
-
- filter_result = OSTREE_REPO_COMMIT_FILTER_ALLOW;
}
else
{
- child_info = g_file_query_info (dir, OSTREE_GIO_FAST_QUERYINFO,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
- cancellable, error);
- if (!child_info)
- goto out;
-
- filter_result = apply_commit_filter (self, modifier, path, child_info, &modified_info);
-
- if (filter_result == OSTREE_REPO_COMMIT_FILTER_ALLOW)
- {
- if (!(modifier && modifier->skip_xattrs))
- {
- xattrs = ostree_get_xattrs_for_file (dir, error);
- if (!xattrs)
- goto out;
- }
-
- if (!stage_directory_meta (self, modified_info, xattrs, &child_file_checksum,
- cancellable, error))
- goto out;
-
- ostree_mutable_tree_set_metadata_checksum (mtree, g_checksum_get_string (child_file_checksum));
-
- g_clear_object (&child_info);
- g_clear_object (&modified_info);
- }
+ ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
}
- if (filter_result == OSTREE_REPO_COMMIT_FILTER_ALLOW)
- {
- dir_enum = g_file_enumerate_children ((GFile*)dir, OSTREE_GIO_FAST_QUERYINFO,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
- cancellable,
- error);
- if (!dir_enum)
- goto out;
-
- while ((child_info = g_file_enumerator_next_file (dir_enum, cancellable, &temp_error)) != NULL)
- {
- const char *name = g_file_info_get_name (child_info);
+ ret = TRUE;
+ ot_transfer_out_value (out_indexes, &ret_indexes);
+ out:
+ g_clear_object (&superindex_path);
+ if (ret_indexes)
+ g_ptr_array_unref (ret_indexes);
+ return ret;
+}
- g_clear_object (&modified_info);
- g_ptr_array_add (path, (char*)name);
- filter_result = apply_commit_filter (self, modifier, path, child_info, &modified_info);
+static gboolean
+create_index_bloom (OstreeRepo *self,
+ const char *pack_checksum,
+ GVariant **out_bloom,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GVariant *ret_bloom;
- if (filter_result == OSTREE_REPO_COMMIT_FILTER_ALLOW)
- {
- g_clear_object (&child);
- child = g_file_get_child (dir, name);
+ /* TODO - define and compute bloom filter */
- if (g_file_info_get_file_type (child_info) == G_FILE_TYPE_DIRECTORY)
- {
- g_clear_object (&child_mtree);
- if (!ostree_mutable_tree_ensure_dir (mtree, name, &child_mtree, error))
- goto out;
+ ret_bloom = g_variant_new_fixed_array (G_VARIANT_TYPE ("y"), NULL, 0, 1);
+ g_variant_ref_sink (ret_bloom);
- if (!stage_directory_to_mtree_internal (self, child, child_mtree,
- modifier, path, cancellable, error))
- goto out;
- }
- else if (repo_dir)
- {
- if (!ostree_mutable_tree_replace_file (mtree, name,
- ostree_repo_file_get_checksum ((OstreeRepoFile*) child),
- error))
- goto out;
- }
- else
- {
- ot_clear_checksum (&child_file_checksum);
- ot_clear_gvariant (&xattrs);
- g_clear_object (&file_input);
+ ret = TRUE;
+ ot_transfer_out_value (out_bloom, &ret_bloom);
+ /* out: */
+ ot_clear_gvariant (&ret_bloom);
+ return ret;
+}
- if (g_file_info_get_file_type (modified_info) == G_FILE_TYPE_REGULAR)
- {
- file_input = (GInputStream*)g_file_read (child, cancellable, error);
- if (!file_input)
- goto out;
- }
+/**
+ * Regenerate the pack superindex file based on the set of pack
+ * indexes currently in the filesystem.
+ */
+gboolean
+ostree_repo_regenerate_pack_index (OstreeRepo *self,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ GFile *index_path = NULL;
+ GPtrArray *pack_indexes = NULL;
+ GVariantBuilder *index_content_builder = NULL;
+ GVariant *index_variant = NULL;
+ guint i;
- if (!(modifier && modifier->skip_xattrs))
- {
- xattrs = ostree_get_xattrs_for_file (child, error);
- if (!xattrs)
- goto out;
- }
+ if (!list_pack_indexes_from_dir (self, &pack_indexes, cancellable, error))
+ goto out;
- if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_RAW_FILE,
- modified_info, xattrs, file_input, NULL,
- &child_file_checksum, cancellable, error))
- goto out;
+ index_path = g_file_get_child (priv->pack_dir, "index");
- if (!ostree_mutable_tree_replace_file (mtree, name,
- g_checksum_get_string (child_file_checksum),
- error))
- goto out;
- }
+ index_content_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(ayay)"));
+
+ for (i = 0; i < pack_indexes->len; i++)
+ {
+ const char *pack_checksum = pack_indexes->pdata[i];
+ GVariant *bloom;
- g_ptr_array_remove_index (path, path->len - 1);
- }
+ if (!create_index_bloom (self, pack_checksum, &bloom, cancellable, error))
+ goto out;
- g_clear_object (&child_info);
- }
- if (temp_error != NULL)
- {
- g_propagate_error (error, temp_error);
- goto out;
- }
+ g_variant_builder_add (index_content_builder,
+ "(@ay ay)",
+ ostree_checksum_to_bytes (pack_checksum),
+ bloom);
+ g_variant_unref (bloom);
}
- if (repo_dir && repo_dir_was_empty)
- ostree_mutable_tree_set_contents_checksum (mtree, ostree_repo_file_tree_get_content_checksum (repo_dir));
+ index_variant = g_variant_new ("(s a{sv}@a(ayay))",
+ "OSTv0SUPERPACKINDEX",
+ g_variant_new_array (G_VARIANT_TYPE ("{sv}"),
+ NULL, 0),
+ g_variant_builder_end (index_content_builder));
+ g_variant_ref_sink (index_variant);
+
+ if (!ot_util_variant_save (index_path, index_variant,
+ cancellable, error))
+ goto out;
ret = TRUE;
out:
- g_clear_object (&dir_enum);
- g_clear_object (&child);
- g_clear_object (&modified_info);
- g_clear_object (&child_info);
- g_clear_object (&file_input);
- g_clear_object (&child_mtree);
- ot_clear_checksum (&child_file_checksum);
- ot_clear_gvariant (&xattrs);
+ ot_clear_gvariant (&index_variant);
+ if (index_content_builder)
+ g_variant_builder_unref (index_content_builder);
+ g_clear_object (&index_path);
+ if (pack_indexes)
+ g_ptr_array_unref (pack_indexes);
return ret;
}
+
+static GFile *
+get_pack_index_name_from_checksum (GFile *parent, const char *pack_checksum)
+{
+ return ot_gfile_get_child_strconcat (parent, "ostpack-", pack_checksum, ".index", NULL);
+}
+
+static GFile *
+get_pack_data_name_from_checksum (GFile *parent, const char *pack_checksum)
+{
+ return ot_gfile_get_child_strconcat (parent, "ostpack-", pack_checksum, ".data", NULL);
+}
+
gboolean
-ostree_repo_stage_directory_to_mtree (OstreeRepo *self,
- GFile *dir,
- OstreeMutableTree *mtree,
- OstreeRepoCommitModifier *modifier,
- GCancellable *cancellable,
- GError **error)
+ostree_repo_add_pack_file (OstreeRepo *self,
+ const char *pack_checksum,
+ GFile *index_path,
+ GFile *data_path,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
- GPtrArray *path = NULL;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ GFile *pack_index_path = NULL;
+ GFile *pack_data_path = NULL;
- path = g_ptr_array_new ();
- if (!stage_directory_to_mtree_internal (self, dir, mtree, modifier, path, cancellable, error))
+ if (!ot_gfile_ensure_directory (priv->pack_dir, FALSE, error))
+ goto out;
+
+ pack_data_path = get_pack_data_name_from_checksum (priv->pack_dir, pack_checksum);
+ if (!ot_gfile_rename (data_path, pack_data_path, cancellable, error))
+ goto out;
+
+ pack_index_path = get_pack_index_name_from_checksum (priv->pack_dir, pack_checksum);
+ if (!ot_gfile_rename (index_path, pack_index_path, cancellable, error))
goto out;
+
+ ret = TRUE;
+ out:
+ g_clear_object (&pack_index_path);
+ g_clear_object (&pack_data_path);
+ return ret;
+}
+
+static gboolean
+ensure_remote_cache_dir (OstreeRepo *self,
+ const char *remote_name,
+ GFile **out_cache_dir,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ GFile *ret_cache_dir = NULL;
+
+ ret_cache_dir = g_file_get_child (priv->remote_cache_dir, remote_name);
+ if (!ot_gfile_ensure_directory (ret_cache_dir, FALSE, error))
+ goto out;
+
ret = TRUE;
+ ot_transfer_out_value (out_cache_dir, &ret_cache_dir);
out:
- if (path)
- g_ptr_array_free (path, TRUE);
+ g_clear_object (&ret_cache_dir);
return ret;
}
+/**
+ * Take a pack superindex file @superindex_path, and clean up any
+ * no-longer-referenced pack files in the lookaside cache for
+ * @remote_name. The updated index file will also be saved into the
+ * cache.
+ *
+ * Upon successful return, @out_cached_indexes will hold checksum
+ * strings for indexes which are already in the cache, and
+ * @out_uncached_indexes will hold strings for those which are not.
+ */
gboolean
-ostree_repo_stage_mtree (OstreeRepo *self,
- OstreeMutableTree *mtree,
- char **out_contents_checksum,
- GCancellable *cancellable,
- GError **error)
+ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
+ const char *remote_name,
+ GFile *superindex_path,
+ GPtrArray **out_cached_indexes,
+ GPtrArray **out_uncached_indexes,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
- GChecksum *ret_contents_checksum_obj = NULL;
- char *ret_contents_checksum = NULL;
- GHashTable *dir_metadata_checksums = NULL;
- GHashTable *dir_contents_checksums = NULL;
- GVariant *serialized_tree = NULL;
+ GVariant *superindex_variant = NULL;
+ GVariantIter *superindex_contents_iter = NULL;
+ GFile *cache_path = NULL;
+ GFile *superindex_cache_path = NULL;
+ GPtrArray *index_files = NULL;
+ GHashTable *new_pack_indexes = NULL;
GHashTableIter hash_iter;
gpointer key, value;
- const char *existing_checksum;
+ GPtrArray *ret_cached_indexes = NULL;
+ GPtrArray *ret_uncached_indexes = NULL;
+ guint i;
+ GVariant *csum_bytes = NULL;
+ GVariant *bloom = NULL;
+ char *pack_checksum = NULL;
- existing_checksum = ostree_mutable_tree_get_contents_checksum (mtree);
- if (existing_checksum)
+ if (!ensure_remote_cache_dir (self, remote_name, &cache_path, cancellable, error))
+ goto out;
+
+ ret_cached_indexes = g_ptr_array_new_with_free_func (g_free);
+ ret_uncached_indexes = g_ptr_array_new_with_free_func (g_free);
+
+ if (!ot_util_variant_map (superindex_path, OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT,
+ &superindex_variant, error))
+ goto out;
+
+ if (!ostree_validate_structureof_pack_superindex (superindex_variant, error))
+ goto out;
+
+ new_pack_indexes = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+ g_variant_get_child (superindex_variant, 2, "a(ayay)",
+ &superindex_contents_iter);
+
+ while (g_variant_iter_loop (superindex_contents_iter,
+ "(@ay ay)", &csum_bytes, &bloom))
{
- ret_contents_checksum = g_strdup (existing_checksum);
+ pack_checksum = ostree_checksum_from_bytes (csum_bytes);
+ g_hash_table_insert (new_pack_indexes, pack_checksum, pack_checksum);
+ pack_checksum = NULL; /* transfer ownership */
}
- else
+
+ if (!list_files_in_dir_matching (cache_path,
+ "ostpack-", ".index",
+ &index_files,
+ cancellable, error))
+ goto out;
+
+ for (i = 0; i < index_files->len; i++)
{
- dir_contents_checksums = g_hash_table_new_full (g_str_hash, g_str_equal,
- (GDestroyNotify)g_free, (GDestroyNotify)g_free);
- dir_metadata_checksums = g_hash_table_new_full (g_str_hash, g_str_equal,
- (GDestroyNotify)g_free, (GDestroyNotify)g_free);
+ GFile *index_file = index_files->pdata[i];
- g_hash_table_iter_init (&hash_iter, ostree_mutable_tree_get_subdirs (mtree));
- while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ g_free (pack_checksum);
+ pack_checksum = get_checksum_from_pack_name (ot_gfile_get_basename_cached (index_file));
+
+ if (!g_hash_table_lookup (new_pack_indexes, pack_checksum))
{
- const char *name = key;
- const char *metadata_checksum;
- OstreeMutableTree *child_dir = value;
- char *child_dir_contents_checksum;
-
- if (!ostree_repo_stage_mtree (self, child_dir, &child_dir_contents_checksum,
- cancellable, error))
+ if (!ot_gfile_unlink (index_file, cancellable, error))
goto out;
-
- g_assert (child_dir_contents_checksum);
- g_hash_table_replace (dir_contents_checksums, g_strdup (name),
- child_dir_contents_checksum); /* Transfer ownership */
- metadata_checksum = ostree_mutable_tree_get_metadata_checksum (child_dir);
- g_assert (metadata_checksum);
- g_hash_table_replace (dir_metadata_checksums, g_strdup (name),
- g_strdup (metadata_checksum));
}
-
- serialized_tree = create_tree_variant_from_hashes (ostree_mutable_tree_get_files (mtree),
- dir_contents_checksums,
- dir_metadata_checksums);
- if (!stage_gvariant_object (self, OSTREE_OBJECT_TYPE_DIR_TREE,
- serialized_tree, &ret_contents_checksum_obj,
- cancellable, error))
- goto out;
- ret_contents_checksum = g_strdup (g_checksum_get_string (ret_contents_checksum_obj));
+ g_ptr_array_add (ret_cached_indexes, pack_checksum);
+ pack_checksum = NULL; /* transfer ownership */
}
+ g_hash_table_iter_init (&hash_iter, new_pack_indexes);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ const char *cur_pack_checksum = key;
+ gboolean found = FALSE;
+
+ for (i = 0; i < ret_cached_indexes->len; i++)
+ {
+ if (strcmp (cur_pack_checksum, ret_cached_indexes->pdata[i]) == 0)
+ {
+ found = TRUE;
+ break;
+ }
+ }
+
+ if (!found)
+ g_ptr_array_add (ret_uncached_indexes, g_strdup (cur_pack_checksum));
+ }
+
+ superindex_cache_path = g_file_get_child (cache_path, "index");
+ if (!ot_util_variant_save (superindex_cache_path, superindex_variant, cancellable, error))
+ goto out;
+
ret = TRUE;
- ot_transfer_out_value(out_contents_checksum, &ret_contents_checksum);
+ ot_transfer_out_value (out_cached_indexes, &ret_cached_indexes);
+ ot_transfer_out_value (out_uncached_indexes, &ret_uncached_indexes);
out:
- if (dir_contents_checksums)
- g_hash_table_destroy (dir_contents_checksums);
- if (dir_metadata_checksums)
- g_hash_table_destroy (dir_metadata_checksums);
- g_free (ret_contents_checksum);
- ot_clear_checksum (&ret_contents_checksum_obj);
- ot_clear_gvariant (&serialized_tree);
+ g_free (pack_checksum);
+ g_clear_object (&cache_path);
+ g_clear_object (&superindex_cache_path);
+ if (superindex_contents_iter)
+ g_variant_iter_free (superindex_contents_iter);
+ ot_clear_ptrarray (&ret_cached_indexes);
+ ot_clear_ptrarray (&ret_uncached_indexes);
+ ot_clear_hashtable (&new_pack_indexes);
+ ot_clear_ptrarray (&index_files);
return ret;
}
-#ifdef HAVE_LIBARCHIVE
+/**
+ * Load the index for pack @pack_checksum from cache directory for
+ * @remote_name.
+ */
+gboolean
+ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GVariant *ret_variant = NULL;
+ GFile *cache_dir = NULL;
+ GFile *cached_pack_path = NULL;
-static void
-propagate_libarchive_error (GError **error,
- struct archive *a)
+ if (!ensure_remote_cache_dir (self, remote_name, &cache_dir,
+ cancellable, error))
+ goto out;
+
+ cached_pack_path = get_pack_index_name_from_checksum (cache_dir, pack_checksum);
+ if (!ot_util_variant_map (cached_pack_path, OSTREE_PACK_INDEX_VARIANT_FORMAT,
+ &ret_variant, error))
+ goto out;
+
+ ret = TRUE;
+ ot_transfer_out_value (out_variant, &ret_variant);
+ out:
+ g_clear_object (&cache_dir);
+ g_clear_object (&cached_pack_path);
+ ot_clear_gvariant (&ret_variant);
+ return ret;
+}
+
+/**
+ * The variable @cached_path should refer to a file containing a pack
+ * index. It will be validated and added to the cache directory for
+ * @remote_name.
+ */
+gboolean
+ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GFile *cached_path,
+ GCancellable *cancellable,
+ GError **error)
{
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "%s", archive_error_string (a));
+ gboolean ret = FALSE;
+ GFile *cachedir = NULL;
+ GFile *target_path = NULL;
+ GVariant *input_index_variant = NULL;
+ GVariant *output_index_variant = NULL;
+
+ if (!map_variant_file_check_header_string (cached_path,
+ OSTREE_PACK_INDEX_VARIANT_FORMAT,
+ "OSTv0PACKINDEX",
+ &input_index_variant,
+ cancellable, error))
+ goto out;
+
+ if (!ostree_validate_structureof_pack_index (input_index_variant, error))
+ goto out;
+
+ output_index_variant = g_variant_get_normal_form (input_index_variant);
+
+ if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error))
+ goto out;
+
+ target_path = get_pack_index_name_from_checksum (cachedir, pack_checksum);
+ if (!ot_util_variant_save (target_path, output_index_variant, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ g_clear_object (&cachedir);
+ g_clear_object (&target_path);
+ ot_clear_gvariant (&input_index_variant);
+ ot_clear_gvariant (&output_index_variant);
+ return ret;
}
-static GFileInfo *
-file_info_from_archive_entry_and_modifier (struct archive_entry *entry,
- OstreeRepoCommitModifier *modifier)
+/**
+ * Check for availability of the pack index pointing to @pack_checksum
+ * in the lookaside cache for @remote_name. If not found, then the
+ * output parameter @out_cached_path will be %NULL.
+ */
+gboolean
+ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GFile **out_cached_path,
+ GCancellable *cancellable,
+ GError **error)
{
- GFileInfo *info = g_file_info_new ();
- GFileInfo *modified_info = NULL;
- const struct stat *st;
- guint32 file_type;
+ gboolean ret = FALSE;
+ GFile *cache_dir = NULL;
+ GFile *cached_pack_path = NULL;
+ GFile *ret_cached_path = NULL;
- st = archive_entry_stat (entry);
+ if (!ensure_remote_cache_dir (self, remote_name, &cache_dir,
+ cancellable, error))
+ goto out;
- file_type = ot_gfile_type_for_mode (st->st_mode);
- g_file_info_set_attribute_boolean (info, "standard::is-symlink", S_ISLNK (st->st_mode));
- g_file_info_set_attribute_uint32 (info, "standard::type", file_type);
- g_file_info_set_attribute_uint32 (info, "unix::uid", st->st_uid);
- g_file_info_set_attribute_uint32 (info, "unix::gid", st->st_gid);
- g_file_info_set_attribute_uint32 (info, "unix::mode", st->st_mode);
+ cached_pack_path = get_pack_data_name_from_checksum (cache_dir, pack_checksum);
+ if (g_file_query_exists (cached_pack_path, cancellable))
+ {
+ ret_cached_path = cached_pack_path;
+ cached_pack_path = NULL;
+ }
- if (file_type == G_FILE_TYPE_REGULAR)
+ ret = TRUE;
+ ot_transfer_out_value (out_cached_path, &ret_cached_path);
+ out:
+ g_clear_object (&cache_dir);
+ g_clear_object (&cached_pack_path);
+ return ret;
+}
+
+/**
+ * Add file @cached_path into the cache for given @remote_name.
+ *
+ * <note>
+ * This unlinks @cached_path.
+ * </note>
+ */
+gboolean
+ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GFile *cached_path,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *cachedir = NULL;
+ GFile *target_path = NULL;
+
+ if (!ensure_remote_cache_dir (self, remote_name, &cachedir, cancellable, error))
+ goto out;
+
+ target_path = get_pack_data_name_from_checksum (cachedir, pack_checksum);
+
+ if (!ot_gfile_rename (cached_path, target_path, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ g_clear_object (&cachedir);
+ g_clear_object (&target_path);
+ return ret;
+}
+
+static GVariant *
+create_tree_variant_from_hashes (GHashTable *file_checksums,
+ GHashTable *dir_contents_checksums,
+ GHashTable *dir_metadata_checksums)
+{
+ GVariantBuilder files_builder;
+ GVariantBuilder dirs_builder;
+ GHashTableIter hash_iter;
+ GSList *sorted_filenames = NULL;
+ GSList *iter;
+ gpointer key, value;
+ GVariant *serialized_tree;
+
+ g_variant_builder_init (&files_builder, G_VARIANT_TYPE ("a(ss)"));
+ g_variant_builder_init (&dirs_builder, G_VARIANT_TYPE ("a(sss)"));
+
+ g_hash_table_iter_init (&hash_iter, file_checksums);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
- g_file_info_set_attribute_uint64 (info, "standard::size", st->st_size);
+ const char *name = key;
+ sorted_filenames = g_slist_prepend (sorted_filenames, (char*)name);
}
- else if (file_type == G_FILE_TYPE_SYMBOLIC_LINK)
+
+ sorted_filenames = g_slist_sort (sorted_filenames, (GCompareFunc)strcmp);
+
+ for (iter = sorted_filenames; iter; iter = iter->next)
{
- g_file_info_set_attribute_byte_string (info, "standard::symlink-target", archive_entry_symlink (entry));
+ const char *name = iter->data;
+ const char *value;
+
+ value = g_hash_table_lookup (file_checksums, name);
+ g_variant_builder_add (&files_builder, "(ss)", name, value);
}
- else if (file_type == G_FILE_TYPE_SPECIAL)
+
+ g_slist_free (sorted_filenames);
+ sorted_filenames = NULL;
+
+ g_hash_table_iter_init (&hash_iter, dir_metadata_checksums);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
- g_file_info_set_attribute_uint32 (info, "unix::rdev", st->st_rdev);
+ const char *name = key;
+ sorted_filenames = g_slist_prepend (sorted_filenames, (char*)name);
}
- modified_info = create_modified_file_info (info, modifier);
+ sorted_filenames = g_slist_sort (sorted_filenames, (GCompareFunc)strcmp);
- g_object_unref (info);
+ for (iter = sorted_filenames; iter; iter = iter->next)
+ {
+ const char *name = iter->data;
+
+ g_variant_builder_add (&dirs_builder, "(sss)",
+ name,
+ g_hash_table_lookup (dir_contents_checksums, name),
+ g_hash_table_lookup (dir_metadata_checksums, name));
+ }
+
+ g_slist_free (sorted_filenames);
+ sorted_filenames = NULL;
+
+ serialized_tree = g_variant_new ("(u a{sv}@a(ss)@a(sss))",
+ GUINT32_TO_BE (0),
+ create_empty_gvariant_dict (),
+ g_variant_builder_end (&files_builder),
+ g_variant_builder_end (&dirs_builder));
+ g_variant_ref_sink (serialized_tree);
+
+ return serialized_tree;
+}
+
+static GFileInfo *
+create_modified_file_info (GFileInfo *info,
+ OstreeRepoCommitModifier *modifier)
+{
+ GFileInfo *ret;
+
+ if (!modifier)
+ return (GFileInfo*)g_object_ref (info);
+
+ ret = g_file_info_dup (info);
- return modified_info;
+ return ret;
+}
+
+static OstreeRepoCommitFilterResult
+apply_commit_filter (OstreeRepo *self,
+ OstreeRepoCommitModifier *modifier,
+ GPtrArray *path,
+ GFileInfo *file_info,
+ GFileInfo **out_modified_info)
+{
+ GString *path_buf;
+ guint i;
+ OstreeRepoCommitFilterResult result;
+ GFileInfo *modified_info;
+
+ if (modifier == NULL || modifier->filter == NULL)
+ {
+ *out_modified_info = g_object_ref (file_info);
+ return OSTREE_REPO_COMMIT_FILTER_ALLOW;
+ }
+
+ path_buf = g_string_new ("");
+
+ if (path->len == 0)
+ g_string_append_c (path_buf, '/');
+ else
+ {
+ for (i = 0; i < path->len; i++)
+ {
+ const char *elt = path->pdata[i];
+
+ g_string_append_c (path_buf, '/');
+ g_string_append (path_buf, elt);
+ }
+ }
+
+ modified_info = g_file_info_dup (file_info);
+ result = modifier->filter (self, path_buf->str, modified_info, modifier->user_data);
+ *out_modified_info = modified_info;
+
+ g_string_free (path_buf, TRUE);
+ return result;
}
static gboolean
-import_libarchive_entry_file (OstreeRepo *self,
- struct archive *a,
- struct archive_entry *entry,
- GFileInfo *file_info,
- GChecksum **out_checksum,
- GCancellable *cancellable,
- GError **error)
+stage_directory_to_mtree_internal (OstreeRepo *self,
+ GFile *dir,
+ OstreeMutableTree *mtree,
+ OstreeRepoCommitModifier *modifier,
+ GPtrArray *path,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
- GInputStream *archive_stream = NULL;
- GChecksum *ret_checksum = NULL;
-
- if (g_cancellable_set_error_if_cancelled (cancellable, error))
- return FALSE;
+ OstreeRepoFile *repo_dir = NULL;
+ GError *temp_error = NULL;
+ GFileInfo *child_info = NULL;
+ OstreeMutableTree *child_mtree = NULL;
+ GFileEnumerator *dir_enum = NULL;
+ GFileInfo *modified_info = NULL;
+ GFile *child = NULL;
+ GChecksum *child_file_checksum = NULL;
+ GVariant *xattrs = NULL;
+ GInputStream *file_input = NULL;
+ gboolean repo_dir_was_empty = FALSE;
+ OstreeRepoCommitFilterResult filter_result;
- if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
- archive_stream = ostree_libarchive_input_stream_new (a);
-
- if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_RAW_FILE,
- file_info, NULL, archive_stream,
- NULL, &ret_checksum,
- cancellable, error))
- goto out;
+ /* We can only reuse checksums directly if there's no modifier */
+ if (OSTREE_IS_REPO_FILE (dir) && modifier == NULL)
+ repo_dir = (OstreeRepoFile*)dir;
+
+ if (repo_dir)
+ {
+ if (!ostree_repo_file_ensure_resolved (repo_dir, error))
+ goto out;
+
+ ostree_mutable_tree_set_metadata_checksum (mtree, ostree_repo_file_get_checksum (repo_dir));
+ repo_dir_was_empty =
+ g_hash_table_size (ostree_mutable_tree_get_files (mtree)) == 0
+ && g_hash_table_size (ostree_mutable_tree_get_subdirs (mtree)) == 0;
+
+ filter_result = OSTREE_REPO_COMMIT_FILTER_ALLOW;
+ }
+ else
+ {
+ child_info = g_file_query_info (dir, OSTREE_GIO_FAST_QUERYINFO,
+ G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+ cancellable, error);
+ if (!child_info)
+ goto out;
+
+ filter_result = apply_commit_filter (self, modifier, path, child_info, &modified_info);
+
+ if (filter_result == OSTREE_REPO_COMMIT_FILTER_ALLOW)
+ {
+ if (!(modifier && modifier->skip_xattrs))
+ {
+ xattrs = ostree_get_xattrs_for_file (dir, error);
+ if (!xattrs)
+ goto out;
+ }
+
+ if (!stage_directory_meta (self, modified_info, xattrs, &child_file_checksum,
+ cancellable, error))
+ goto out;
+
+ ostree_mutable_tree_set_metadata_checksum (mtree, g_checksum_get_string (child_file_checksum));
+
+ g_clear_object (&child_info);
+ g_clear_object (&modified_info);
+ }
+ }
+
+ if (filter_result == OSTREE_REPO_COMMIT_FILTER_ALLOW)
+ {
+ dir_enum = g_file_enumerate_children ((GFile*)dir, OSTREE_GIO_FAST_QUERYINFO,
+ G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+ cancellable,
+ error);
+ if (!dir_enum)
+ goto out;
+
+ while ((child_info = g_file_enumerator_next_file (dir_enum, cancellable, &temp_error)) != NULL)
+ {
+ const char *name = g_file_info_get_name (child_info);
+
+ g_clear_object (&modified_info);
+ g_ptr_array_add (path, (char*)name);
+ filter_result = apply_commit_filter (self, modifier, path, child_info, &modified_info);
+
+ if (filter_result == OSTREE_REPO_COMMIT_FILTER_ALLOW)
+ {
+ g_clear_object (&child);
+ child = g_file_get_child (dir, name);
+
+ if (g_file_info_get_file_type (child_info) == G_FILE_TYPE_DIRECTORY)
+ {
+ g_clear_object (&child_mtree);
+ if (!ostree_mutable_tree_ensure_dir (mtree, name, &child_mtree, error))
+ goto out;
+
+ if (!stage_directory_to_mtree_internal (self, child, child_mtree,
+ modifier, path, cancellable, error))
+ goto out;
+ }
+ else if (repo_dir)
+ {
+ if (!ostree_mutable_tree_replace_file (mtree, name,
+ ostree_repo_file_get_checksum ((OstreeRepoFile*) child),
+ error))
+ goto out;
+ }
+ else
+ {
+ ot_clear_checksum (&child_file_checksum);
+ ot_clear_gvariant (&xattrs);
+ g_clear_object (&file_input);
+
+ if (g_file_info_get_file_type (modified_info) == G_FILE_TYPE_REGULAR)
+ {
+ file_input = (GInputStream*)g_file_read (child, cancellable, error);
+ if (!file_input)
+ goto out;
+ }
+
+ if (!(modifier && modifier->skip_xattrs))
+ {
+ xattrs = ostree_get_xattrs_for_file (child, error);
+ if (!xattrs)
+ goto out;
+ }
+
+ if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_RAW_FILE, FALSE,
+ modified_info, xattrs, file_input, NULL,
+ &child_file_checksum, cancellable, error))
+ goto out;
+
+ if (!ostree_mutable_tree_replace_file (mtree, name,
+ g_checksum_get_string (child_file_checksum),
+ error))
+ goto out;
+ }
+
+ g_ptr_array_remove_index (path, path->len - 1);
+ }
+
+ g_clear_object (&child_info);
+ }
+ if (temp_error != NULL)
+ {
+ g_propagate_error (error, temp_error);
+ goto out;
+ }
+ }
+
+ if (repo_dir && repo_dir_was_empty)
+ ostree_mutable_tree_set_contents_checksum (mtree, ostree_repo_file_tree_get_content_checksum (repo_dir));
+
+ ret = TRUE;
+ out:
+ g_clear_object (&dir_enum);
+ g_clear_object (&child);
+ g_clear_object (&modified_info);
+ g_clear_object (&child_info);
+ g_clear_object (&file_input);
+ g_clear_object (&child_mtree);
+ ot_clear_checksum (&child_file_checksum);
+ ot_clear_gvariant (&xattrs);
+ return ret;
+}
+
+gboolean
+ostree_repo_stage_directory_to_mtree (OstreeRepo *self,
+ GFile *dir,
+ OstreeMutableTree *mtree,
+ OstreeRepoCommitModifier *modifier,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GPtrArray *path = NULL;
+
+ path = g_ptr_array_new ();
+ if (!stage_directory_to_mtree_internal (self, dir, mtree, modifier, path, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ if (path)
+ g_ptr_array_free (path, TRUE);
+ return ret;
+}
+
+gboolean
+ostree_repo_stage_mtree (OstreeRepo *self,
+ OstreeMutableTree *mtree,
+ char **out_contents_checksum,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GChecksum *ret_contents_checksum_obj = NULL;
+ char *ret_contents_checksum = NULL;
+ GHashTable *dir_metadata_checksums = NULL;
+ GHashTable *dir_contents_checksums = NULL;
+ GVariant *serialized_tree = NULL;
+ GHashTableIter hash_iter;
+ gpointer key, value;
+ const char *existing_checksum;
+
+ existing_checksum = ostree_mutable_tree_get_contents_checksum (mtree);
+ if (existing_checksum)
+ {
+ ret_contents_checksum = g_strdup (existing_checksum);
+ }
+ else
+ {
+ dir_contents_checksums = g_hash_table_new_full (g_str_hash, g_str_equal,
+ (GDestroyNotify)g_free, (GDestroyNotify)g_free);
+ dir_metadata_checksums = g_hash_table_new_full (g_str_hash, g_str_equal,
+ (GDestroyNotify)g_free, (GDestroyNotify)g_free);
+
+ g_hash_table_iter_init (&hash_iter, ostree_mutable_tree_get_subdirs (mtree));
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ const char *name = key;
+ const char *metadata_checksum;
+ OstreeMutableTree *child_dir = value;
+ char *child_dir_contents_checksum;
+
+ if (!ostree_repo_stage_mtree (self, child_dir, &child_dir_contents_checksum,
+ cancellable, error))
+ goto out;
+
+ g_assert (child_dir_contents_checksum);
+ g_hash_table_replace (dir_contents_checksums, g_strdup (name),
+ child_dir_contents_checksum); /* Transfer ownership */
+ metadata_checksum = ostree_mutable_tree_get_metadata_checksum (child_dir);
+ g_assert (metadata_checksum);
+ g_hash_table_replace (dir_metadata_checksums, g_strdup (name),
+ g_strdup (metadata_checksum));
+ }
+
+ serialized_tree = create_tree_variant_from_hashes (ostree_mutable_tree_get_files (mtree),
+ dir_contents_checksums,
+ dir_metadata_checksums);
+
+ if (!stage_gvariant_object (self, OSTREE_OBJECT_TYPE_DIR_TREE,
+ serialized_tree, &ret_contents_checksum_obj,
+ cancellable, error))
+ goto out;
+ ret_contents_checksum = g_strdup (g_checksum_get_string (ret_contents_checksum_obj));
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value(out_contents_checksum, &ret_contents_checksum);
+ out:
+ if (dir_contents_checksums)
+ g_hash_table_destroy (dir_contents_checksums);
+ if (dir_metadata_checksums)
+ g_hash_table_destroy (dir_metadata_checksums);
+ g_free (ret_contents_checksum);
+ ot_clear_checksum (&ret_contents_checksum_obj);
+ ot_clear_gvariant (&serialized_tree);
+ return ret;
+}
+
+#ifdef HAVE_LIBARCHIVE
+
+static void
+propagate_libarchive_error (GError **error,
+ struct archive *a)
+{
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "%s", archive_error_string (a));
+}
+
+static GFileInfo *
+file_info_from_archive_entry_and_modifier (struct archive_entry *entry,
+ OstreeRepoCommitModifier *modifier)
+{
+ GFileInfo *info = g_file_info_new ();
+ GFileInfo *modified_info = NULL;
+ const struct stat *st;
+ guint32 file_type;
+
+ st = archive_entry_stat (entry);
+
+ file_type = ot_gfile_type_for_mode (st->st_mode);
+ g_file_info_set_attribute_boolean (info, "standard::is-symlink", S_ISLNK (st->st_mode));
+ g_file_info_set_attribute_uint32 (info, "standard::type", file_type);
+ g_file_info_set_attribute_uint32 (info, "unix::uid", st->st_uid);
+ g_file_info_set_attribute_uint32 (info, "unix::gid", st->st_gid);
+ g_file_info_set_attribute_uint32 (info, "unix::mode", st->st_mode);
+
+ if (file_type == G_FILE_TYPE_REGULAR)
+ {
+ g_file_info_set_attribute_uint64 (info, "standard::size", st->st_size);
+ }
+ else if (file_type == G_FILE_TYPE_SYMBOLIC_LINK)
+ {
+ g_file_info_set_attribute_byte_string (info, "standard::symlink-target", archive_entry_symlink (entry));
+ }
+ else if (file_type == G_FILE_TYPE_SPECIAL)
+ {
+ g_file_info_set_attribute_uint32 (info, "unix::rdev", st->st_rdev);
+ }
+
+ modified_info = create_modified_file_info (info, modifier);
+
+ g_object_unref (info);
+
+ return modified_info;
+}
+
+static gboolean
+import_libarchive_entry_file (OstreeRepo *self,
+ struct archive *a,
+ struct archive_entry *entry,
+ GFileInfo *file_info,
+ GChecksum **out_checksum,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GInputStream *archive_stream = NULL;
+ GChecksum *ret_checksum = NULL;
+
+ if (g_cancellable_set_error_if_cancelled (cancellable, error))
+ return FALSE;
+
+ if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
+ archive_stream = ostree_libarchive_input_stream_new (a);
+
+ if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_RAW_FILE, FALSE,
+ file_info, NULL, archive_stream,
+ NULL, &ret_checksum,
+ cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ ot_transfer_out_value(out_checksum, &ret_checksum);
+ out:
+ g_clear_object (&archive_stream);
+ ot_clear_checksum (&ret_checksum);
+ return ret;
+}
+
+static gboolean
+stage_libarchive_entry_to_mtree (OstreeRepo *self,
+ OstreeMutableTree *root,
+ struct archive *a,
+ struct archive_entry *entry,
+ OstreeRepoCommitModifier *modifier,
+ const char *tmp_dir_checksum,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ const char *pathname;
+ const char *hardlink;
+ const char *basename;
+ GFileInfo *file_info = NULL;
+ GChecksum *tmp_checksum = NULL;
+ GPtrArray *split_path = NULL;
+ GPtrArray *hardlink_split_path = NULL;
+ OstreeMutableTree *subdir = NULL;
+ OstreeMutableTree *parent = NULL;
+ OstreeMutableTree *hardlink_source_parent = NULL;
+ char *hardlink_source_checksum = NULL;
+ OstreeMutableTree *hardlink_source_subdir = NULL;
+
+ pathname = archive_entry_pathname (entry);
+
+ if (!ot_util_path_split_validate (pathname, &split_path, error))
+ goto out;
+
+ if (split_path->len == 0)
+ {
+ parent = NULL;
+ basename = NULL;
+ }
+ else
+ {
+ if (tmp_dir_checksum)
+ {
+ if (!ostree_mutable_tree_ensure_parent_dirs (root, split_path,
+ tmp_dir_checksum,
+ &parent,
+ error))
+ goto out;
+ }
+ else
+ {
+ if (!ostree_mutable_tree_walk (root, split_path, 0, &parent, error))
+ goto out;
+ }
+ basename = (char*)split_path->pdata[split_path->len-1];
+ }
+
+ hardlink = archive_entry_hardlink (entry);
+ if (hardlink)
+ {
+ const char *hardlink_basename;
+
+ g_assert (parent != NULL);
+
+ if (!ot_util_path_split_validate (hardlink, &hardlink_split_path, error))
+ goto out;
+ if (hardlink_split_path->len == 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid hardlink path %s", hardlink);
+ goto out;
+ }
+
+ hardlink_basename = hardlink_split_path->pdata[hardlink_split_path->len - 1];
+
+ if (!ostree_mutable_tree_walk (root, hardlink_split_path, 0, &hardlink_source_parent, error))
+ goto out;
+
+ if (!ostree_mutable_tree_lookup (hardlink_source_parent, hardlink_basename,
+ &hardlink_source_checksum,
+ &hardlink_source_subdir,
+ error))
+ {
+ g_prefix_error (error, "While resolving hardlink target: ");
+ goto out;
+ }
+
+ if (hardlink_source_subdir)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Hardlink %s refers to directory %s",
+ pathname, hardlink);
+ goto out;
+ }
+ g_assert (hardlink_source_checksum);
+
+ if (!ostree_mutable_tree_replace_file (parent,
+ basename,
+ hardlink_source_checksum,
+ error))
+ goto out;
+ }
+ else
+ {
+ file_info = file_info_from_archive_entry_and_modifier (entry, modifier);
+
+ if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_UNKNOWN)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Unsupported file for import: %s", pathname);
+ goto out;
+ }
+
+ if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY)
+ {
+
+ if (!stage_directory_meta (self, file_info, NULL, &tmp_checksum, cancellable, error))
+ goto out;
+
+ if (parent == NULL)
+ {
+ subdir = g_object_ref (root);
+ }
+ else
+ {
+ if (!ostree_mutable_tree_ensure_dir (parent, basename, &subdir, error))
+ goto out;
+ }
+
+ ostree_mutable_tree_set_metadata_checksum (subdir, g_checksum_get_string (tmp_checksum));
+ }
+ else
+ {
+ if (parent == NULL)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Can't import file as root");
+ goto out;
+ }
+
+ if (!import_libarchive_entry_file (self, a, entry, file_info, &tmp_checksum, cancellable, error))
+ goto out;
+
+ if (!ostree_mutable_tree_replace_file (parent, basename,
+ g_checksum_get_string (tmp_checksum),
+ error))
+ goto out;
+ }
+ }
+
+ ret = TRUE;
+ out:
+ g_clear_object (&file_info);
+ ot_clear_checksum (&tmp_checksum);
+ g_clear_object (&parent);
+ g_clear_object (&subdir);
+ g_clear_object (&hardlink_source_parent);
+ g_free (hardlink_source_checksum);
+ g_clear_object (&hardlink_source_subdir);
+ if (hardlink_split_path)
+ g_ptr_array_unref (hardlink_split_path);
+ if (split_path)
+ g_ptr_array_unref (split_path);
+ return ret;
+}
+#endif
+
+gboolean
+ostree_repo_stage_archive_to_mtree (OstreeRepo *self,
+ GFile *archive_f,
+ OstreeMutableTree *root,
+ OstreeRepoCommitModifier *modifier,
+ gboolean autocreate_parents,
+ GCancellable *cancellable,
+ GError **error)
+{
+#ifdef HAVE_LIBARCHIVE
+ gboolean ret = FALSE;
+ struct archive *a = NULL;
+ struct archive_entry *entry;
+ int r;
+ GFileInfo *tmp_dir_info = NULL;
+ GChecksum *tmp_dir_checksum = NULL;
+
+ a = archive_read_new ();
+ archive_read_support_compression_all (a);
+ archive_read_support_format_all (a);
+ if (archive_read_open_filename (a, ot_gfile_get_path_cached (archive_f), 8192) != ARCHIVE_OK)
+ {
+ propagate_libarchive_error (error, a);
+ goto out;
+ }
+
+ while (TRUE)
+ {
+ r = archive_read_next_header (a, &entry);
+ if (r == ARCHIVE_EOF)
+ break;
+ else if (r != ARCHIVE_OK)
+ {
+ propagate_libarchive_error (error, a);
+ goto out;
+ }
+
+ if (autocreate_parents && !tmp_dir_checksum)
+ {
+ tmp_dir_info = g_file_info_new ();
+
+ g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", archive_entry_uid (entry));
+ g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", archive_entry_gid (entry));
+ g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR);
+
+ if (!stage_directory_meta (self, tmp_dir_info, NULL, &tmp_dir_checksum, cancellable, error))
+ goto out;
+ }
+
+ if (!stage_libarchive_entry_to_mtree (self, root, a,
+ entry, modifier,
+ autocreate_parents ? g_checksum_get_string (tmp_dir_checksum) : NULL,
+ cancellable, error))
+ goto out;
+ }
+ if (archive_read_close (a) != ARCHIVE_OK)
+ {
+ propagate_libarchive_error (error, a);
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ g_clear_object (&tmp_dir_info);
+ ot_clear_checksum (&tmp_dir_checksum);
+ if (a)
+ (void)archive_read_close (a);
+ return ret;
+#else
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
+ "This version of ostree is not compiled with libarchive support");
+ return FALSE;
+#endif
+}
+
+OstreeRepoCommitModifier *
+ostree_repo_commit_modifier_new (void)
+{
+ OstreeRepoCommitModifier *modifier = g_new0 (OstreeRepoCommitModifier, 1);
+
+ modifier->refcount = 1;
+
+ return modifier;
+}
+
+void
+ostree_repo_commit_modifier_unref (OstreeRepoCommitModifier *modifier)
+{
+ if (!modifier)
+ return;
+ if (!g_atomic_int_dec_and_test (&modifier->refcount))
+ return;
+
+ g_free (modifier);
+ return;
+}
+
+static gboolean
+list_loose_object_dir (OstreeRepo *self,
+ GFile *dir,
+ GHashTable *inout_objects,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GError *temp_error = NULL;
+ GFileEnumerator *enumerator = NULL;
+ GFileInfo *file_info = NULL;
+ const char *dirname = NULL;
+ char *dot = NULL;
+ GString *checksum = NULL;
+
+ dirname = ot_gfile_get_basename_cached (dir);
+
+ /* We're only querying name */
+ enumerator = g_file_enumerate_children (dir, "standard::name,standard::type",
+ G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+ cancellable,
+ error);
+ if (!enumerator)
+ goto out;
+
+ while ((file_info = g_file_enumerator_next_file (enumerator, cancellable, &temp_error)) != NULL)
+ {
+ const char *name;
+ guint32 type;
+ OstreeObjectType objtype;
+
+ name = g_file_info_get_attribute_byte_string (file_info, "standard::name");
+ type = g_file_info_get_attribute_uint32 (file_info, "standard::type");
+
+ if (type == G_FILE_TYPE_DIRECTORY)
+ goto loop_next;
+
+ if (g_str_has_suffix (name, ".file"))
+ objtype = OSTREE_OBJECT_TYPE_RAW_FILE;
+ else if (g_str_has_suffix (name, ".archive-meta"))
+ objtype = OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META;
+ else if (g_str_has_suffix (name, ".archive-content"))
+ objtype = OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT;
+ else if (g_str_has_suffix (name, ".dirtree"))
+ objtype = OSTREE_OBJECT_TYPE_DIR_TREE;
+ else if (g_str_has_suffix (name, ".dirmeta"))
+ objtype = OSTREE_OBJECT_TYPE_DIR_META;
+ else if (g_str_has_suffix (name, ".commit"))
+ objtype = OSTREE_OBJECT_TYPE_COMMIT;
+ else
+ goto loop_next;
+
+ dot = strrchr (name, '.');
+ g_assert (dot);
+
+ if ((dot - name) == 62)
+ {
+ GVariant *key, *value;
+
+ if (checksum)
+ g_string_free (checksum, TRUE);
+ checksum = g_string_new (dirname);
+ g_string_append_len (checksum, name, 62);
+
+ key = ostree_object_name_serialize (checksum->str, objtype);
+ value = g_variant_new ("(b as)",
+ TRUE, g_variant_new_strv (NULL, 0));
+ /* transfer ownership */
+ g_hash_table_replace (inout_objects, g_variant_ref_sink (key),
+ g_variant_ref_sink (value));
+ }
+ loop_next:
+ g_clear_object (&file_info);
+ }
+ if (temp_error != NULL)
+ {
+ g_propagate_error (error, temp_error);
+ goto out;
+ }
+ if (!g_file_enumerator_close (enumerator, NULL, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ g_clear_object (&file_info);
+ g_clear_object (&enumerator);
+ if (checksum)
+ g_string_free (checksum, TRUE);
+ return ret;
+}
+
+static gboolean
+list_loose_objects (OstreeRepo *self,
+ GHashTable *inout_objects,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ GFileEnumerator *enumerator = NULL;
+ GFileInfo *file_info = NULL;
+ GError *temp_error = NULL;
+ GFile *objdir = NULL;
+
+ enumerator = g_file_enumerate_children (priv->objects_dir, OSTREE_GIO_FAST_QUERYINFO,
+ G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+ cancellable,
+ error);
+ if (!enumerator)
+ goto out;
+
+ while ((file_info = g_file_enumerator_next_file (enumerator, cancellable, &temp_error)) != NULL)
+ {
+ const char *name;
+ guint32 type;
+
+ name = g_file_info_get_attribute_byte_string (file_info, "standard::name");
+ type = g_file_info_get_attribute_uint32 (file_info, "standard::type");
+
+ if (strlen (name) == 2 && type == G_FILE_TYPE_DIRECTORY)
+ {
+ g_clear_object (&objdir);
+ objdir = g_file_get_child (priv->objects_dir, name);
+ if (!list_loose_object_dir (self, objdir, inout_objects, cancellable, error))
+ goto out;
+ }
+ g_clear_object (&file_info);
+ }
+ if (file_info == NULL && temp_error != NULL)
+ {
+ g_propagate_error (error, temp_error);
+ goto out;
+ }
+ if (!g_file_enumerator_close (enumerator, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ g_clear_object (&objdir);
+ g_clear_object (&file_info);
+ g_clear_object (&enumerator);
+ return ret;
+}
+
+GFile *
+ostree_repo_get_pack_index_path (OstreeRepo *self,
+ const char *checksum)
+{
+ char *name;
+ GFile *ret;
+
+ name = g_strconcat ("ostpack-", checksum, ".index", NULL);
+ ret = g_file_get_child (GET_PRIVATE (self)->pack_dir, name);
+ g_free (name);
- ret = TRUE;
- ot_transfer_out_value(out_checksum, &ret_checksum);
- out:
- g_clear_object (&archive_stream);
- ot_clear_checksum (&ret_checksum);
return ret;
}
-static gboolean
-stage_libarchive_entry_to_mtree (OstreeRepo *self,
- OstreeMutableTree *root,
- struct archive *a,
- struct archive_entry *entry,
- OstreeRepoCommitModifier *modifier,
- const char *tmp_dir_checksum,
- GCancellable *cancellable,
- GError **error)
+GFile *
+ostree_repo_get_pack_data_path (OstreeRepo *self,
+ const char *checksum)
{
- gboolean ret = FALSE;
- const char *pathname;
- const char *hardlink;
- const char *basename;
- GFileInfo *file_info = NULL;
- GChecksum *tmp_checksum = NULL;
- GPtrArray *split_path = NULL;
- GPtrArray *hardlink_split_path = NULL;
- OstreeMutableTree *subdir = NULL;
- OstreeMutableTree *parent = NULL;
- OstreeMutableTree *hardlink_source_parent = NULL;
- char *hardlink_source_checksum = NULL;
- OstreeMutableTree *hardlink_source_subdir = NULL;
+ char *name;
+ GFile *ret;
- pathname = archive_entry_pathname (entry);
-
- if (!ot_util_path_split_validate (pathname, &split_path, error))
- goto out;
+ name = g_strconcat ("ostpack-", checksum, ".data", NULL);
+ ret = g_file_get_child (GET_PRIVATE (self)->pack_dir, name);
+ g_free (name);
- if (split_path->len == 0)
+ return ret;
+}
+
+gboolean
+ostree_repo_load_pack_index (OstreeRepo *self,
+ const char *sha256,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ GVariant *ret_variant = NULL;
+ GFile *path = NULL;
+
+ ret_variant = g_hash_table_lookup (priv->pack_index_mappings, sha256);
+ if (ret_variant)
{
- parent = NULL;
- basename = NULL;
+ g_variant_ref (ret_variant);
}
else
{
- if (tmp_dir_checksum)
- {
- if (!ostree_mutable_tree_ensure_parent_dirs (root, split_path,
- tmp_dir_checksum,
- &parent,
- error))
- goto out;
- }
- else
- {
- if (!ostree_mutable_tree_walk (root, split_path, 0, &parent, error))
- goto out;
- }
- basename = (char*)split_path->pdata[split_path->len-1];
+ path = ostree_repo_get_pack_index_path (self, sha256);
+ if (!map_variant_file_check_header_string (path,
+ OSTREE_PACK_INDEX_VARIANT_FORMAT,
+ "OSTv0PACKINDEX",
+ &ret_variant,
+ cancellable, error))
+ goto out;
+ g_hash_table_insert (priv->pack_index_mappings, g_strdup (sha256),
+ g_variant_ref (ret_variant));
}
- hardlink = archive_entry_hardlink (entry);
- if (hardlink)
+ ret = TRUE;
+ ot_transfer_out_value (out_variant, &ret_variant);
+ out:
+ g_clear_object (&path);
+ ot_clear_gvariant (&ret_variant);
+ return ret;
+}
+
+/**
+ * ostree_repo_map_pack_file:
+ * @self:
+ * @sha256: Checksum of pack file
+ * @out_data: (out): Pointer to pack file data
+ * @cancellable:
+ * @error:
+ *
+ * Ensure that the given pack file is mapped into
+ * memory.
+ */
+gboolean
+ostree_repo_map_pack_file (OstreeRepo *self,
+ const char *sha256,
+ guchar **out_data,
+ guint64 *out_len,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ GMappedFile *map;
+ gpointer ret_data;
+ guint64 ret_len;
+ GFile *path = NULL;
+
+ map = g_hash_table_lookup (priv->pack_data_mappings, sha256);
+ if (map == NULL)
{
- const char *hardlink_basename;
-
- g_assert (parent != NULL);
+ path = ostree_repo_get_pack_data_path (self, sha256);
- if (!ot_util_path_split_validate (hardlink, &hardlink_split_path, error))
+ map = g_mapped_file_new (ot_gfile_get_path_cached (path), FALSE, error);
+ if (!map)
goto out;
- if (hardlink_split_path->len == 0)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Invalid hardlink path %s", hardlink);
- goto out;
- }
-
- hardlink_basename = hardlink_split_path->pdata[hardlink_split_path->len - 1];
-
- if (!ostree_mutable_tree_walk (root, hardlink_split_path, 0, &hardlink_source_parent, error))
+
+ g_hash_table_insert (priv->pack_data_mappings, g_strdup (sha256), map);
+ ret_data = g_mapped_file_get_contents (map);
+ }
+
+ ret_data = g_mapped_file_get_contents (map);
+ ret_len = (guint64)g_mapped_file_get_length (map);
+
+ ret = TRUE;
+ if (out_data)
+ *out_data = ret_data;
+ if (out_len)
+ *out_len = ret_len;
+ out:
+ g_clear_object (&path);
+ return ret;
+}
+
+
+gboolean
+ostree_repo_load_file (OstreeRepo *self,
+ const char *checksum,
+ GInputStream **out_input,
+ GFileInfo **out_file_info,
+ GVariant **out_xattrs,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GVariant *archive_meta = NULL;
+ GFile *content_loose_path = NULL;
+ GFileInfo *content_loose_info = NULL;
+ char *content_pack_checksum = NULL;
+ guint64 content_pack_offset;
+ guchar *content_pack_data;
+ guint64 content_pack_len;
+ GVariant *packed_object = NULL;
+ GInputStream *ret_input = NULL;
+ GFileInfo *ret_file_info = NULL;
+ GVariant *ret_xattrs = NULL;
+
+ if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_ARCHIVE)
+ {
+ /* First, read the metadata */
+ if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, checksum,
+ &archive_meta, error))
goto out;
-
- if (!ostree_mutable_tree_lookup (hardlink_source_parent, hardlink_basename,
- &hardlink_source_checksum,
- &hardlink_source_subdir,
- error))
- {
- g_prefix_error (error, "While resolving hardlink target: ");
- goto out;
- }
-
- if (hardlink_source_subdir)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Hardlink %s refers to directory %s",
- pathname, hardlink);
- goto out;
- }
- g_assert (hardlink_source_checksum);
-
- if (!ostree_mutable_tree_replace_file (parent,
- basename,
- hardlink_source_checksum,
- error))
+ if (!ostree_parse_archived_file_meta (archive_meta,
+ &ret_file_info,
+ &ret_xattrs,
+ error))
goto out;
- }
- else
- {
- file_info = file_info_from_archive_entry_and_modifier (entry, modifier);
- if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_UNKNOWN)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Unsupported file for import: %s", pathname);
- goto out;
- }
+ /* Blah, right now we need to look up the content too to get the file size */
+ if (!ostree_repo_find_object (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
+ checksum, &content_loose_path, NULL,
+ &content_pack_checksum, &content_pack_offset,
+ cancellable, error))
+ goto out;
- if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY)
+ if (content_loose_path)
{
-
- if (!stage_directory_meta (self, file_info, NULL, &tmp_checksum, cancellable, error))
+ content_loose_info = g_file_query_info (content_loose_path, OSTREE_GIO_FAST_QUERYINFO,
+ G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error);
+ if (!content_loose_info)
goto out;
- if (parent == NULL)
+ g_file_info_set_attribute_uint64 (ret_file_info,
+ "standard::size",
+ g_file_info_get_attribute_uint64 (content_loose_info, "standard::size"));
+ }
+ /* fixme - don't have file size for packed =/ */
+
+ /* Now, look for the content */
+ if (g_file_info_get_file_type (ret_file_info) == G_FILE_TYPE_REGULAR
+ && out_input)
+ {
+ if (content_pack_checksum != NULL)
{
- subdir = g_object_ref (root);
+ if (!ostree_repo_map_pack_file (self, content_pack_checksum,
+ &content_pack_data, &content_pack_len,
+ cancellable, error))
+ goto out;
+ if (!ostree_read_pack_entry_raw (content_pack_data, content_pack_len,
+ content_pack_offset, TRUE,
+ &packed_object, cancellable, error))
+ goto out;
+ ret_input = ostree_read_pack_entry_as_stream (packed_object);
}
- else
+ else if (content_loose_path != NULL)
{
- if (!ostree_mutable_tree_ensure_dir (parent, basename, &subdir, error))
+ ret_input = (GInputStream*)g_file_read (content_loose_path, cancellable, error);
+ if (!ret_input)
goto out;
}
-
- ostree_mutable_tree_set_metadata_checksum (subdir, g_checksum_get_string (tmp_checksum));
- }
- else
- {
- if (parent == NULL)
+ else
{
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Can't import file as root");
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
+ "Couldn't find object '%s'", checksum);
goto out;
}
-
- if (!import_libarchive_entry_file (self, a, entry, file_info, &tmp_checksum, cancellable, error))
- goto out;
-
- if (!ostree_mutable_tree_replace_file (parent, basename,
- g_checksum_get_string (tmp_checksum),
- error))
+ }
+ }
+ else
+ {
+ content_loose_path = ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_RAW_FILE);
+ ret_file_info = g_file_query_info (content_loose_path, OSTREE_GIO_FAST_QUERYINFO,
+ G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error);
+ if (!ret_file_info)
+ goto out;
+ if (out_xattrs)
+ {
+ ret_xattrs = ostree_get_xattrs_for_file (content_loose_path, error);
+ if (!ret_xattrs)
goto out;
}
}
ret = TRUE;
+ ot_transfer_out_value (out_input, &ret_input);
+ ot_transfer_out_value (out_file_info, &ret_file_info);
+ ot_transfer_out_value (out_xattrs, &ret_xattrs);
out:
- g_clear_object (&file_info);
- ot_clear_checksum (&tmp_checksum);
- g_clear_object (&parent);
- g_clear_object (&subdir);
- g_clear_object (&hardlink_source_parent);
- g_free (hardlink_source_checksum);
- g_clear_object (&hardlink_source_subdir);
- if (hardlink_split_path)
- g_ptr_array_unref (hardlink_split_path);
- if (split_path)
- g_ptr_array_unref (split_path);
+ g_free (content_pack_checksum);
+ g_clear_object (&ret_input);
+ g_clear_object (&content_loose_info);
+ g_clear_object (&ret_file_info);
+ ot_clear_gvariant (&ret_xattrs);
+ ot_clear_gvariant (&archive_meta);
+ ot_clear_gvariant (&packed_object);
return ret;
}
-#endif
-
-gboolean
-ostree_repo_stage_archive_to_mtree (OstreeRepo *self,
- GFile *archive_f,
- OstreeMutableTree *root,
- OstreeRepoCommitModifier *modifier,
- gboolean autocreate_parents,
- GCancellable *cancellable,
- GError **error)
+
+static gboolean
+list_objects_in_index (OstreeRepo *self,
+ const char *pack_checksum,
+ GHashTable *inout_objects,
+ GCancellable *cancellable,
+ GError **error)
{
-#ifdef HAVE_LIBARCHIVE
gboolean ret = FALSE;
- struct archive *a = NULL;
- struct archive_entry *entry;
- int r;
- GFileInfo *tmp_dir_info = NULL;
- GChecksum *tmp_dir_checksum = NULL;
+ GFile *index_path = NULL;
+ GVariant *index_variant = NULL;
+ GVariant *contents;
+ GVariantIter content_iter;
+ GVariant *csum_bytes;
+ char *checksum = NULL;
+ guint32 objtype_u32;
+ guint64 offset;
- a = archive_read_new ();
- archive_read_support_compression_all (a);
- archive_read_support_format_all (a);
- if (archive_read_open_filename (a, ot_gfile_get_path_cached (archive_f), 8192) != ARCHIVE_OK)
- {
- propagate_libarchive_error (error, a);
- goto out;
- }
+ index_path = ostree_repo_get_pack_index_path (self, pack_checksum);
- while (TRUE)
+ if (!ostree_repo_load_pack_index (self, pack_checksum, &index_variant, cancellable, error))
+ goto out;
+
+ contents = g_variant_get_child_value (index_variant, 2);
+ g_variant_iter_init (&content_iter, contents);
+
+ while (g_variant_iter_loop (&content_iter, "(u ayt)", &objtype_u32, &csum_bytes, &offset))
{
- r = archive_read_next_header (a, &entry);
- if (r == ARCHIVE_EOF)
- break;
- else if (r != ARCHIVE_OK)
- {
- propagate_libarchive_error (error, a);
- goto out;
- }
+ GVariant *obj_key;
+ GVariant *objdata;
+ OstreeObjectType objtype;
+ GVariantBuilder pack_contents_builder;
+ gboolean is_loose;
- if (autocreate_parents && !tmp_dir_checksum)
+ objtype = (OstreeObjectType) GUINT32_FROM_BE (objtype_u32);
+ offset = GUINT64_FROM_BE (offset);
+
+ g_variant_builder_init (&pack_contents_builder,
+ G_VARIANT_TYPE_STRING_ARRAY);
+
+ g_free (checksum);
+ checksum = ostree_checksum_from_bytes (csum_bytes);
+ obj_key = ostree_object_name_serialize (checksum, objtype);
+ ot_util_variant_take_ref (obj_key);
+
+ objdata = g_hash_table_lookup (inout_objects, obj_key);
+ if (!objdata)
{
- tmp_dir_info = g_file_info_new ();
-
- g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", archive_entry_uid (entry));
- g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", archive_entry_gid (entry));
- g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR);
-
- if (!stage_directory_meta (self, tmp_dir_info, NULL, &tmp_dir_checksum, cancellable, error))
- goto out;
+ is_loose = FALSE;
}
+ else
+ {
+ GVariantIter *current_packs_iter;
+ const char *current_pack_checksum;
- if (!stage_libarchive_entry_to_mtree (self, root, a,
- entry, modifier,
- autocreate_parents ? g_checksum_get_string (tmp_dir_checksum) : NULL,
- cancellable, error))
- goto out;
- }
- if (archive_read_close (a) != ARCHIVE_OK)
- {
- propagate_libarchive_error (error, a);
- goto out;
+ g_variant_get (objdata, "(bas)", &is_loose, ¤t_packs_iter);
+
+ while (g_variant_iter_loop (current_packs_iter, "&s", ¤t_pack_checksum))
+ {
+ g_variant_builder_add (&pack_contents_builder, "s", current_pack_checksum);
+ }
+ g_variant_iter_free (current_packs_iter);
+ }
+ g_variant_builder_add (&pack_contents_builder, "s", pack_checksum);
+ objdata = g_variant_new ("(b as)", is_loose,
+ g_variant_builder_end (&pack_contents_builder));
+ g_hash_table_replace (inout_objects,
+ obj_key,
+ g_variant_ref (objdata));
}
ret = TRUE;
out:
- g_clear_object (&tmp_dir_info);
- ot_clear_checksum (&tmp_dir_checksum);
- if (a)
- (void)archive_read_close (a);
+ g_free (checksum);
+ g_clear_object (&index_path);
+ ot_clear_gvariant (&index_variant);
+ ot_clear_gvariant (&contents);
return ret;
-#else
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
- "This version of ostree is not compiled with libarchive support");
- return FALSE;
-#endif
}
-OstreeRepoCommitModifier *
-ostree_repo_commit_modifier_new (void)
+static gboolean
+list_packed_objects (OstreeRepo *self,
+ GHashTable *inout_objects,
+ GCancellable *cancellable,
+ GError **error)
{
- OstreeRepoCommitModifier *modifier = g_new0 (OstreeRepoCommitModifier, 1);
-
- modifier->refcount = 1;
+ gboolean ret = FALSE;
+ GPtrArray *index_checksums = NULL;
+ guint i;
- return modifier;
-}
+ if (!ostree_repo_list_pack_indexes (self, &index_checksums, cancellable, error))
+ goto out;
-void
-ostree_repo_commit_modifier_unref (OstreeRepoCommitModifier *modifier)
-{
- if (!modifier)
- return;
- if (!g_atomic_int_dec_and_test (&modifier->refcount))
- return;
+ for (i = 0; i < index_checksums->len; i++)
+ {
+ const char *checksum = index_checksums->pdata[i];
- g_free (modifier);
- return;
+ if (!list_objects_in_index (self, checksum, inout_objects, cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ if (index_checksums)
+ g_ptr_array_unref (index_checksums);
+ return ret;
}
-
static gboolean
-iter_object_dir (OstreeRepo *self,
- GFile *dir,
- OstreeRepoObjectIter callback,
- gpointer user_data,
- GError **error)
+find_object_in_packs (OstreeRepo *self,
+ const char *checksum,
+ OstreeObjectType objtype,
+ char **out_pack_checksum,
+ guint64 *out_pack_offset,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
- GError *temp_error = NULL;
- GFileEnumerator *enumerator = NULL;
- GFileInfo *file_info = NULL;
- const char *dirname = NULL;
+ GPtrArray *index_checksums = NULL;
+ char *ret_pack_checksum = NULL;
+ guint64 ret_pack_offset;
+ GFile *index_path = NULL;
+ GVariant *csum_bytes = NULL;
+ GVariant *index_variant = NULL;
+ guint i;
- dirname = ot_gfile_get_basename_cached (dir);
+ csum_bytes = ostree_checksum_to_bytes (checksum);
- enumerator = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
- NULL,
- error);
- if (!enumerator)
+ if (!ostree_repo_list_pack_indexes (self, &index_checksums, cancellable, error))
goto out;
+
+ for (i = 0; i < index_checksums->len; i++)
+ {
+ const char *pack_checksum = index_checksums->pdata[i];
+ guint64 offset;
+
+ g_clear_object (&index_path);
+ index_path = ostree_repo_get_pack_index_path (self, pack_checksum);
+
+ ot_clear_gvariant (&index_variant);
+ if (!ostree_repo_load_pack_index (self, pack_checksum, &index_variant, cancellable, error))
+ goto out;
+
+ if (!ostree_pack_index_search (index_variant, csum_bytes, objtype, &offset))
+ continue;
+
+ ret_pack_checksum = g_strdup (pack_checksum);
+ ret_pack_offset = offset;
+ break;
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum);
+ if (out_pack_offset)
+ *out_pack_offset = ret_pack_offset;
+ out:
+ g_free (ret_pack_checksum);
+ if (index_checksums)
+ g_ptr_array_unref (index_checksums);
+ g_clear_object (&index_path);
+ ot_clear_gvariant (&index_variant);
+ ot_clear_gvariant (&csum_bytes);
+ return ret;
+}
+
+gboolean
+ostree_repo_find_object (OstreeRepo *self,
+ OstreeObjectType objtype,
+ const char *checksum,
+ GFile **out_stored_path,
+ GFile **out_pending_path,
+ char **out_pack_checksum,
+ guint64 *out_pack_offset,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *object_path = NULL;
+ GFile *ret_stored_path = NULL;
+ GFile *ret_pending_path = NULL;
+ char *ret_pack_checksum = NULL;
+ guint64 ret_pack_offset = 0;
+ struct stat stbuf;
+
+ object_path = ostree_repo_get_object_path (self, checksum, objtype);
- while ((file_info = g_file_enumerator_next_file (enumerator, NULL, &temp_error)) != NULL)
+ if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0)
{
- const char *name;
- guint32 type;
- char *dot = NULL;
- GFile *child = NULL;
- GString *checksum = NULL;
- OstreeObjectType objtype;
+ ret_stored_path = object_path;
+ object_path = NULL;
+ }
+ else
+ {
+ g_clear_object (&object_path);
+ if (out_pending_path)
+ {
+ object_path = get_pending_object_path (self, checksum, objtype);
+ if (lstat (ot_gfile_get_path_cached (object_path), &stbuf) == 0)
+ {
+ ret_pending_path = object_path;
+ object_path = NULL;
+ }
+ }
+ }
- name = g_file_info_get_attribute_byte_string (file_info, "standard::name");
- type = g_file_info_get_attribute_uint32 (file_info, "standard::type");
+ if (out_pack_checksum)
+ {
+ if (!find_object_in_packs (self, checksum, objtype,
+ &ret_pack_checksum, &ret_pack_offset,
+ cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_stored_path, &ret_stored_path);
+ ot_transfer_out_value (out_pending_path, &ret_pending_path);
+ ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum);
+ if (out_pack_offset)
+ *out_pack_offset = ret_pack_offset;
+out:
+ g_clear_object (&object_path);
+ g_clear_object (&ret_stored_path);
+ g_clear_object (&ret_pending_path);
+ g_free (ret_pack_checksum);
+ return ret;
+}
- if (type == G_FILE_TYPE_DIRECTORY)
- goto loop_out;
-
- if (g_str_has_suffix (name, ".file"))
- objtype = OSTREE_OBJECT_TYPE_RAW_FILE;
- else if (g_str_has_suffix (name, ".archive-meta"))
- objtype = OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META;
- else if (g_str_has_suffix (name, ".archive-content"))
- objtype = OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT;
- else if (g_str_has_suffix (name, ".dirtree"))
- objtype = OSTREE_OBJECT_TYPE_DIR_TREE;
- else if (g_str_has_suffix (name, ".dirmeta"))
- objtype = OSTREE_OBJECT_TYPE_DIR_META;
- else if (g_str_has_suffix (name, ".commit"))
- objtype = OSTREE_OBJECT_TYPE_COMMIT;
- else
- goto loop_out;
-
- dot = strrchr (name, '.');
- g_assert (dot);
+gboolean
+ostree_repo_load_variant (OstreeRepo *self,
+ OstreeObjectType objtype,
+ const char *sha256,
+ GVariant **out_variant,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *object_path = NULL;
+ GFile *pending_path = NULL;
+ GVariant *packed_object = NULL;
+ GVariant *ret_variant = NULL;
+ char *pack_checksum = NULL;
+ guchar *pack_data;
+ guint64 pack_len;
+ guint64 object_offset;
+ GCancellable *cancellable = NULL;
- if ((dot - name) != 62)
- goto loop_out;
-
- checksum = g_string_new (dirname);
- g_string_append_len (checksum, name, 62);
-
- child = g_file_get_child (dir, name);
- callback (self, checksum->str, objtype, child, file_info, user_data);
+ g_return_val_if_fail (OSTREE_OBJECT_TYPE_IS_META (objtype), FALSE);
+
+ if (!ostree_repo_find_object (self, objtype, sha256, &object_path, &pending_path,
+ &pack_checksum, &object_offset,
+ cancellable, error))
+ goto out;
+
+ /* Prefer loose metadata for now */
+ if (object_path != NULL || pending_path != NULL)
+ {
+ if (!ostree_map_metadata_file (object_path ? object_path : pending_path,
+ objtype, &ret_variant, error))
+ goto out;
+ }
+ else if (pack_checksum != NULL)
+ {
+ if (!ostree_repo_map_pack_file (self, pack_checksum, &pack_data, &pack_len,
+ cancellable, error))
+ goto out;
- loop_out:
- if (checksum)
- g_string_free (checksum, TRUE);
- g_clear_object (&file_info);
- g_clear_object (&child);
+ if (!ostree_read_pack_entry_raw (pack_data, pack_len, object_offset,
+ TRUE, &packed_object, cancellable, error))
+ goto out;
+
+ if (!ostree_read_pack_entry_variant (packed_object, objtype, TRUE,
+ &ret_variant, cancellable, error))
+ goto out;
}
- if (temp_error != NULL)
+ else
{
- g_propagate_error (error, temp_error);
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "No such metadata object %s.%s",
+ sha256, ostree_object_type_to_string (objtype));
goto out;
}
- if (!g_file_enumerator_close (enumerator, NULL, error))
- goto out;
ret = TRUE;
+ ot_transfer_out_value (out_variant, &ret_variant);
out:
- g_clear_object (&file_info);
+ g_clear_object (&object_path);
+ g_free (pack_checksum);
+ ot_clear_gvariant (&ret_variant);
+ ot_clear_gvariant (&packed_object);
return ret;
}
+/**
+ * ostree_repo_list_objects:
+ * @self:
+ * @flags:
+ * @out_objects: (out): Map of serialized object name to variant data
+ * @cancellable:
+ * @error:
+ *
+ * This function synchronously enumerates all objects in the
+ * repository, returning data in @out_objects. @out_objects
+ * maps from keys returned by ostree_object_name_serialize()
+ * to #GVariant values of type %OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE.
+ *
+ * Returns: %TRUE on success, %FALSE on error, and @error will be set
+ */
gboolean
-ostree_repo_iter_objects (OstreeRepo *self,
- OstreeRepoObjectIter callback,
- gpointer user_data,
- GError **error)
+ostree_repo_list_objects (OstreeRepo *self,
+ OstreeRepoListObjectsFlags flags,
+ GHashTable **out_objects,
+ GCancellable *cancellable,
+ GError **error)
{
- OstreeRepoPrivate *priv = GET_PRIVATE (self);
- GFileEnumerator *enumerator = NULL;
gboolean ret = FALSE;
- GFileInfo *file_info = NULL;
- GError *temp_error = NULL;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ GHashTable *ret_objects = NULL;
g_return_val_if_fail (error == NULL || *error == NULL, FALSE);
g_return_val_if_fail (priv->inited, FALSE);
+
+ ret_objects = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
+ (GDestroyNotify) g_variant_unref,
+ (GDestroyNotify) g_variant_unref);
- enumerator = g_file_enumerate_children (priv->objects_dir, OSTREE_GIO_FAST_QUERYINFO,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
- NULL,
- error);
- if (!enumerator)
- goto out;
+ if (flags & OSTREE_REPO_LIST_OBJECTS_ALL)
+ flags |= (OSTREE_REPO_LIST_OBJECTS_LOOSE | OSTREE_REPO_LIST_OBJECTS_PACKED);
- while ((file_info = g_file_enumerator_next_file (enumerator, NULL, &temp_error)) != NULL)
+ if (flags & OSTREE_REPO_LIST_OBJECTS_LOOSE)
{
- const char *name;
- guint32 type;
-
- name = g_file_info_get_attribute_byte_string (file_info, "standard::name");
- type = g_file_info_get_attribute_uint32 (file_info, "standard::type");
-
- if (strlen (name) == 2 && type == G_FILE_TYPE_DIRECTORY)
- {
- GFile *objdir = g_file_get_child (priv->objects_dir, name);
- if (!iter_object_dir (self, objdir, callback, user_data, error))
- {
- g_object_unref (objdir);
- goto out;
- }
- g_object_unref (objdir);
- }
- g_object_unref (file_info);
+ if (!list_loose_objects (self, ret_objects, cancellable, error))
+ goto out;
}
- if (file_info == NULL && temp_error != NULL)
+
+ if (flags & OSTREE_REPO_LIST_OBJECTS_PACKED)
{
- g_propagate_error (error, temp_error);
- goto out;
+ if (!list_packed_objects (self, ret_objects, cancellable, error))
+ goto out;
}
- if (!g_file_enumerator_close (enumerator, NULL, error))
- goto out;
ret = TRUE;
+ ot_transfer_out_value (out_objects, &ret_objects);
out:
- g_clear_object (&file_info);
- g_clear_object (&enumerator);
+ if (ret_objects)
+ g_hash_table_unref (ret_objects);
return ret;
}
@@ -2574,6 +3781,61 @@ checkout_file_hardlink (OstreeRepo *self,
return ret;
}
+static gboolean
+checkout_one_file (OstreeRepo *self,
+ OstreeRepoCheckoutMode mode,
+ OstreeRepoCheckoutOverwriteMode overwrite_mode,
+ OstreeRepoFile *src,
+ GFileInfo *file_info,
+ GFile *destination,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ GFile *possible_loose_path = NULL;
+ GInputStream *input = NULL;
+ GVariant *xattrs = NULL;
+ const char *checksum;
+ struct stat stbuf;
+
+ checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)src);
+
+ /* First check for a loose object */
+ if (priv->mode == OSTREE_REPO_MODE_ARCHIVE && mode == OSTREE_REPO_CHECKOUT_MODE_USER)
+ {
+ possible_loose_path = ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
+ }
+ else if (priv->mode == OSTREE_REPO_MODE_BARE && mode == OSTREE_REPO_CHECKOUT_MODE_NONE)
+ {
+ possible_loose_path = ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_RAW_FILE);
+ }
+
+ if (possible_loose_path && lstat (ot_gfile_get_path_cached (possible_loose_path), &stbuf) >= 0)
+ {
+ /* If we found one, we can just hardlink */
+ if (!checkout_file_hardlink (self, mode, overwrite_mode, possible_loose_path, destination,
+ cancellable, error) < 0)
+ goto out;
+ }
+ else
+ {
+ if (!ostree_repo_load_file (self, checksum, &input, NULL, &xattrs, cancellable, error))
+ goto out;
+
+ if (!checkout_file_from_input (destination, mode, overwrite_mode, file_info, xattrs,
+ input, cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ g_clear_object (&possible_loose_path);
+ g_clear_object (&input);
+ ot_clear_gvariant (&xattrs);
+ return ret;
+}
+
gboolean
ostree_repo_checkout_tree (OstreeRepo *self,
OstreeRepoCheckoutMode mode,
@@ -2584,18 +3846,13 @@ ostree_repo_checkout_tree (OstreeRepo *self,
GCancellable *cancellable,
GError **error)
{
- OstreeRepoPrivate *priv = GET_PRIVATE (self);
gboolean ret = FALSE;
GError *temp_error = NULL;
- GVariant *archive_metadata = NULL;
GFileInfo *file_info = NULL;
GVariant *xattrs = NULL;
GFileEnumerator *dir_enum = NULL;
GFile *src_child = NULL;
GFile *dest_path = NULL;
- GFile *object_path = NULL;
- GFile *content_object_path = NULL;
- GInputStream *content_input = NULL;
if (!ostree_repo_file_get_xattrs (source, &xattrs, NULL, error))
goto out;
@@ -2637,49 +3894,10 @@ ostree_repo_checkout_tree (OstreeRepo *self,
}
else
{
- const char *checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)src_child);
-
- if (priv->mode == OSTREE_REPO_MODE_ARCHIVE && mode == OSTREE_REPO_CHECKOUT_MODE_USER)
- {
- g_clear_object (&object_path);
- object_path = ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
-
- if (!checkout_file_hardlink (self, mode, overwrite_mode, object_path, dest_path, cancellable, error) < 0)
- goto out;
- }
- else if (priv->mode == OSTREE_REPO_MODE_ARCHIVE)
- {
- ot_clear_gvariant (&archive_metadata);
- if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, checksum, &archive_metadata, error))
- goto out;
-
- ot_clear_gvariant (&xattrs);
- if (!ostree_parse_archived_file_meta (archive_metadata, NULL, &xattrs, error))
- goto out;
-
- g_clear_object (&content_object_path);
- content_object_path = ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
-
- g_clear_object (&content_input);
- if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
- {
- content_input = (GInputStream*)g_file_read (content_object_path, cancellable, error);
- if (!content_input)
- goto out;
- }
-
- if (!checkout_file_from_input (dest_path, mode, overwrite_mode, file_info, xattrs,
- content_input, cancellable, error))
- goto out;
- }
- else
- {
- g_clear_object (&object_path);
- object_path = ostree_repo_get_object_path (self, checksum, OSTREE_OBJECT_TYPE_RAW_FILE);
-
- if (!checkout_file_hardlink (self, mode, overwrite_mode, object_path, dest_path, cancellable, error) < 0)
- goto out;
- }
+ if (!checkout_one_file (self, mode, overwrite_mode,
+ (OstreeRepoFile*)src_child, file_info,
+ dest_path, cancellable, error))
+ goto out;
}
g_clear_object (&file_info);
@@ -2695,15 +3913,12 @@ ostree_repo_checkout_tree (OstreeRepo *self,
g_clear_object (&dir_enum);
g_clear_object (&file_info);
ot_clear_gvariant (&xattrs);
- ot_clear_gvariant (&archive_metadata);
g_clear_object (&src_child);
- g_clear_object (&object_path);
- g_clear_object (&content_object_path);
- g_clear_object (&content_input);
g_clear_object (&dest_path);
g_free (dest_path);
return ret;
}
+
gboolean
ostree_repo_read_commit (OstreeRepo *self,
const char *rev,
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 25ef09e..55c918e 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -97,6 +97,8 @@ gboolean ostree_repo_find_object (OstreeRepo *self,
const char *checksum,
GFile **out_stored_path,
GFile **out_pending_path,
+ char **out_pack_checksum,
+ guint64 *out_pack_offset,
GCancellable *cancellable,
GError **error);
@@ -112,6 +114,7 @@ gboolean ostree_repo_stage_object (OstreeRepo *self,
gboolean ostree_repo_stage_object_trusted (OstreeRepo *self,
OstreeObjectType objtype,
const char *checksum,
+ gboolean store_if_packed,
GFileInfo *file_info,
GVariant *xattrs,
GInputStream *content,
@@ -141,6 +144,33 @@ gboolean ostree_repo_load_variant (OstreeRepo *self,
GVariant **out_variant,
GError **error);
+gboolean ostree_repo_load_pack_index (OstreeRepo *self,
+ const char *sha256,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_load_pack_data (OstreeRepo *self,
+ const char *sha256,
+ guchar **out_data,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_map_pack_file (OstreeRepo *self,
+ const char *sha256,
+ guchar **out_data,
+ guint64 *out_len,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_load_file (OstreeRepo *self,
+ const char *entry_sha256,
+ GInputStream **out_input,
+ GFileInfo **out_file_info,
+ GVariant **out_xattrs,
+ GCancellable *cancellable,
+ GError **error);
+
typedef enum {
OSTREE_REPO_COMMIT_FILTER_ALLOW,
OSTREE_REPO_COMMIT_FILTER_SKIP
@@ -200,6 +230,53 @@ gboolean ostree_repo_stage_commit (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
+gboolean ostree_repo_regenerate_pack_index (OstreeRepo *self,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_add_pack_file (OstreeRepo *self,
+ const char *checksum,
+ GFile *pack_index_path,
+ GFile *pack_data_path,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_resync_cached_remote_pack_indexes (OstreeRepo *self,
+ const char *remote_name,
+ GFile *superindex_path,
+ GPtrArray **out_cached_indexes,
+ GPtrArray **out_uncached_indexes,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_map_cached_remote_pack_index (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_add_cached_remote_pack_index (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GFile *cached_path,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_get_cached_remote_pack_data (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GFile **out_cached_path,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_take_cached_remote_pack_data (OstreeRepo *self,
+ const char *remote_name,
+ const char *pack_checksum,
+ GFile *cached_path,
+ GCancellable *cancellable,
+ GError **error);
+
typedef enum {
OSTREE_REPO_CHECKOUT_MODE_NONE = 0,
OSTREE_REPO_CHECKOUT_MODE_USER = 1
@@ -226,17 +303,36 @@ gboolean ostree_repo_read_commit (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
-typedef void (*OstreeRepoObjectIter) (OstreeRepo *self,
- const char *checksum,
- OstreeObjectType type,
- GFile *path,
- GFileInfo *fileinfo,
- gpointer user_data);
-
-gboolean ostree_repo_iter_objects (OstreeRepo *self,
- OstreeRepoObjectIter callback,
- gpointer user_data,
- GError **error);
+typedef enum {
+ OSTREE_REPO_LIST_OBJECTS_LOOSE = (1 << 0),
+ OSTREE_REPO_LIST_OBJECTS_PACKED = (1 << 1),
+ OSTREE_REPO_LIST_OBJECTS_ALL = (1 << 2)
+} OstreeRepoListObjectsFlags;
+
+/**
+ * OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE:
+ *
+ * b - %TRUE if object is available "loose"
+ * as - List of pack file checksums in which this object appears
+ */
+#define OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE (G_VARIANT_TYPE ("(bas)")
+
+gboolean ostree_repo_list_objects (OstreeRepo *self,
+ OstreeRepoListObjectsFlags flags,
+ GHashTable **out_objects,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_list_pack_indexes (OstreeRepo *self,
+ GPtrArray **out_indexes,
+ GCancellable *cancellable,
+ GError **error);
+
+GFile * ostree_repo_get_pack_index_path (OstreeRepo *self,
+ const char *checksum);
+
+GFile * ostree_repo_get_pack_data_path (OstreeRepo *self,
+ const char *checksum);
G_END_DECLS
diff --git a/src/ostree/main.c b/src/ostree/main.c
index 71a2efc..1b434b6 100644
--- a/src/ostree/main.c
+++ b/src/ostree/main.c
@@ -42,10 +42,12 @@ static OstreeBuiltin builtins[] = {
{ "ls", ostree_builtin_ls, 0 },
{ "prune", ostree_builtin_prune, 0 },
{ "fsck", ostree_builtin_fsck, 0 },
+ { "pack", ostree_builtin_pack, 0 },
{ "remote", ostree_builtin_remote, 0 },
{ "rev-parse", ostree_builtin_rev_parse, 0 },
{ "remote", ostree_builtin_remote, 0 },
{ "show", ostree_builtin_show, 0 },
+ { "unpack", ostree_builtin_unpack, 0 },
{ NULL }
};
diff --git a/src/ostree/ostree-pull.c b/src/ostree/ostree-pull.c
index 40f3d46..1e54356 100644
--- a/src/ostree/ostree-pull.c
+++ b/src/ostree/ostree-pull.c
@@ -55,6 +55,55 @@ log_verbose (const char *fmt,
}
typedef struct {
+ OstreeRepo *repo;
+ char *remote_name;
+ SoupSession *session;
+ SoupURI *base_uri;
+
+ gboolean fetched_packs;
+ GPtrArray *cached_pack_indexes;
+} OtPullData;
+
+static SoupURI *
+suburi_new (SoupURI *base,
+ const char *first,
+ ...) G_GNUC_NULL_TERMINATED;
+
+static SoupURI *
+suburi_new (SoupURI *base,
+ const char *first,
+ ...)
+{
+ va_list args;
+ GPtrArray *arg_array;
+ const char *arg;
+ char *subpath;
+ SoupURI *ret;
+
+ arg_array = g_ptr_array_new ();
+ g_ptr_array_add (arg_array, (char*)soup_uri_get_path (base));
+ g_ptr_array_add (arg_array, (char*)first);
+
+ va_start (args, first);
+
+ while ((arg = va_arg (args, const char *)) != NULL)
+ g_ptr_array_add (arg_array, (char*)arg);
+ g_ptr_array_add (arg_array, NULL);
+
+ subpath = g_build_filenamev ((char**)arg_array->pdata);
+ g_ptr_array_unref (arg_array);
+
+ ret = soup_uri_copy (base);
+ soup_uri_set_path (ret, subpath);
+ g_free (subpath);
+
+ va_end (args);
+
+ return ret;
+}
+
+
+typedef struct {
SoupSession *session;
GOutputStream *stream;
gboolean had_error;
@@ -78,8 +127,7 @@ on_got_chunk (SoupMessage *msg,
}
static gboolean
-fetch_uri (OstreeRepo *repo,
- SoupSession *soup,
+fetch_uri (OtPullData *pull_data,
SoupURI *uri,
const char *tmp_prefix,
GFile **out_temp_filename,
@@ -94,14 +142,14 @@ fetch_uri (OstreeRepo *repo,
GOutputStream *output_stream = NULL;
OstreeSoupChunkData chunkdata;
- if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (repo),
+ if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (pull_data->repo),
tmp_prefix, NULL,
&ret_temp_filename,
&output_stream,
NULL, error))
goto out;
- chunkdata.session = soup;
+ chunkdata.session = pull_data->session;
chunkdata.stream = output_stream;
chunkdata.had_error = FALSE;
chunkdata.error = error;
@@ -114,7 +162,7 @@ fetch_uri (OstreeRepo *repo,
g_signal_connect (msg, "got-chunk", G_CALLBACK (on_got_chunk), &chunkdata);
- response = soup_session_send_message (soup, msg);
+ response = soup_session_send_message (pull_data->session, msg);
if (response != 200)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
@@ -138,8 +186,7 @@ fetch_uri (OstreeRepo *repo,
}
static gboolean
-fetch_uri_contents_utf8 (OstreeRepo *repo,
- SoupSession *soup,
+fetch_uri_contents_utf8 (OtPullData *pull_data,
SoupURI *uri,
char **out_contents,
GCancellable *cancellable,
@@ -150,7 +197,7 @@ fetch_uri_contents_utf8 (OstreeRepo *repo,
char *ret_contents = NULL;
gsize len;
- if (!fetch_uri (repo, soup, uri, "tmp-", &tmpf, cancellable, error))
+ if (!fetch_uri (pull_data, uri, "tmp-", &tmpf, cancellable, error))
goto out;
if (!g_file_load_contents (tmpf, cancellable, &ret_contents, &len, NULL, error))
@@ -173,29 +220,209 @@ fetch_uri_contents_utf8 (OstreeRepo *repo,
return ret;
}
+static gboolean
+fetch_one_pack_file (OtPullData *pull_data,
+ const char *pack_checksum,
+ GFile **out_cached_path,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *ret_cached_path = NULL;
+ GFile *tmp_path = NULL;
+ char *pack_name = NULL;
+ SoupURI *pack_uri = NULL;
+
+ if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
+ pack_checksum, &ret_cached_path,
+ cancellable, error))
+ goto out;
+
+ if (ret_cached_path == NULL)
+ {
+ pack_name = g_strconcat ("ostpack-", pack_checksum, ".data", NULL);
+ pack_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_name, NULL);
+
+ if (!fetch_uri (pull_data, pack_uri, "packdata-", &tmp_path, cancellable, error))
+ goto out;
+
+ if (!ostree_repo_take_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
+ pack_checksum, tmp_path,
+ cancellable, error))
+ goto out;
+ }
+
+ if (!ostree_repo_get_cached_remote_pack_data (pull_data->repo, pull_data->remote_name,
+ pack_checksum, &ret_cached_path,
+ cancellable, error))
+ goto out;
+
+ g_assert (ret_cached_path != NULL);
+
+ ret = TRUE;
+ ot_transfer_out_value (out_cached_path, &ret_cached_path);
+ out:
+ g_clear_object (&ret_cached_path);
+ g_clear_object (&tmp_path);
+ g_free (pack_name);
+ if (pack_uri)
+ soup_uri_free (pack_uri);
+ return ret;
+}
+
+static gboolean
+find_object_in_remote_packs (OtPullData *pull_data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ char **out_pack_checksum,
+ guint64 *out_offset,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GVariant *mapped_pack = NULL;
+ GVariant *csum_bytes = NULL;
+ char *ret_pack_checksum = NULL;
+ guint64 offset;
+ guint i;
+
+ csum_bytes = ostree_checksum_to_bytes (checksum);
+
+ for (i = 0; i < pull_data->cached_pack_indexes->len; i++)
+ {
+ const char *pack_checksum = pull_data->cached_pack_indexes->pdata[i];
+
+ ot_clear_gvariant (&mapped_pack);
+ if (!ostree_repo_map_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
+ pack_checksum, &mapped_pack,
+ cancellable, error))
+ goto out;
+
+ if (ostree_pack_index_search (mapped_pack, csum_bytes, objtype, &offset))
+ {
+ ret_pack_checksum = g_strdup (pack_checksum);
+ break;
+ }
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_pack_checksum, &ret_pack_checksum);
+ if (out_offset)
+ *out_offset = offset;
+ out:
+ ot_clear_gvariant (&mapped_pack);
+ g_free (ret_pack_checksum);
+ ot_clear_gvariant (&csum_bytes);
+ return ret;
+}
static gboolean
-fetch_object (OstreeRepo *repo,
- SoupSession *soup,
- SoupURI *baseuri,
- const char *checksum,
- OstreeObjectType objtype,
- GFile **out_temp_path,
- GCancellable *cancellable,
- GError **error)
+fetch_one_cache_index (OtPullData *pull_data,
+ const char *pack_checksum,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ SoupURI *index_uri = NULL;
+ GFile *tmp_path = NULL;
+ char *pack_index_name = NULL;
+
+ pack_index_name = g_strconcat ("ostpack-", pack_checksum, ".index", NULL);
+ index_uri = suburi_new (pull_data->base_uri, "objects", "pack", pack_index_name, NULL);
+
+ if (!fetch_uri (pull_data, index_uri, "packindex-", &tmp_path,
+ cancellable, error))
+ goto out;
+
+ if (!ostree_repo_add_cached_remote_pack_index (pull_data->repo, pull_data->remote_name,
+ pack_checksum, tmp_path,
+ cancellable, error))
+ goto out;
+
+ if (!ot_gfile_unlink (tmp_path, cancellable, error))
+ goto out;
+
+ g_clear_object (&tmp_path);
+
+ ret = TRUE;
+ out:
+ if (tmp_path != NULL)
+ (void) ot_gfile_unlink (tmp_path, NULL, NULL);
+ g_clear_object (&tmp_path);
+ g_free (pack_index_name);
+ if (index_uri)
+ soup_uri_free (index_uri);
+ return ret;
+}
+
+static gboolean
+fetch_and_cache_pack_indexes (OtPullData *pull_data,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ SoupURI *superindex_uri = NULL;
+ GFile *superindex_tmppath = NULL;
+ GPtrArray *cached_indexes = NULL;
+ GPtrArray *uncached_indexes = NULL;
+ GVariant *superindex_variant = NULL;
+ GVariantIter *contents_iter = NULL;
+ guint i;
+
+ superindex_uri = suburi_new (pull_data->base_uri, "objects", "pack", "index", NULL);
+
+ if (!fetch_uri (pull_data, superindex_uri, "index-",
+ &superindex_tmppath, cancellable, error))
+ goto out;
+
+ if (!ostree_repo_resync_cached_remote_pack_indexes (pull_data->repo, pull_data->remote_name,
+ superindex_tmppath,
+ &cached_indexes, &uncached_indexes,
+ cancellable, error))
+ goto out;
+
+ for (i = 0; i < cached_indexes->len; i++)
+ g_ptr_array_add (pull_data->cached_pack_indexes,
+ g_strdup (cached_indexes->pdata[i]));
+
+ for (i = 0; i < uncached_indexes->len; i++)
+ {
+ const char *pack_checksum = uncached_indexes->pdata[i];
+
+ if (!fetch_one_cache_index (pull_data, pack_checksum, cancellable, error))
+ goto out;
+
+ g_ptr_array_add (pull_data->cached_pack_indexes, g_strdup (pack_checksum));
+ }
+
+ ret = TRUE;
+ out:
+ if (superindex_uri)
+ soup_uri_free (superindex_uri);
+ g_clear_object (&superindex_tmppath);
+ ot_clear_gvariant (&superindex_variant);
+ if (contents_iter)
+ g_variant_iter_free (contents_iter);
+ return ret;
+}
+
+static gboolean
+fetch_loose_object (OtPullData *pull_data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ GFile **out_temp_path,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
char *objpath = NULL;
- char *relpath = NULL;
SoupURI *obj_uri = NULL;
GFile *ret_temp_path = NULL;
objpath = ostree_get_relative_object_path (checksum, objtype);
- obj_uri = soup_uri_copy (baseuri);
- relpath = g_build_filename (soup_uri_get_path (obj_uri), objpath, NULL);
- soup_uri_set_path (obj_uri, relpath);
+ obj_uri = suburi_new (pull_data->base_uri, objpath, NULL);
- if (!fetch_uri (repo, soup, obj_uri, ostree_object_type_to_string (objtype), &ret_temp_path,
+ if (!fetch_uri (pull_data, obj_uri, ostree_object_type_to_string (objtype), &ret_temp_path,
cancellable, error))
goto out;
@@ -206,93 +433,321 @@ fetch_object (OstreeRepo *repo,
soup_uri_free (obj_uri);
g_clear_object (&ret_temp_path);
g_free (objpath);
- g_free (relpath);
return ret;
}
static gboolean
-fetch_and_store_object (OstreeRepo *repo,
- SoupSession *soup,
- SoupURI *baseuri,
- const char *checksum,
- OstreeObjectType objtype,
- gboolean *out_is_pending,
- GVariant **out_metadata,
- GCancellable *cancellable,
- GError **error)
+find_object (OtPullData *pull_data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ gboolean *out_is_stored,
+ gboolean *out_is_pending,
+ char **out_remote_pack_checksum,
+ guint64 *out_offset,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
- GFileInfo *file_info = NULL;
- GInputStream *input = NULL;
+ gboolean ret_is_stored;
+ gboolean ret_is_pending;
GFile *stored_path = NULL;
GFile *pending_path = NULL;
- GFile *temp_path = NULL;
- GVariant *ret_metadata = NULL;
- gboolean ret_is_pending;
+ char *local_pack_checksum = NULL;
+ char *ret_remote_pack_checksum = NULL;
+ guint64 offset;
+
+ if (!ostree_repo_find_object (pull_data->repo, objtype, checksum,
+ &stored_path, &pending_path,
+ &local_pack_checksum, NULL,
+ cancellable, error))
+ goto out;
- g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE);
+ ret_is_stored = (stored_path != NULL || local_pack_checksum != NULL);
+ ret_is_pending = pending_path != NULL;
- if (!ostree_repo_find_object (repo, objtype, checksum,
- &stored_path, &pending_path, NULL, error))
+ if (!(ret_is_stored || ret_is_pending))
+ {
+ if (!find_object_in_remote_packs (pull_data, checksum, objtype,
+ &ret_remote_pack_checksum, &offset,
+ cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ if (out_is_stored)
+ *out_is_stored = ret_is_stored;
+ if (out_is_pending)
+ *out_is_pending = ret_is_pending;
+ ot_transfer_out_value (out_remote_pack_checksum, &ret_remote_pack_checksum);
+ if (out_offset)
+ *out_offset = offset;
+ out:
+ g_free (local_pack_checksum);
+ g_free (ret_remote_pack_checksum);
+ g_clear_object (&stored_path);
+ return ret;
+}
+
+static void
+unlink_file_on_unref (GFile *f)
+{
+ (void) ot_gfile_unlink (f, NULL, NULL);
+ g_object_unref (f);
+}
+
+static gboolean
+fetch_object_if_not_stored (OtPullData *pull_data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ gboolean *out_is_stored,
+ gboolean *out_is_pending,
+ GInputStream **out_input,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gboolean ret_is_stored = FALSE;
+ gboolean ret_is_pending = FALSE;
+ GInputStream *ret_input = NULL;
+ GFile *temp_path = NULL;
+ GFile *pack_path = NULL;
+ GMappedFile *pack_map = NULL;
+ char *remote_pack_checksum = NULL;
+ guint64 pack_offset = 0;
+ GVariant *pack_entry = NULL;
+
+ if (!find_object (pull_data, checksum, objtype, &ret_is_stored,
+ &ret_is_pending, &remote_pack_checksum,
+ &pack_offset, cancellable, error))
goto out;
- if (!(stored_path || pending_path))
+ if (remote_pack_checksum != NULL)
{
- if (!fetch_object (repo, soup, baseuri, checksum, objtype, &temp_path, cancellable, error))
+ g_assert (!(ret_is_stored || ret_is_pending));
+
+ if (!fetch_one_pack_file (pull_data, remote_pack_checksum, &pack_path,
+ cancellable, error))
+ goto out;
+
+ pack_map = g_mapped_file_new (ot_gfile_get_path_cached (pack_path), FALSE, error);
+ if (!pack_map)
+ goto out;
+
+ if (!ostree_read_pack_entry_raw ((guchar*)g_mapped_file_get_contents (pack_map),
+ g_mapped_file_get_length (pack_map),
+ pack_offset, FALSE, &pack_entry,
+ cancellable, error))
goto out;
- }
- if (temp_path)
+ ret_input = ostree_read_pack_entry_as_stream (pack_entry);
+ g_object_set_data_full ((GObject*)ret_input, "ostree-pull-pack-map",
+ pack_map, (GDestroyNotify) g_mapped_file_unref);
+ pack_map = NULL; /* Transfer ownership */
+ }
+ else if (!(ret_is_stored || ret_is_pending))
{
- file_info = g_file_query_info (temp_path, OSTREE_GIO_FAST_QUERYINFO,
- G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error);
- if (!file_info)
+ if (!fetch_loose_object (pull_data, checksum, objtype, &temp_path, cancellable, error))
goto out;
- input = (GInputStream*)g_file_read (temp_path, cancellable, error);
- if (!input)
+ ret_input = (GInputStream*)g_file_read (temp_path, cancellable, error);
+ if (!ret_input)
goto out;
+ g_object_set_data_full ((GObject*)ret_input, "ostree-tmpfile-unlink",
+ g_object_ref (temp_path),
+ (GDestroyNotify)unlink_file_on_unref);
}
-
- if (pending_path || temp_path)
+
+ ret = TRUE;
+ ot_transfer_out_value (out_input, &ret_input);
+ if (out_is_stored)
+ *out_is_stored = ret_is_stored;
+ if (out_is_pending)
+ *out_is_pending = ret_is_pending;
+ out:
+ g_clear_object (&temp_path);
+ g_clear_object (&pack_path);
+ if (pack_map)
+ g_mapped_file_unref (pack_map);
+ ot_clear_gvariant (&pack_entry);
+ g_clear_object (&pack_path);
+ g_clear_object (&ret_input);
+ return ret;
+}
+
+static gboolean
+fetch_and_store_object (OtPullData *pull_data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ gboolean *out_was_stored,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFileInfo *file_info = NULL;
+ GInputStream *input = NULL;
+ GFile *stored_path = NULL;
+ GFile *pending_path = NULL;
+ char *pack_checksum = NULL;
+ gboolean is_stored;
+ gboolean is_pending;
+
+ g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE);
+
+ if (!fetch_object_if_not_stored (pull_data, checksum, objtype,
+ &is_stored, &is_pending, &input,
+ cancellable, error))
+ goto out;
+
+ if (is_pending || input)
{
- if (!ostree_repo_stage_object (repo, objtype, checksum, file_info, NULL, input, cancellable, error))
+ if (!ostree_repo_stage_object (pull_data->repo, objtype, checksum, NULL, NULL,
+ input, cancellable, error))
goto out;
log_verbose ("Staged object: %s.%s", checksum, ostree_object_type_to_string (objtype));
+ }
+
+ ret = TRUE;
+ if (out_was_stored)
+ *out_was_stored = is_stored;
+ out:
+ g_clear_object (&file_info);
+ g_clear_object (&input);
+ g_clear_object (&stored_path);
+ g_clear_object (&pending_path);
+ g_free (pack_checksum);
+ return ret;
+}
+
+static gboolean
+fetch_and_store_metadata (OtPullData *pull_data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ gboolean *out_was_stored,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gboolean ret_was_stored;
+ GVariant *ret_variant = NULL;
+
+ if (!fetch_and_store_object (pull_data, checksum, objtype,
+ &ret_was_stored, cancellable, error))
+ goto out;
- ret_is_pending = TRUE;
- if (out_metadata)
+ if (!ostree_repo_load_variant (pull_data->repo, objtype, checksum,
+ &ret_variant, error))
+ goto out;
+
+ ret = TRUE;
+ ot_transfer_out_value (out_variant, &ret_variant);
+ if (out_was_stored)
+ *out_was_stored = ret_was_stored;
+ out:
+ ot_clear_gvariant (&ret_variant);
+ return ret;
+}
+
+static gboolean
+fetch_and_store_file (OtPullData *pull_data,
+ const char *checksum,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GInputStream *input = NULL;
+ GFile *stored_path = NULL;
+ GFile *pending_path = NULL;
+ char *pack_checksum = NULL;
+ GVariant *archive_metadata_container = NULL;
+ GVariant *archive_metadata = NULL;
+ GFileInfo *archive_file_info = NULL;
+ GVariant *archive_xattrs = NULL;
+ gboolean skip_archive_fetch;
+
+ /* If we're fetching from an archive into a bare repository, we need
+ * to explicitly check for raw file types locally.
+ */
+ if (ostree_repo_get_mode (pull_data->repo) == OSTREE_REPO_MODE_BARE)
+ {
+ if (!ostree_repo_find_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE,
+ checksum, &stored_path, &pending_path, &pack_checksum,
+ NULL, cancellable, error))
+ goto out;
+
+ if (stored_path || pack_checksum)
+ skip_archive_fetch = TRUE;
+ else if (pending_path != NULL)
{
- if (!ostree_map_metadata_file (pending_path ? pending_path : temp_path, objtype, &ret_metadata, error))
+ skip_archive_fetch = TRUE;
+ if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE,
+ checksum, NULL, NULL, NULL, cancellable, error))
goto out;
}
+ else
+ skip_archive_fetch = FALSE;
+
+ g_clear_object (&stored_path);
}
else
{
- ret_is_pending = FALSE;
+ skip_archive_fetch = FALSE;
}
+ if (!skip_archive_fetch)
+ {
+ if (!fetch_object_if_not_stored (pull_data, checksum,
+ OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
+ NULL, NULL, &input, cancellable, error))
+ goto out;
+
+ if (input != NULL)
+ {
+ if (!ot_util_variant_from_stream (input, OSTREE_SERIALIZED_VARIANT_FORMAT,
+ FALSE, &archive_metadata_container, cancellable, error))
+ goto out;
+
+ if (!ostree_unwrap_metadata (archive_metadata_container, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
+ &archive_metadata, error))
+ goto out;
+
+ if (!ostree_parse_archived_file_meta (archive_metadata, &archive_file_info,
+ &archive_xattrs, error))
+ goto out;
+
+ g_clear_object (&input);
+ if (g_file_info_get_file_type (archive_file_info) == G_FILE_TYPE_REGULAR)
+ {
+ if (!fetch_object_if_not_stored (pull_data, checksum,
+ OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
+ NULL, NULL, &input,
+ cancellable, error))
+ goto out;
+ }
+
+ if (!ostree_repo_stage_object (pull_data->repo, OSTREE_OBJECT_TYPE_RAW_FILE, checksum,
+ archive_file_info, archive_xattrs, input,
+ cancellable, error))
+ goto out;
+ }
+ }
+
ret = TRUE;
- if (out_is_pending)
- *out_is_pending = ret_is_pending;
- ot_transfer_out_value (out_metadata, &ret_metadata);
out:
- if (temp_path)
- (void) unlink (ot_gfile_get_path_cached (temp_path));
- ot_clear_gvariant (&ret_metadata);
- g_clear_object (&temp_path);
- g_clear_object (&file_info);
- g_clear_object (&input);
+ g_free (pack_checksum);
g_clear_object (&stored_path);
g_clear_object (&pending_path);
+ g_clear_object (&input);
+ ot_clear_gvariant (&archive_metadata_container);
+ ot_clear_gvariant (&archive_metadata);
+ ot_clear_gvariant (&archive_xattrs);
+ g_clear_object (&archive_file_info);
return ret;
}
static gboolean
-fetch_and_store_tree_recurse (OstreeRepo *repo,
- SoupSession *soup,
- SoupURI *base_uri,
+fetch_and_store_tree_recurse (OtPullData *pull_data,
const char *rev,
GCancellable *cancellable,
GError **error)
@@ -301,22 +756,17 @@ fetch_and_store_tree_recurse (OstreeRepo *repo,
GVariant *tree = NULL;
GVariant *files_variant = NULL;
GVariant *dirs_variant = NULL;
- gboolean is_pending;
+ gboolean was_stored;
int i, n;
- GVariant *archive_metadata = NULL;
- GFileInfo *archive_file_info = NULL;
- GVariant *archive_xattrs = NULL;
- GFile *meta_temp_path = NULL;
- GFile *content_temp_path = NULL;
GFile *stored_path = NULL;
GFile *pending_path = NULL;
- GInputStream *input = NULL;
+ char *pack_checksum = NULL;
- if (!fetch_and_store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_DIR_TREE,
- &is_pending, &tree, cancellable, error))
+ if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_DIR_TREE,
+ &was_stored, &tree, cancellable, error))
goto out;
- if (!is_pending)
+ if (was_stored)
log_verbose ("Already have tree %s", rev);
else
{
@@ -337,81 +787,8 @@ fetch_and_store_tree_recurse (OstreeRepo *repo,
if (!ostree_validate_checksum_string (checksum, error))
goto out;
- g_clear_object (&stored_path);
- g_clear_object (&pending_path);
- /* If we're fetching from an archive into a bare repository, we need
- * to explicitly check for raw file types locally.
- */
- if (ostree_repo_get_mode (repo) == OSTREE_REPO_MODE_BARE)
- {
- if (!ostree_repo_find_object (repo, OSTREE_OBJECT_TYPE_RAW_FILE, checksum,
- &stored_path, &pending_path, cancellable, error))
- goto out;
- }
- else
- {
- if (!ostree_repo_find_object (repo, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT, checksum,
- &stored_path, &pending_path, cancellable, error))
- goto out;
- }
-
- g_clear_object (&input);
- g_clear_object (&archive_file_info);
- ot_clear_gvariant (&archive_xattrs);
- if (!(stored_path || pending_path))
- {
- g_clear_object (&meta_temp_path);
- if (!fetch_object (repo, soup, base_uri, checksum,
- OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
- &meta_temp_path,
- cancellable,
- error))
- goto out;
-
- if (!ostree_map_metadata_file (meta_temp_path,
- OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META,
- &archive_metadata, error))
- goto out;
-
- if (!ostree_parse_archived_file_meta (archive_metadata, &archive_file_info, &archive_xattrs, error))
- goto out;
-
- if (g_file_info_get_file_type (archive_file_info) == G_FILE_TYPE_REGULAR)
- {
- if (!fetch_object (repo, soup, base_uri, checksum,
- OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT,
- &content_temp_path,
- cancellable,
- error))
- goto out;
-
- input = (GInputStream*)g_file_read (content_temp_path, cancellable, error);
- if (!input)
- goto out;
- }
- }
-
- if (!stored_path)
- {
- log_verbose ("Staged file object: %s", checksum);
-
- if (!ostree_repo_stage_object (repo, OSTREE_OBJECT_TYPE_RAW_FILE,
- checksum,
- archive_file_info, archive_xattrs, input,
- cancellable, error))
- goto out;
- }
-
- if (meta_temp_path)
- {
- (void) unlink (ot_gfile_get_path_cached (meta_temp_path));
- g_clear_object (&meta_temp_path);
- }
- if (content_temp_path)
- {
- (void) unlink (ot_gfile_get_path_cached (content_temp_path));
- g_clear_object (&content_temp_path);
- }
+ if (!fetch_and_store_file (pull_data, checksum, cancellable, error))
+ goto out;
}
n = g_variant_n_children (dirs_variant);
@@ -431,11 +808,11 @@ fetch_and_store_tree_recurse (OstreeRepo *repo,
if (!ostree_validate_checksum_string (meta_checksum, error))
goto out;
- if (!fetch_and_store_object (repo, soup, base_uri, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META,
- NULL, NULL, cancellable, error))
+ if (!fetch_and_store_object (pull_data, meta_checksum, OSTREE_OBJECT_TYPE_DIR_META,
+ NULL, cancellable, error))
goto out;
- if (!fetch_and_store_tree_recurse (repo, soup, base_uri, tree_checksum, cancellable, error))
+ if (!fetch_and_store_tree_recurse (pull_data, tree_checksum, cancellable, error))
goto out;
}
}
@@ -445,29 +822,14 @@ fetch_and_store_tree_recurse (OstreeRepo *repo,
ot_clear_gvariant (&tree);
ot_clear_gvariant (&files_variant);
ot_clear_gvariant (&dirs_variant);
- ot_clear_gvariant (&archive_metadata);
- ot_clear_gvariant (&archive_xattrs);
- g_clear_object (&archive_file_info);
- g_clear_object (&input);
g_clear_object (&stored_path);
g_clear_object (&pending_path);
- if (content_temp_path)
- {
- (void) unlink (ot_gfile_get_path_cached (content_temp_path));
- g_clear_object (&content_temp_path);
- }
- if (meta_temp_path)
- {
- (void) unlink (ot_gfile_get_path_cached (meta_temp_path));
- g_clear_object (&meta_temp_path);
- }
+ g_free (pack_checksum);
return ret;
}
static gboolean
-fetch_and_store_commit_recurse (OstreeRepo *repo,
- SoupSession *soup,
- SoupURI *base_uri,
+fetch_and_store_commit_recurse (OtPullData *pull_data,
const char *rev,
GCancellable *cancellable,
GError **error)
@@ -476,13 +838,13 @@ fetch_and_store_commit_recurse (OstreeRepo *repo,
GVariant *commit = NULL;
const char *tree_contents_checksum;
const char *tree_meta_checksum;
- gboolean is_pending;
+ gboolean was_stored;
- if (!fetch_and_store_object (repo, soup, base_uri, rev, OSTREE_OBJECT_TYPE_COMMIT,
- &is_pending, &commit, cancellable, error))
+ if (!fetch_and_store_metadata (pull_data, rev, OSTREE_OBJECT_TYPE_COMMIT,
+ &was_stored, &commit, cancellable, error))
goto out;
- if (!is_pending)
+ if (was_stored)
log_verbose ("Already have commit %s", rev);
else
{
@@ -490,11 +852,11 @@ fetch_and_store_commit_recurse (OstreeRepo *repo,
g_variant_get_child (commit, 6, "&s", &tree_contents_checksum);
g_variant_get_child (commit, 7, "&s", &tree_meta_checksum);
- if (!fetch_and_store_object (repo, soup, base_uri, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META,
- NULL, NULL, cancellable, error))
+ if (!fetch_and_store_object (pull_data, tree_meta_checksum, OSTREE_OBJECT_TYPE_DIR_META,
+ NULL, cancellable, error))
goto out;
- if (!fetch_and_store_tree_recurse (repo, soup, base_uri, tree_contents_checksum,
+ if (!fetch_and_store_tree_recurse (pull_data, tree_contents_checksum,
cancellable, error))
goto out;
}
@@ -506,9 +868,7 @@ fetch_and_store_commit_recurse (OstreeRepo *repo,
}
static gboolean
-fetch_ref_contents (OstreeRepo *repo,
- SoupSession *soup,
- SoupURI *base_uri,
+fetch_ref_contents (OtPullData *pull_data,
const char *ref,
char **out_contents,
GCancellable *cancellable,
@@ -516,14 +876,11 @@ fetch_ref_contents (OstreeRepo *repo,
{
gboolean ret = FALSE;
char *ret_contents = NULL;
- char *refpath = NULL;
SoupURI *target_uri = NULL;
- target_uri = soup_uri_copy (base_uri);
- refpath = g_build_filename (soup_uri_get_path (target_uri), "refs", "heads", ref, NULL);
- soup_uri_set_path (target_uri, refpath);
+ target_uri = suburi_new (pull_data->base_uri, "refs", "heads", ref, NULL);
- if (!fetch_uri_contents_utf8 (repo, soup, target_uri, &ret_contents, cancellable, error))
+ if (!fetch_uri_contents_utf8 (pull_data, target_uri, &ret_contents, cancellable, error))
goto out;
g_strchomp (ret_contents);
@@ -534,7 +891,6 @@ fetch_ref_contents (OstreeRepo *repo,
ret = TRUE;
ot_transfer_out_value (out_contents, &ret_contents);
out:
- g_free (refpath);
g_free (ret_contents);
if (target_uri)
soup_uri_free (target_uri);
@@ -542,12 +898,9 @@ fetch_ref_contents (OstreeRepo *repo,
}
static gboolean
-pull_one_commit (OstreeRepo *repo,
- const char *remote,
+pull_one_commit (OtPullData *pull_data,
const char *branch,
const char *rev,
- SoupSession *soup,
- SoupURI *base_uri,
GCancellable *cancellable,
GError **error)
{
@@ -557,9 +910,9 @@ pull_one_commit (OstreeRepo *repo,
char *baseurl = NULL;
char *original_rev = NULL;
- remote_ref = g_strdup_printf ("%s/%s", remote, branch);
+ remote_ref = g_strdup_printf ("%s/%s", pull_data->remote_name, branch);
- if (!ostree_repo_resolve_rev (repo, remote_ref, TRUE, &original_rev, error))
+ if (!ostree_repo_resolve_rev (pull_data->repo, remote_ref, TRUE, &original_rev, error))
goto out;
if (original_rev && strcmp (rev, original_rev) == 0)
@@ -571,16 +924,27 @@ pull_one_commit (OstreeRepo *repo,
if (!ostree_validate_checksum_string (rev, error))
goto out;
- if (!ostree_repo_prepare_transaction (repo, NULL, error))
+ if (!pull_data->fetched_packs)
+ {
+ pull_data->fetched_packs = TRUE;
+ pull_data->cached_pack_indexes = g_ptr_array_new_with_free_func (g_free);
+
+ g_print ("Fetching packs\n");
+
+ if (!fetch_and_cache_pack_indexes (pull_data, cancellable, error))
+ goto out;
+ }
+
+ if (!ostree_repo_prepare_transaction (pull_data->repo, NULL, error))
goto out;
- if (!fetch_and_store_commit_recurse (repo, soup, base_uri, rev, cancellable, error))
+ if (!fetch_and_store_commit_recurse (pull_data, rev, cancellable, error))
goto out;
- if (!ostree_repo_commit_transaction (repo, cancellable, error))
+ if (!ostree_repo_commit_transaction (pull_data->repo, cancellable, error))
goto out;
- if (!ostree_repo_write_ref (repo, remote, branch, rev, error))
+ if (!ostree_repo_write_ref (pull_data->repo, pull_data->remote_name, branch, rev, error))
goto out;
g_print ("remote %s is now %s\n", remote_ref, rev);
@@ -656,9 +1020,9 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
{
GOptionContext *context;
gboolean ret = FALSE;
+ OtPullData pull_data_real;
+ OtPullData *pull_data = &pull_data_real;
OstreeRepo *repo = NULL;
- const char *remote;
- SoupSession *soup = NULL;
char *path = NULL;
char *baseurl = NULL;
char *summary_data = NULL;
@@ -682,27 +1046,29 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
if (!ostree_repo_check (repo, error))
goto out;
+ memset (pull_data, 0, sizeof (*pull_data));
+ pull_data->repo = repo;
+
if (argc < 2)
{
ot_util_usage_error (context, "REMOTE must be specified", error);
goto out;
}
- remote = argv[1];
-
- soup = soup_session_sync_new_with_options (SOUP_SESSION_USER_AGENT, "ostree ",
- SOUP_SESSION_ADD_FEATURE_BY_TYPE, SOUP_TYPE_COOKIE_JAR,
- NULL);
+ pull_data->remote_name = g_strdup (argv[1]);
+ pull_data->session = soup_session_sync_new_with_options (SOUP_SESSION_USER_AGENT, "ostree ",
+ SOUP_SESSION_ADD_FEATURE_BY_TYPE, SOUP_TYPE_COOKIE_JAR,
+ NULL);
config = ostree_repo_get_config (repo);
- key = g_strdup_printf ("remote \"%s\"", remote);
+ key = g_strdup_printf ("remote \"%s\"", pull_data->remote_name);
baseurl = g_key_file_get_string (config, key, "url", error);
if (!baseurl)
goto out;
- base_uri = soup_uri_new (baseurl);
+ pull_data->base_uri = soup_uri_new (baseurl);
- if (!base_uri)
+ if (!pull_data->base_uri)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse url '%s'", baseurl);
@@ -717,7 +1083,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
const char *branch = argv[i];
char *contents;
- if (!fetch_ref_contents (repo, soup, base_uri, branch, &contents, cancellable, error))
+ if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
goto out;
/* Transfer ownership of contents */
@@ -730,7 +1096,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
path = g_build_filename (soup_uri_get_path (summary_uri), "refs", "summary", NULL);
soup_uri_set_path (summary_uri, path);
- if (!fetch_uri_contents_utf8 (repo, soup, summary_uri, &summary_data, cancellable, error))
+ if (!fetch_uri_contents_utf8 (pull_data, summary_uri, &summary_data, cancellable, error))
goto out;
if (!parse_ref_summary (summary_data, &refs_to_fetch, error))
@@ -744,7 +1110,7 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
const char *ref = key;
const char *sha256 = value;
- if (!pull_one_commit (repo, remote, ref, sha256, soup, base_uri, cancellable, error))
+ if (!pull_one_commit (pull_data, ref, sha256, cancellable, error))
goto out;
}
@@ -758,13 +1124,14 @@ ostree_builtin_pull (int argc, char **argv, GFile *repo_path, GError **error)
g_free (branch_rev);
if (context)
g_option_context_free (context);
- g_clear_object (&soup);
- if (base_uri)
- soup_uri_free (base_uri);
+ g_clear_object (&pull_data->session);
+ if (pull_data->base_uri)
+ soup_uri_free (pull_data->base_uri);
+ if (pull_data->cached_pack_indexes)
+ g_ptr_array_unref (pull_data->cached_pack_indexes);
if (summary_uri)
soup_uri_free (summary_uri);
g_clear_object (&repo);
- g_clear_object (&soup);
return ret;
}
diff --git a/src/ostree/ot-builtin-fsck.c b/src/ostree/ot-builtin-fsck.c
index 9671ffa..f4f567a 100644
--- a/src/ostree/ot-builtin-fsck.c
+++ b/src/ostree/ot-builtin-fsck.c
@@ -26,6 +26,7 @@
#include "ostree.h"
#include <glib/gi18n.h>
+#include <glib/gprintf.h>
static gboolean quiet;
static gboolean delete;
@@ -38,8 +39,8 @@ static GOptionEntry options[] = {
typedef struct {
OstreeRepo *repo;
- guint n_objects;
- gboolean had_error;
+ guint n_loose_objects;
+ guint n_pack_files;
} OtFsckData;
static gboolean
@@ -123,62 +124,157 @@ checksum_archived_file (OtFsckData *data,
return ret;
}
-static void
-object_iter_callback (OstreeRepo *repo,
- const char *exp_checksum,
- OstreeObjectType objtype,
- GFile *objf,
- GFileInfo *file_info,
- gpointer user_data)
+static gboolean
+fsck_loose_object (OtFsckData *data,
+ const char *exp_checksum,
+ OstreeObjectType objtype,
+ GCancellable *cancellable,
+ GError **error)
{
- OtFsckData *data = user_data;
+ gboolean ret = FALSE;
+ GFile *objf = NULL;
GChecksum *real_checksum = NULL;
- GError *error = NULL;
- /* nlinks = g_file_info_get_attribute_uint32 (file_info, "unix::nlink");
- if (nlinks < 2 && !quiet)
- g_printerr ("note: floating object: %s\n", path); */
+ objf = ostree_repo_get_object_path (data->repo, exp_checksum, objtype);
if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META)
{
if (!g_str_has_suffix (ot_gfile_get_path_cached (objf), ".archive-meta"))
{
- g_set_error (&error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid archive filename '%s'",
ot_gfile_get_path_cached (objf));
goto out;
}
- if (!checksum_archived_file (data, exp_checksum, objf, &real_checksum, &error))
+ if (!checksum_archived_file (data, exp_checksum, objf, &real_checksum, error))
goto out;
}
else if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT)
; /* Handled above */
else
{
- if (!ostree_checksum_file (objf, objtype, &real_checksum, NULL, &error))
+ if (!ostree_checksum_file (objf, objtype, &real_checksum, NULL, error))
goto out;
}
if (real_checksum && strcmp (exp_checksum, g_checksum_get_string (real_checksum)) != 0)
{
- data->had_error = TRUE;
- g_printerr ("ERROR: corrupted object '%s'; actual checksum: %s\n",
- ot_gfile_get_path_cached (objf), g_checksum_get_string (real_checksum));
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "corrupted loose object '%s'; actual checksum: %s",
+ ot_gfile_get_path_cached (objf), g_checksum_get_string (real_checksum));
if (delete)
(void) unlink (ot_gfile_get_path_cached (objf));
+ goto out;
}
- data->n_objects++;
+ data->n_loose_objects++;
+ ret = TRUE;
out:
ot_clear_checksum (&real_checksum);
- if (error != NULL)
+ return ret;
+}
+
+static gboolean
+fsck_pack_files (OtFsckData *data,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GPtrArray *pack_indexes = NULL;
+ GVariant *index_variant = NULL;
+ GFile *pack_index_path = NULL;
+ GFile *pack_data_path = NULL;
+ GFileInfo *pack_info = NULL;
+ GInputStream *input = NULL;
+ GChecksum *pack_content_checksum = NULL;
+ GVariantIter *index_content_iter = NULL;
+ guint i;
+ guint32 objtype;
+ guint64 offset;
+ guint64 pack_size;
+
+ if (!ostree_repo_list_pack_indexes (data->repo, &pack_indexes, cancellable, error))
+ goto out;
+
+ for (i = 0; i < pack_indexes->len; i++)
{
- g_printerr ("%s\n", error->message);
- g_clear_error (&error);
+ const char *checksum = pack_indexes->pdata[i];
+
+ g_clear_object (&pack_index_path);
+ pack_index_path = ostree_repo_get_pack_index_path (data->repo, checksum);
+
+ ot_clear_gvariant (&index_variant);
+ if (!ot_util_variant_map (pack_index_path,
+ OSTREE_PACK_INDEX_VARIANT_FORMAT,
+ &index_variant, error))
+ goto out;
+
+ if (!ostree_validate_structureof_pack_index (index_variant, error))
+ goto out;
+
+ g_clear_object (&pack_data_path);
+ pack_data_path = ostree_repo_get_pack_data_path (data->repo, checksum);
+
+ g_clear_object (&input);
+ input = (GInputStream*)g_file_read (pack_data_path, cancellable, error);
+ if (!input)
+ goto out;
+
+ g_clear_object (&pack_info);
+ pack_info = g_file_input_stream_query_info ((GFileInputStream*)input, OSTREE_GIO_FAST_QUERYINFO,
+ cancellable, error);
+ if (!pack_info)
+ goto out;
+ pack_size = g_file_info_get_attribute_uint64 (pack_info, "standard::size");
+
+ if (pack_content_checksum)
+ g_checksum_free (pack_content_checksum);
+ if (!ot_gio_checksum_stream (input, &pack_content_checksum, cancellable, error))
+ goto out;
+
+ if (strcmp (g_checksum_get_string (pack_content_checksum), checksum) != 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "corrupted pack '%s', expected checksum %s",
+ checksum, g_checksum_get_string (pack_content_checksum));
+ goto out;
+ }
+
+ g_variant_get_child (index_variant, 2, "a(uayt)", &index_content_iter);
+
+ while (g_variant_iter_loop (index_content_iter, "(u ayt)",
+ &objtype, NULL, &offset))
+ {
+ offset = GUINT64_FROM_BE (offset);
+ if (offset > pack_size)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "corrupted pack '%s', offset %" G_GUINT64_FORMAT " larger than file size %" G_GUINT64_FORMAT,
+ checksum,
+ offset, pack_size);
+ goto out;
+ }
+ }
+
+ data->n_pack_files++;
}
+
+ ret = TRUE;
+ out:
+ if (index_content_iter)
+ g_variant_iter_free (index_content_iter);
+ if (pack_content_checksum)
+ g_checksum_free (pack_content_checksum);
+ if (pack_indexes)
+ g_ptr_array_unref (pack_indexes);
+ g_clear_object (&pack_info);
+ g_clear_object (&pack_data_path);
+ g_clear_object (&input);
+ return ret;
}
+
gboolean
ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error)
{
@@ -186,6 +282,10 @@ ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error)
OtFsckData data;
gboolean ret = FALSE;
OstreeRepo *repo = NULL;
+ GHashTable *objects = NULL;
+ GCancellable *cancellable = NULL;
+ GHashTableIter hash_iter;
+ gpointer key, value;
context = g_option_context_new ("- Check the repository for consistency");
g_option_context_add_main_entries (context, options, NULL);
@@ -197,26 +297,47 @@ ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error)
if (!ostree_repo_check (repo, error))
goto out;
+ memset (&data, 0, sizeof (data));
data.repo = repo;
- data.n_objects = 0;
- data.had_error = FALSE;
- if (!ostree_repo_iter_objects (repo, object_iter_callback, &data, error))
+ if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL,
+ &objects, cancellable, error))
goto out;
+
+ g_hash_table_iter_init (&hash_iter, objects);
- if (data.had_error)
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Encountered filesystem consistency errors");
- goto out;
+ GVariant *serialized_key = key;
+ GVariant *objdata = value;
+ const char *checksum;
+ OstreeObjectType objtype;
+ gboolean is_loose;
+
+ ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+
+ g_variant_get_child (objdata, 0, "b", &is_loose);
+
+ if (is_loose)
+ {
+ if (!fsck_loose_object (&data, checksum, objtype, cancellable, error))
+ goto out;
+ }
}
+
+ if (!fsck_pack_files (&data, cancellable, error))
+ goto out;
+
if (!quiet)
- g_printerr ("Total Objects: %u\n", data.n_objects);
+ g_print ("Loose Objects: %u\n", data.n_loose_objects);
+ g_print ("Pack files: %u\n", data.n_pack_files);
ret = TRUE;
out:
if (context)
g_option_context_free (context);
g_clear_object (&repo);
+ if (objects)
+ g_hash_table_unref (objects);
return ret;
}
diff --git a/src/ostree/ot-builtin-init.c b/src/ostree/ot-builtin-init.c
index 935bae7..5d648d8 100644
--- a/src/ostree/ot-builtin-init.c
+++ b/src/ostree/ot-builtin-init.c
@@ -45,7 +45,9 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error)
gboolean ret = FALSE;
GFile *child = NULL;
GFile *grandchild = NULL;
+ GCancellable *cancellable = NULL;
GString *config_data = NULL;
+ OstreeRepo *repo = NULL;
context = g_option_context_new ("- Initialize a new empty repository");
g_option_context_add_main_entries (context, options, NULL);
@@ -63,38 +65,53 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error)
NULL, FALSE, 0, NULL,
NULL, error))
goto out;
- g_clear_object (&child);
+ g_clear_object (&child);
child = g_file_get_child (repo_path, "objects");
if (!g_file_make_directory (child, NULL, error))
goto out;
- g_clear_object (&child);
+ g_clear_object (&grandchild);
+ grandchild = g_file_get_child (child, "pack");
+ if (!g_file_make_directory (grandchild, NULL, error))
+ goto out;
+
+ g_clear_object (&child);
child = g_file_get_child (repo_path, "tmp");
if (!g_file_make_directory (child, NULL, error))
goto out;
- g_clear_object (&child);
+ g_clear_object (&child);
child = g_file_get_child (repo_path, "refs");
if (!g_file_make_directory (child, NULL, error))
goto out;
+ g_clear_object (&grandchild);
grandchild = g_file_get_child (child, "heads");
if (!g_file_make_directory (grandchild, NULL, error))
goto out;
- g_clear_object (&grandchild);
+ g_clear_object (&grandchild);
grandchild = g_file_get_child (child, "remotes");
if (!g_file_make_directory (grandchild, NULL, error))
goto out;
- g_clear_object (&grandchild);
g_clear_object (&child);
-
child = g_file_get_child (repo_path, "tags");
if (!g_file_make_directory (child, NULL, error))
goto out;
+
g_clear_object (&child);
+ child = g_file_get_child (repo_path, "remote-cache");
+ if (!g_file_make_directory (child, NULL, error))
+ goto out;
+
+ repo = ostree_repo_new (repo_path);
+ if (!ostree_repo_check (repo, error))
+ goto out;
+
+ if (!ostree_repo_regenerate_pack_index (repo, cancellable, error))
+ goto out;
ret = TRUE;
out:
@@ -104,5 +121,6 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error)
g_string_free (config_data, TRUE);
g_clear_object (&child);
g_clear_object (&grandchild);
+ g_clear_object (&repo);
return ret;
}
diff --git a/src/ostree/ot-builtin-local-clone.c b/src/ostree/ot-builtin-local-clone.c
index 87b0151..1c9a3d0 100644
--- a/src/ostree/ot-builtin-local-clone.c
+++ b/src/ostree/ot-builtin-local-clone.c
@@ -97,23 +97,29 @@ copy_dir_contents_recurse (GFile *src,
return ret;
}
-static void
-object_iter_callback (OstreeRepo *repo,
- const char *checksum,
- OstreeObjectType objtype,
- GFile *objfile,
- GFileInfo *file_info,
- gpointer user_data)
+static gboolean
+import_loose_object (OtLocalCloneData *data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ GCancellable *cancellable,
+ GError **error)
{
- OtLocalCloneData *data = user_data;
- GError *real_error = NULL;
- GError **error = &real_error;
+ gboolean ret = FALSE;
+ GFile *objfile = NULL;
+ GFileInfo *file_info = NULL;
GFile *content_path = NULL;
GFileInfo *archive_info = NULL;
GVariant *archive_metadata = NULL;
GVariant *xattrs = NULL;
GInputStream *input = NULL;
+ objfile = ostree_repo_get_object_path (data->src_repo, checksum, objtype);
+ file_info = g_file_query_info (objfile, OSTREE_GIO_FAST_QUERYINFO,
+ G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, cancellable, error);
+
+ if (file_info == NULL)
+ goto out;
+
if (objtype == OSTREE_OBJECT_TYPE_RAW_FILE)
xattrs = ostree_get_xattrs_for_file (objfile, error);
@@ -121,13 +127,13 @@ object_iter_callback (OstreeRepo *repo,
;
else if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META)
{
- if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, checksum, &archive_metadata, error))
+ if (!ostree_repo_load_variant (data->src_repo, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META, checksum, &archive_metadata, error))
goto out;
if (!ostree_parse_archived_file_meta (archive_metadata, &archive_info, &xattrs, error))
goto out;
- content_path = ostree_repo_get_object_path (repo, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
+ content_path = ostree_repo_get_object_path (data->src_repo, checksum, OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
if (g_file_info_get_file_type (archive_info) == G_FILE_TYPE_REGULAR)
{
@@ -136,8 +142,8 @@ object_iter_callback (OstreeRepo *repo,
goto out;
}
- if (!ostree_repo_stage_object_trusted (data->dest_repo, OSTREE_OBJECT_TYPE_RAW_FILE, checksum,
- archive_info, xattrs, input,
+ if (!ostree_repo_stage_object_trusted (data->dest_repo, OSTREE_OBJECT_TYPE_RAW_FILE,
+ checksum, FALSE, archive_info, xattrs, input,
NULL, error))
goto out;
}
@@ -151,23 +157,21 @@ object_iter_callback (OstreeRepo *repo,
}
if (!ostree_repo_stage_object_trusted (data->dest_repo, objtype, checksum,
- file_info, xattrs, input,
+ FALSE, file_info, xattrs, input,
NULL, error))
goto out;
}
+ ret = TRUE;
out:
ot_clear_gvariant (&archive_metadata);
ot_clear_gvariant (&xattrs);
g_clear_object (&archive_info);
g_clear_object (&input);
g_clear_object (&content_path);
- if (real_error != NULL)
- {
- g_printerr ("%s\n", real_error->message);
- g_clear_error (error);
- exit (1);
- }
+ g_clear_object (&file_info);
+ g_clear_object (&objfile);
+ return ret;
}
static gboolean
@@ -209,6 +213,7 @@ ostree_builtin_local_clone (int argc, char **argv, GFile *repo_path, GError **er
{
gboolean ret = FALSE;
GCancellable *cancellable = NULL;
+ GHashTable *objects = NULL;
GOptionContext *context;
const char *destination;
GFile *dest_f = NULL;
@@ -220,6 +225,8 @@ ostree_builtin_local_clone (int argc, char **argv, GFile *repo_path, GError **er
GFile *src_dir = NULL;
GFile *dest_dir = NULL;
int i;
+ GHashTableIter hash_iter;
+ gpointer key, value;
context = g_option_context_new ("DEST ... - Create new repository DEST");
g_option_context_add_main_entries (context, options, NULL);
@@ -266,11 +273,33 @@ ostree_builtin_local_clone (int argc, char **argv, GFile *repo_path, GError **er
data.uids_differ = g_file_info_get_attribute_uint32 (src_info, "unix::uid") != g_file_info_get_attribute_uint32 (dest_info, "unix::uid");
- if (!ostree_repo_prepare_transaction (data.dest_repo, NULL, error))
+ if (!ostree_repo_list_objects (data.src_repo, OSTREE_REPO_LIST_OBJECTS_ALL,
+ &objects, cancellable, error))
goto out;
- if (!ostree_repo_iter_objects (data.src_repo, object_iter_callback, &data, error))
+ if (!ostree_repo_prepare_transaction (data.dest_repo, NULL, error))
goto out;
+
+ g_hash_table_iter_init (&hash_iter, objects);
+
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ GVariant *serialized_key = key;
+ GVariant *objdata = value;
+ const char *checksum;
+ OstreeObjectType objtype;
+ gboolean is_loose;
+
+ ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+
+ g_variant_get_child (objdata, 0, "b", &is_loose);
+
+ if (is_loose)
+ {
+ if (!import_loose_object (&data, checksum, objtype, cancellable, error))
+ goto out;
+ }
+ }
if (!ostree_repo_commit_transaction (data.dest_repo, NULL, error))
goto out;
@@ -311,5 +340,7 @@ ostree_builtin_local_clone (int argc, char **argv, GFile *repo_path, GError **er
g_clear_object (&dest_dir);
g_clear_object (&data.src_repo);
g_clear_object (&data.dest_repo);
+ if (objects)
+ g_hash_table_unref (objects);
return ret;
}
diff --git a/src/ostree/ot-builtin-pack.c b/src/ostree/ot-builtin-pack.c
new file mode 100644
index 0000000..cdef0f0
--- /dev/null
+++ b/src/ostree/ot-builtin-pack.c
@@ -0,0 +1,920 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2012 Colin Walters <walters verbum org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ *
+ * Author: Colin Walters <walters verbum org>
+ */
+
+#include "config.h"
+
+#include "ot-builtins.h"
+#include "ostree.h"
+
+#include <glib/gi18n.h>
+#include <glib/gprintf.h>
+
+#include <gio/gunixinputstream.h>
+#include <gio/gunixoutputstream.h>
+
+#define OT_DEFAULT_PACK_SIZE_BYTES (50*1024*1024)
+#define OT_GZIP_COMPRESSION_LEVEL (8)
+
+static gboolean opt_analyze_only;
+static gboolean opt_reindex_only;
+static gboolean opt_keep_loose;
+static char* opt_pack_size;
+static char* opt_int_compression;
+static char* opt_ext_compression;
+
+typedef enum {
+ OT_COMPRESSION_NONE,
+ OT_COMPRESSION_GZIP,
+ OT_COMPRESSION_XZ
+} OtCompressionType;
+
+static GOptionEntry options[] = {
+ { "pack-size", 0, 0, G_OPTION_ARG_STRING, &opt_pack_size, "Maximum uncompressed size of packfiles in bytes; may be suffixed with k, m, or g", "BYTES" },
+ { "internal-compression", 0, 0, G_OPTION_ARG_STRING, &opt_int_compression, "Compress objects using COMPRESSION", "COMPRESSION" },
+ { "external-compression", 0, 0, G_OPTION_ARG_STRING, &opt_ext_compression, "Compress entire packfiles using COMPRESSION", "COMPRESSION" },
+ { "analyze-only", 0, 0, G_OPTION_ARG_NONE, &opt_analyze_only, "Just analyze current state", NULL },
+ { "reindex-only", 0, 0, G_OPTION_ARG_NONE, &opt_reindex_only, "Regenerate pack index", NULL },
+ { "keep-loose", 0, 0, G_OPTION_ARG_NONE, &opt_keep_loose, "Don't delete loose objects", NULL },
+ { NULL }
+};
+
+typedef struct {
+ OstreeRepo *repo;
+
+ guint64 pack_size;
+ OtCompressionType int_compression;
+ OtCompressionType ext_compression;
+
+ gboolean had_error;
+ GError **error;
+} OtRepackData;
+
+typedef struct {
+ GOutputStream *out;
+ GPtrArray *compressor_argv;
+ GPid compress_child_pid;
+} OtBuildRepackFile;
+
+static gint
+compare_object_data_by_size (gconstpointer ap,
+ gconstpointer bp)
+{
+ GVariant *a = *(void **)ap;
+ GVariant *b = *(void **)bp;
+ guint64 a_size;
+ guint64 b_size;
+
+ g_variant_get_child (a, 2, "t", &a_size);
+ g_variant_get_child (b, 2, "t", &b_size);
+ if (a == b)
+ return 0;
+ else if (a > b)
+ return 1;
+ else
+ return -1;
+}
+
+static gboolean
+write_bytes_update_checksum (GOutputStream *output,
+ gconstpointer bytes,
+ gsize len,
+ GChecksum *checksum,
+ guint64 *inout_offset,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gsize bytes_written;
+
+ if (len > 0)
+ {
+ g_checksum_update (checksum, (guchar*) bytes, len);
+ if (!g_output_stream_write_all (output, bytes, len, &bytes_written,
+ cancellable, error))
+ goto out;
+ g_assert_cmpint (bytes_written, ==, len);
+ *inout_offset += bytes_written;
+ }
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+static gboolean
+write_padding (GOutputStream *output,
+ guint alignment,
+ GChecksum *checksum,
+ guint64 *inout_offset,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ guint bits;
+ guint padding_len;
+ guchar padding_nuls[8] = {0, 0, 0, 0, 0, 0, 0, 0};
+
+ if (alignment == 8)
+ bits = ((*inout_offset) & 7);
+ else
+ bits = ((*inout_offset) & 3);
+
+ if (bits > 0)
+ {
+ padding_len = alignment - bits;
+ if (!write_bytes_update_checksum (output, (guchar*)padding_nuls, padding_len,
+ checksum, inout_offset, cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+static gboolean
+write_variant_with_size (GOutputStream *output,
+ GVariant *variant,
+ GChecksum *checksum,
+ guint64 *inout_offset,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ guint64 variant_size;
+ guint32 variant_size_u32_be;
+
+ g_assert ((*inout_offset & 3) == 0);
+
+ /* Write variant size */
+ variant_size = g_variant_get_size (variant);
+ g_assert (variant_size < G_MAXUINT32);
+ variant_size_u32_be = GUINT32_TO_BE((guint32) variant_size);
+
+ if (!write_bytes_update_checksum (output, (guchar*)&variant_size_u32_be, 4,
+ checksum, inout_offset, cancellable, error))
+ goto out;
+
+ /* Pad to offset of 8, write variant */
+ if (!write_padding (output, 8, checksum, inout_offset, cancellable, error))
+ goto out;
+ g_assert ((*inout_offset & 7) == 0);
+
+ if (!write_bytes_update_checksum (output, g_variant_get_data (variant),
+ variant_size, checksum,
+ inout_offset, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+static gint
+compare_index_content (gconstpointer ap,
+ gconstpointer bp)
+{
+ gpointer a = *((gpointer*)ap);
+ gpointer b = *((gpointer*)bp);
+ GVariant *a_v = a;
+ GVariant *b_v = b;
+ GVariant *a_csum_bytes;
+ GVariant *b_csum_bytes;
+ guint32 a_objtype;
+ guint32 b_objtype;
+ guint64 a_offset;
+ guint64 b_offset;
+ int c;
+
+ g_variant_get (a_v, "(u ayt)", &a_objtype, &a_csum_bytes, &a_offset);
+ g_variant_get (b_v, "(u ayt)", &b_objtype, &b_csum_bytes, &b_offset);
+ a_objtype = GUINT32_FROM_BE (a_objtype);
+ b_objtype = GUINT32_FROM_BE (b_objtype);
+ c = ostree_cmp_checksum_bytes (a_csum_bytes, b_csum_bytes);
+ if (c == 0)
+ {
+ if (a_objtype < b_objtype)
+ c = -1;
+ else if (a_objtype > b_objtype)
+ c = 1;
+ }
+ return c;
+}
+
+static gboolean
+delete_loose_object (OtRepackData *data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *object_path = NULL;
+ GFile *content_object_path = NULL;
+ GVariant *archive_meta = NULL;
+ GFileInfo *file_info = NULL;
+ GVariant *xattrs = NULL;
+
+ object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
+
+ /* This is gross - we need to specially clean up symbolic link object content */
+ if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META)
+ {
+ if (!ostree_map_metadata_file (object_path, objtype, &archive_meta, error))
+ goto out;
+ if (!ostree_parse_archived_file_meta (archive_meta, &file_info, &xattrs, error))
+ goto out;
+
+ if (g_file_info_get_file_type (file_info) != G_FILE_TYPE_REGULAR)
+ {
+ content_object_path = ostree_repo_get_object_path (data->repo, checksum,
+ OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
+ if (!ot_gfile_unlink (content_object_path, cancellable, error))
+ {
+ g_prefix_error (error, "Failed to delete archived content '%s'",
+ ot_gfile_get_path_cached (content_object_path));
+ goto out;
+ }
+ }
+ }
+
+ if (!ot_gfile_unlink (object_path, cancellable, error))
+ {
+ g_prefix_error (error, "Failed to delete archived file metadata '%s'",
+ ot_gfile_get_path_cached (object_path));
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ g_clear_object (&object_path);
+ g_clear_object (&content_object_path);
+ ot_clear_gvariant (&archive_meta);
+ g_clear_object (&file_info);
+ ot_clear_gvariant (&xattrs);
+ return ret;
+}
+
+static gboolean
+create_pack_file (OtRepackData *data,
+ GPtrArray *objects,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *pack_dir = NULL;
+ GFile *index_temppath = NULL;
+ GOutputStream *index_out = NULL;
+ GFile *pack_temppath = NULL;
+ GOutputStream *pack_out = NULL;
+ GFile *object_path = NULL;
+ GFileInfo *object_file_info = NULL;
+ GFileInputStream *object_input = NULL;
+ GConverter *compressor = NULL;
+ GConverterInputStream *compressed_object_input = NULL;
+ guint i;
+ guint64 offset;
+ gsize bytes_written;
+ GPtrArray *index_content_list = NULL;
+ GVariant *pack_header = NULL;
+ GVariant *packed_object = NULL;
+ GVariant *index_content = NULL;
+ GVariantBuilder index_content_builder;
+ GChecksum *pack_checksum = NULL;
+ char *pack_name = NULL;
+ GFile *pack_file_path = NULL;
+ GFile *pack_index_path = NULL;
+ GMemoryOutputStream *object_data_stream = NULL;
+
+ if (g_cancellable_set_error_if_cancelled (cancellable, error))
+ return FALSE;
+
+ if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo),
+ "pack-index", NULL,
+ &index_temppath,
+ &index_out,
+ cancellable, error))
+ goto out;
+
+ if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo),
+ "pack-content", NULL,
+ &pack_temppath,
+ &pack_out,
+ cancellable, error))
+ goto out;
+
+ index_content_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
+
+ offset = 0;
+ pack_checksum = g_checksum_new (G_CHECKSUM_SHA256);
+
+ pack_header = g_variant_new ("(s a{sv}t)",
+ "OSTv0PACKFILE",
+ g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
+ (guint64)objects->len);
+
+ if (!write_variant_with_size (pack_out, pack_header, pack_checksum, &offset,
+ cancellable, error))
+ goto out;
+
+ for (i = 0; i < objects->len; i++)
+ {
+ GVariant *object_data = objects->pdata[i];
+ const char *checksum;
+ guint32 objtype_u32;
+ OstreeObjectType objtype;
+ guint64 expected_objsize;
+ guint64 objsize;
+ GInputStream *read_object_in;
+ guchar entry_flags = 0;
+ GVariant *index_entry;
+
+ g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize);
+
+ objtype = (OstreeObjectType) objtype_u32;
+
+ switch (data->int_compression)
+ {
+ case OT_COMPRESSION_GZIP:
+ {
+ entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP;
+ break;
+ }
+ default:
+ {
+ g_assert_not_reached ();
+ }
+ }
+
+ g_clear_object (&object_path);
+ object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
+
+ g_clear_object (&object_input);
+ object_input = g_file_read (object_path, cancellable, error);
+ if (!object_input)
+ goto out;
+
+ g_clear_object (&object_file_info);
+ object_file_info = g_file_input_stream_query_info (object_input, OSTREE_GIO_FAST_QUERYINFO, cancellable, error);
+ if (!object_file_info)
+ goto out;
+
+ objsize = g_file_info_get_attribute_uint64 (object_file_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
+
+ g_assert_cmpint (objsize, ==, expected_objsize);
+
+ g_clear_object (&object_data_stream);
+ object_data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
+
+ if (entry_flags & OSTREE_PACK_FILE_ENTRY_FLAG_GZIP)
+ {
+ g_clear_object (&compressor);
+ compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, OT_GZIP_COMPRESSION_LEVEL);
+
+ g_clear_object (&compressed_object_input);
+ compressed_object_input = (GConverterInputStream*)g_object_new (G_TYPE_CONVERTER_INPUT_STREAM,
+ "converter", compressor,
+ "base-stream", object_input,
+ "close-base-stream", TRUE,
+ NULL);
+ read_object_in = (GInputStream*)compressed_object_input;
+ }
+ else
+ {
+ read_object_in = (GInputStream*)object_input;
+ }
+
+ if (!g_output_stream_splice ((GOutputStream*)object_data_stream, read_object_in,
+ G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
+ cancellable, error))
+ goto out;
+
+ ot_clear_gvariant (&packed_object);
+ {
+ guchar *data = g_memory_output_stream_get_data (object_data_stream);
+ gsize data_len = g_memory_output_stream_get_data_size (object_data_stream);
+ packed_object = g_variant_new ("(uy ay@ay)", GUINT32_TO_BE ((guint32)objtype),
+ entry_flags,
+ ostree_checksum_to_bytes (checksum),
+ g_variant_new_fixed_array (G_VARIANT_TYPE ("y"),
+ data, data_len,
+ 1));
+ g_clear_object (&object_data_stream);
+ }
+
+ if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error))
+ goto out;
+
+ /* offset points to aligned header size */
+ index_entry = g_variant_new ("(u ayt)",
+ GUINT32_TO_BE ((guint32)objtype),
+ ostree_checksum_to_bytes (checksum),
+ GUINT64_TO_BE (offset));
+ g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry));
+
+ if (!write_variant_with_size (pack_out, packed_object, pack_checksum,
+ &offset, cancellable, error))
+ goto out;
+ }
+
+ if (!g_output_stream_close (pack_out, cancellable, error))
+ goto out;
+
+ g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(uayt)"));
+ g_ptr_array_sort (index_content_list, compare_index_content);
+ for (i = 0; i < index_content_list->len; i++)
+ {
+ GVariant *index_item = index_content_list->pdata[i];
+ g_variant_builder_add_value (&index_content_builder, index_item);
+ }
+ index_content = g_variant_new ("(s a{sv}@a(uayt))",
+ "OSTv0PACKINDEX",
+ g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
+ g_variant_builder_end (&index_content_builder));
+
+ if (!g_output_stream_write_all (index_out,
+ g_variant_get_data (index_content),
+ g_variant_get_size (index_content),
+ &bytes_written,
+ cancellable,
+ error))
+ goto out;
+
+ if (!g_output_stream_close (index_out, cancellable, error))
+ goto out;
+
+ if (!ostree_repo_add_pack_file (data->repo,
+ g_checksum_get_string (pack_checksum),
+ index_temppath,
+ pack_temppath,
+ cancellable,
+ error))
+ goto out;
+
+ if (!ostree_repo_regenerate_pack_index (data->repo, cancellable, error))
+ goto out;
+
+ g_print ("Created pack file '%s' with %u objects\n", g_checksum_get_string (pack_checksum), objects->len);
+
+ if (!opt_keep_loose)
+ {
+ for (i = 0; i < objects->len; i++)
+ {
+ GVariant *object_data = objects->pdata[i];
+ const char *checksum;
+ guint32 objtype_u32;
+ OstreeObjectType objtype;
+ guint64 expected_objsize;
+
+ g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize);
+
+ objtype = (OstreeObjectType) objtype_u32;
+
+ if (!delete_loose_object (data, checksum, objtype, cancellable, error))
+ goto out;
+ }
+ }
+
+ ret = TRUE;
+ out:
+ if (index_temppath)
+ (void) unlink (ot_gfile_get_path_cached (index_temppath));
+ g_clear_object (&index_temppath);
+ g_clear_object (&index_out);
+ if (pack_temppath)
+ (void) unlink (ot_gfile_get_path_cached (pack_temppath));
+ g_clear_object (&pack_temppath);
+ g_clear_object (&pack_out);
+ g_clear_object (&object_path);
+ g_clear_object (&object_input);
+ g_clear_object (&compressor);
+ g_clear_object (&compressed_object_input);
+ g_clear_object (&object_file_info);
+ if (pack_checksum)
+ g_checksum_free (pack_checksum);
+ g_clear_object (&pack_dir);
+ ot_clear_gvariant (&index_content);
+ g_free (pack_name);
+ g_clear_object (&pack_file_path);
+ g_clear_object (&pack_index_path);
+ if (index_content_list)
+ g_ptr_array_unref (index_content_list);
+ return ret;
+}
+
+/**
+ * cluster_objects_stupidly:
+ * @objects: Map from serialized object name to objdata
+ * @out_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref().
+ *
+ * Just sorts by size currently. Also filters out non-regular object
+ * content.
+ */
+static gboolean
+cluster_objects_stupidly (OtRepackData *data,
+ GHashTable *objects,
+ GPtrArray **out_clusters,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GPtrArray *ret_clusters = NULL;
+ GPtrArray *object_list = NULL;
+ guint i;
+ guint64 current_size;
+ guint current_offset;
+ GHashTableIter hash_iter;
+ gpointer key, value;
+ GFile *object_path = NULL;
+ GFileInfo *object_info = NULL;
+
+ object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
+
+ g_hash_table_iter_init (&hash_iter, objects);
+
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ GVariant *serialized_key = key;
+ const char *checksum;
+ OstreeObjectType objtype;
+ guint64 size;
+
+ ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+
+ g_clear_object (&object_path);
+ object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
+
+ g_clear_object (&object_info);
+ object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO,
+ G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+ cancellable, error);
+ if (!object_info)
+ goto out;
+
+ if (g_file_info_get_file_type (object_info) != G_FILE_TYPE_REGULAR)
+ continue;
+
+ size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
+
+ g_ptr_array_add (object_list,
+ g_variant_ref_sink (g_variant_new ("(sut)", checksum, (guint32)objtype, size)));
+ }
+
+ g_ptr_array_sort (object_list, compare_object_data_by_size);
+
+ ret_clusters = g_ptr_array_new_with_free_func ((GDestroyNotify)g_ptr_array_unref);
+
+ current_size = 0;
+ current_offset = 0;
+ for (i = 0; i < object_list->len; i++)
+ {
+ GVariant *objdata = object_list->pdata[i];
+ guint64 objsize;
+
+ g_variant_get_child (objdata, 2, "t", &objsize);
+
+ if (current_size + objsize > data->pack_size || i == (object_list->len - 1))
+ {
+ guint j;
+ GPtrArray *current;
+
+ if (current_offset < i)
+ {
+ current = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
+ for (j = current_offset; j <= i; j++)
+ {
+ g_ptr_array_add (current, g_variant_ref (object_list->pdata[j]));
+ }
+ g_ptr_array_add (ret_clusters, current);
+ current_size = objsize;
+ current_offset = i+1;
+ }
+ }
+ else if (objsize > data->pack_size)
+ {
+ break;
+ }
+ else
+ {
+ current_size += objsize;
+ }
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_clusters, &ret_clusters);
+ out:
+ if (object_list)
+ g_ptr_array_unref (object_list);
+ return ret;
+}
+
+static gboolean
+parse_size_spec_with_suffix (const char *spec,
+ guint64 default_value,
+ guint64 *out_size,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ char *endptr = NULL;
+ guint64 ret_size;
+
+ if (spec == NULL)
+ {
+ ret_size = default_value;
+ endptr = NULL;
+ }
+ else
+ {
+ ret_size = g_ascii_strtoull (spec, &endptr, 10);
+
+ if (endptr && *endptr)
+ {
+ char suffix = *endptr;
+
+ switch (suffix)
+ {
+ case 'k':
+ case 'K':
+ {
+ ret_size *= 1024;
+ break;
+ }
+ case 'm':
+ case 'M':
+ {
+ ret_size *= (1024 * 1024);
+ break;
+ }
+ case 'g':
+ case 'G':
+ {
+ ret_size *= (1024 * 1024 * 1024);
+ break;
+ }
+ default:
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid size suffix '%c'", suffix);
+ goto out;
+ }
+ }
+ }
+
+ ret = TRUE;
+ *out_size = ret_size;
+ out:
+ return ret;
+}
+
+static gboolean
+parse_compression_string (const char *compstr,
+ OtCompressionType *out_comptype,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ OtCompressionType ret_comptype;
+
+ if (compstr == NULL)
+ ret_comptype = OT_COMPRESSION_NONE;
+ else if (strcmp (compstr, "gzip") == 0)
+ ret_comptype = OT_COMPRESSION_GZIP;
+ else if (strcmp (compstr, "xz") == 0)
+ ret_comptype = OT_COMPRESSION_XZ;
+ else
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid compression '%s'", compstr);
+ goto out;
+ }
+
+ ret = TRUE;
+ *out_comptype = ret_comptype;
+ out:
+ return ret;
+}
+
+static gboolean
+do_stats_gather_loose (OtRepackData *data,
+ GHashTable *objects,
+ GHashTable **out_loose,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GHashTable *ret_loose = NULL;
+ guint n_loose = 0;
+ guint n_loose_and_packed = 0;
+ guint n_packed = 0;
+ guint n_dup_packed = 0;
+ guint n_commits = 0;
+ guint n_dirmeta = 0;
+ guint n_dirtree = 0;
+ guint n_files = 0;
+ GHashTableIter hash_iter;
+ gpointer key, value;
+
+ ret_loose = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
+ (GDestroyNotify) g_variant_unref,
+ NULL);
+
+ g_hash_table_iter_init (&hash_iter, objects);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ GVariant *serialized_key = key;
+ GVariant *objdata = value;
+ const char *checksum;
+ OstreeObjectType objtype;
+ gboolean is_loose;
+ gboolean is_packed;
+ GVariant *pack_array;
+
+ ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+
+ g_variant_get (objdata, "(b as)", &is_loose, &pack_array);
+
+ is_packed = g_variant_n_children (pack_array) > 0;
+
+ if (is_loose && is_packed)
+ {
+ n_loose_and_packed++;
+ }
+ else if (is_loose)
+ {
+ GVariant *copy = g_variant_ref (serialized_key);
+ g_hash_table_replace (ret_loose, copy, copy);
+ n_loose++;
+ }
+ else if (g_variant_n_children (pack_array) > 1)
+ {
+ n_dup_packed++;
+ }
+ else
+ {
+ n_packed++;
+ }
+
+ switch (objtype)
+ {
+ case OSTREE_OBJECT_TYPE_COMMIT:
+ n_commits++;
+ break;
+ case OSTREE_OBJECT_TYPE_DIR_TREE:
+ n_dirtree++;
+ break;
+ case OSTREE_OBJECT_TYPE_DIR_META:
+ n_dirmeta++;
+ break;
+ case OSTREE_OBJECT_TYPE_RAW_FILE:
+ case OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META:
+ n_files++;
+ break;
+ case OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT:
+ /* Counted under files by META */
+ break;
+ }
+ }
+
+ g_print ("Commits: %u\n", n_commits);
+ g_print ("Tree contents: %u\n", n_dirtree);
+ g_print ("Tree meta: %u\n", n_dirmeta);
+ g_print ("Files: %u\n", n_files);
+ g_print ("\n");
+ g_print ("Loose+packed objects: %u\n", n_loose_and_packed);
+ g_print ("Loose-only objects: %u\n", n_loose);
+ g_print ("Duplicate packed objects: %u\n", n_dup_packed);
+ g_print ("Packed-only objects: %u\n", n_packed);
+
+ ret = TRUE;
+ ot_transfer_out_value (out_loose, &ret_loose);
+ /* out: */
+ if (ret_loose)
+ g_hash_table_unref (ret_loose);
+ return ret;
+}
+
+static gboolean
+do_incremental_pack (OtRepackData *data,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GHashTable *objects = NULL;
+ guint i;
+ GPtrArray *clusters = NULL;
+ GHashTable *loose_objects = NULL;
+
+ if (!ostree_repo_list_objects (data->repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects,
+ cancellable, error))
+ goto out;
+
+ if (!do_stats_gather_loose (data, objects, &loose_objects, cancellable, error))
+ goto out;
+
+ g_print ("\n");
+ g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data->pack_size);
+
+ if (!cluster_objects_stupidly (data, loose_objects, &clusters, cancellable, error))
+ goto out;
+
+ if (clusters->len > 0)
+ g_print ("Going to create %u packfiles\n", clusters->len);
+ else
+ g_print ("Nothing to do\n");
+
+ for (i = 0; i < clusters->len; i++)
+ {
+ GPtrArray *cluster = clusters->pdata[i];
+
+ if (!opt_analyze_only)
+ {
+ if (!create_pack_file (data, cluster, cancellable, error))
+ goto out;
+ }
+ }
+
+ ret = TRUE;
+ out:
+ if (clusters)
+ g_ptr_array_unref (clusters);
+ if (loose_objects)
+ g_hash_table_unref (loose_objects);
+ if (objects)
+ g_hash_table_unref (objects);
+ return ret;
+}
+
+gboolean
+ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error)
+{
+ gboolean ret = FALSE;
+ GOptionContext *context;
+ OtRepackData data;
+ OstreeRepo *repo = NULL;
+ GCancellable *cancellable = NULL;
+
+ memset (&data, 0, sizeof (data));
+
+ context = g_option_context_new ("- Recompress objects");
+ g_option_context_add_main_entries (context, options, NULL);
+
+ if (!g_option_context_parse (context, &argc, &argv, error))
+ goto out;
+
+ repo = ostree_repo_new (repo_path);
+ if (!ostree_repo_check (repo, error))
+ goto out;
+
+ if (ostree_repo_get_mode (repo) != OSTREE_REPO_MODE_ARCHIVE)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
+ "Can't repack bare repositories yet");
+ goto out;
+ }
+
+ data.repo = repo;
+ data.error = error;
+
+ if (!parse_size_spec_with_suffix (opt_pack_size, OT_DEFAULT_PACK_SIZE_BYTES, &data.pack_size, error))
+ goto out;
+ /* Default internal compression to gzip */
+ if (!parse_compression_string (opt_int_compression ? opt_int_compression : "gzip", &data.int_compression, error))
+ goto out;
+ if (!parse_compression_string (opt_ext_compression, &data.ext_compression, error))
+ goto out;
+
+ if (opt_reindex_only)
+ {
+ if (!ostree_repo_regenerate_pack_index (repo, cancellable, error))
+ goto out;
+ }
+ else
+ {
+ if (!do_incremental_pack (&data, cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ if (context)
+ g_option_context_free (context);
+ g_clear_object (&repo);
+ return ret;
+}
diff --git a/src/ostree/ot-builtin-prune.c b/src/ostree/ot-builtin-prune.c
index 5090036..2c50f51 100644
--- a/src/ostree/ot-builtin-prune.c
+++ b/src/ostree/ot-builtin-prune.c
@@ -187,24 +187,27 @@ compute_reachable_objects_from_commit (OstreeRepo *repo,
return ret;
}
-static void
-object_iter_callback (OstreeRepo *repo,
- const char *checksum,
- OstreeObjectType objtype,
- GFile *objf,
- GFileInfo *file_info,
- gpointer user_data)
+static gboolean
+prune_loose_object (OtPruneData *data,
+ const char *checksum,
+ OstreeObjectType objtype,
+ GCancellable *cancellable,
+ GError **error)
{
- OtPruneData *data = user_data;
+ gboolean ret = FALSE;
char *key;
+ GFile *objf = NULL;
key = ostree_object_to_string (checksum, objtype);
+ objf = ostree_repo_get_object_path (data->repo, checksum, objtype);
+
if (!g_hash_table_lookup_extended (data->reachable, key, NULL, NULL))
{
if (delete)
{
- (void) unlink (ot_gfile_get_path_cached (objf));
+ if (!g_file_delete (objf, cancellable, error))
+ goto out;
g_print ("Deleted: %s\n", key);
}
else
@@ -216,16 +219,21 @@ object_iter_callback (OstreeRepo *repo,
else
data->n_reachable++;
+ ret = TRUE;
+ out:
+ g_clear_object (&objf);
g_free (key);
+ return ret;
}
gboolean
ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error)
{
+ gboolean ret = FALSE;
GOptionContext *context;
OtPruneData data;
- gboolean ret = FALSE;
+ GHashTable *objects = NULL;
OstreeRepo *repo = NULL;
GHashTable *all_refs = NULL;
GHashTableIter hash_iter;
@@ -266,10 +274,36 @@ ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error)
goto out;
}
- g_hash_table_iter_init (&hash_iter, data.reachable);
+ if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, cancellable, error))
+ goto out;
+
+ g_hash_table_iter_init (&hash_iter, objects);
+
- if (!ostree_repo_iter_objects (repo, object_iter_callback, &data, error))
+ if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL,
+ &objects, cancellable, error))
goto out;
+
+ g_hash_table_iter_init (&hash_iter, objects);
+
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ GVariant *serialized_key = key;
+ GVariant *objdata = value;
+ const char *checksum;
+ OstreeObjectType objtype;
+ gboolean is_loose;
+
+ ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+
+ g_variant_get_child (objdata, 0, "b", &is_loose);
+
+ if (is_loose)
+ {
+ if (!prune_loose_object (&data, checksum, objtype, cancellable, error))
+ goto out;
+ }
+ }
if (data.had_error)
goto out;
@@ -286,5 +320,7 @@ ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error)
if (context)
g_option_context_free (context);
g_clear_object (&repo);
+ if (objects)
+ g_hash_table_unref (objects);
return ret;
}
diff --git a/src/ostree/ot-builtin-unpack.c b/src/ostree/ot-builtin-unpack.c
new file mode 100644
index 0000000..eae4b04
--- /dev/null
+++ b/src/ostree/ot-builtin-unpack.c
@@ -0,0 +1,305 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2012 Colin Walters <walters verbum org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ *
+ * Author: Colin Walters <walters verbum org>
+ */
+
+#include "config.h"
+
+#include "ot-builtins.h"
+#include "ostree.h"
+
+#include <glib/gi18n.h>
+#include <glib/gprintf.h>
+
+#include <gio/gunixinputstream.h>
+#include <gio/gunixoutputstream.h>
+
+static GOptionEntry options[] = {
+ { NULL }
+};
+
+typedef struct {
+ OstreeRepo *repo;
+} OtUnpackData;
+
+static gboolean
+gather_packed (OtUnpackData *data,
+ GHashTable *objects,
+ GHashTable **out_packed,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GHashTable *ret_packed = NULL;
+ GHashTableIter hash_iter;
+ gpointer key, value;
+ GVariant *pack_array = NULL;
+
+ ret_packed = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
+ (GDestroyNotify) g_variant_unref,
+ NULL);
+
+ g_hash_table_iter_init (&hash_iter, objects);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ GVariant *serialized_key = key;
+ GVariant *key_copy;
+ GVariant *objdata = value;
+ const char *checksum;
+ OstreeObjectType objtype;
+ gboolean is_loose;
+ gboolean is_packed;
+
+ ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+
+ ot_clear_gvariant (&pack_array);
+ g_variant_get (objdata, "(b as)", &is_loose, &pack_array);
+
+ is_packed = g_variant_n_children (pack_array) > 0;
+
+ if (is_loose)
+ continue;
+
+ g_assert (is_packed);
+
+ key_copy = g_variant_ref (serialized_key);
+ g_hash_table_replace (ret_packed, key_copy, key_copy);
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_packed, &ret_packed);
+ /* out: */
+ ot_clear_gvariant (&pack_array);
+ if (ret_packed)
+ g_hash_table_unref (ret_packed);
+ return ret;
+}
+
+static gboolean
+unpack_one_object (OstreeRepo *repo,
+ const char *checksum,
+ OstreeObjectType objtype,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GInputStream *input = NULL;
+ GFileInfo *file_info = NULL;
+ GVariant *xattrs = NULL;
+ GVariant *meta = NULL;
+ GVariant *serialized_meta = NULL;
+
+ g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE);
+
+ if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META)
+ {
+ if (!ostree_repo_load_file (repo, checksum,
+ &input, &file_info, &xattrs,
+ cancellable, error))
+ goto out;
+
+ if (!ostree_repo_stage_object_trusted (repo, OSTREE_OBJECT_TYPE_RAW_FILE,
+ checksum, TRUE, file_info, xattrs, input,
+ cancellable, error))
+ goto out;
+ }
+ else if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT)
+ {
+ /* nothing; handled in META case */
+ }
+ else
+ {
+ if (!ostree_repo_load_variant (repo, objtype, checksum, &meta, error))
+ goto out;
+
+ serialized_meta = ostree_wrap_metadata_variant (objtype, meta);
+
+ input = g_memory_input_stream_new_from_data (g_variant_get_data (serialized_meta),
+ g_variant_get_size (serialized_meta), NULL);
+
+ if (!ostree_repo_stage_object_trusted (repo, objtype, checksum, TRUE,
+ NULL, NULL, input, cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ g_clear_object (&input);
+ g_clear_object (&file_info);
+ ot_clear_gvariant (&xattrs);
+ ot_clear_gvariant (&meta);
+ ot_clear_gvariant (&serialized_meta);
+ return ret;
+}
+
+static gboolean
+delete_one_packfile (OstreeRepo *repo,
+ const char *pack_checksum,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *data_path = NULL;
+ GFile *index_path = NULL;
+
+ index_path = ostree_repo_get_pack_index_path (repo, pack_checksum);
+ data_path = ostree_repo_get_pack_data_path (repo, pack_checksum);
+
+ if (!ot_gfile_unlink (index_path, cancellable, error))
+ {
+ g_prefix_error (error, "Failed to delete pack index '%s': ", ot_gfile_get_path_cached (index_path));
+ goto out;
+ }
+ if (!ot_gfile_unlink (data_path, cancellable, error))
+ {
+ g_prefix_error (error, "Failed to delete pack data '%s': ", ot_gfile_get_path_cached (data_path));
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ g_clear_object (&index_path);
+ g_clear_object (&data_path);
+ return ret;
+}
+
+gboolean
+ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
+{
+ gboolean ret = FALSE;
+ GOptionContext *context;
+ gboolean in_transaction = FALSE;
+ OtUnpackData data;
+ OstreeRepo *repo = NULL;
+ GHashTable *objects = NULL;
+ GCancellable *cancellable = NULL;
+ GPtrArray *clusters = NULL;
+ GHashTable *packed_objects = NULL;
+ GHashTableIter hash_iter;
+ GHashTable *packfiles_to_delete = NULL;
+ gpointer key, value;
+ GFile *objpath = NULL;
+ guint64 unpacked_object_count = 0;
+
+ memset (&data, 0, sizeof (data));
+
+ context = g_option_context_new ("- Uncompress objects");
+ g_option_context_add_main_entries (context, options, NULL);
+
+ if (!g_option_context_parse (context, &argc, &argv, error))
+ goto out;
+
+ repo = ostree_repo_new (repo_path);
+ if (!ostree_repo_check (repo, error))
+ goto out;
+
+ if (ostree_repo_get_mode (repo) != OSTREE_REPO_MODE_ARCHIVE)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
+ "Can't unpack bare repositories yet");
+ goto out;
+ }
+
+ data.repo = repo;
+
+ if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, cancellable, error))
+ goto out;
+
+ if (!gather_packed (&data, objects, &packed_objects, cancellable, error))
+ goto out;
+
+ if (!ostree_repo_prepare_transaction (repo, cancellable, error))
+ goto out;
+
+ in_transaction = TRUE;
+
+ packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+
+ g_hash_table_iter_init (&hash_iter, packed_objects);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ GVariant *objkey = key;
+ GVariant *objdata;
+ const char *checksum;
+ const char *pack_checksum;
+ OstreeObjectType objtype;
+ gboolean is_loose;
+ GVariantIter *pack_array_iter;
+
+ objdata = g_hash_table_lookup (objects, objkey);
+ g_assert (objdata);
+
+ g_variant_get (objdata, "(bas)", &is_loose, &pack_array_iter);
+
+ g_assert (!is_loose);
+
+ while (g_variant_iter_loop (pack_array_iter, "&s", &pack_checksum))
+ {
+ if (!g_hash_table_lookup (packfiles_to_delete, pack_checksum))
+ {
+ gchar *duped_checksum = g_strdup (pack_checksum);
+ g_hash_table_replace (packfiles_to_delete, duped_checksum, duped_checksum);
+ }
+ }
+ g_variant_iter_free (pack_array_iter);
+
+ ostree_object_name_deserialize (objkey, &checksum, &objtype);
+
+ if (!unpack_one_object (repo, checksum, objtype, cancellable, error))
+ goto out;
+
+ unpacked_object_count++;
+ }
+
+ if (!ostree_repo_commit_transaction (repo, cancellable, error))
+ goto out;
+
+ if (g_hash_table_size (packfiles_to_delete) == 0)
+ g_print ("No pack files; nothing to do\n");
+
+ g_hash_table_iter_init (&hash_iter, packfiles_to_delete);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ const char *pack_checksum = key;
+
+ if (!delete_one_packfile (repo, pack_checksum, cancellable, error))
+ goto out;
+
+ g_print ("Deleted packfile '%s'\n", pack_checksum);
+ }
+
+ ret = TRUE;
+ out:
+ if (in_transaction)
+ (void) ostree_repo_abort_transaction (repo, cancellable, NULL);
+ g_clear_object (&objpath);
+ if (context)
+ g_option_context_free (context);
+ g_clear_object (&repo);
+ if (clusters)
+ g_ptr_array_unref (clusters);
+ if (packfiles_to_delete)
+ g_hash_table_unref (packfiles_to_delete);
+ if (packed_objects)
+ g_hash_table_unref (packed_objects);
+ if (objects)
+ g_hash_table_unref (objects);
+ return ret;
+}
diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h
index befd203..73afccc 100644
--- a/src/ostree/ot-builtins.h
+++ b/src/ostree/ot-builtins.h
@@ -40,8 +40,10 @@ gboolean ostree_builtin_ls (int argc, char **argv, GFile *repo_path, GError **er
gboolean ostree_builtin_prune (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_fsck (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_show (int argc, char **argv, GFile *repo_path, GError **error);
+gboolean ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_rev_parse (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_remote (int argc, char **argv, GFile *repo_path, GError **error);
+gboolean ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error);
G_END_DECLS
diff --git a/tests/t0001-archive.sh b/tests/t0001-archive.sh
index 6bdc9aa..4712732 100755
--- a/tests/t0001-archive.sh
+++ b/tests/t0001-archive.sh
@@ -21,7 +21,7 @@ set -e
. libtest.sh
-echo '1..10'
+echo '1..19'
setup_test_repository "archive"
echo "ok setup"
@@ -67,3 +67,36 @@ cd ${test_tmpdir}
$OSTREE cat test2 /baz/cow > cow-contents
assert_file_has_content cow-contents "moo"
echo "ok cat-file"
+
+cd ${test_tmpdir}
+$OSTREE pack --keep-loose
+echo "ok pack"
+
+cd ${test_tmpdir}
+$OSTREE fsck
+echo "ok fsck"
+
+$OSTREE checkout test2 checkout-test2-from-packed
+echo "ok checkout union 1"
+
+cd ${test_tmpdir}
+$OSTREE pack
+echo "ok pack delete loose"
+
+cd ${test_tmpdir}
+$OSTREE fsck
+echo "ok fsck"
+
+$OSTREE pack --analyze-only
+echo "ok pack analyze"
+
+$OSTREE unpack
+echo "ok unpack"
+
+cd ${test_tmpdir}
+$OSTREE fsck
+echo "ok fsck"
+
+cd ${test_tmpdir}
+$OSTREE checkout test2 checkout-test2-from-unpacked
+echo "ok checkout union 2"
diff --git a/tests/t0010-pull.sh b/tests/t0010-pull.sh
index 53c85b2..5f58d11 100755
--- a/tests/t0010-pull.sh
+++ b/tests/t0010-pull.sh
@@ -21,7 +21,7 @@ set -e
. libtest.sh
-echo '1..2'
+echo '1..4'
setup_fake_remote_repo1
cd ${test_tmpdir}
@@ -29,6 +29,7 @@ mkdir repo
ostree --repo=repo init
ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo
ostree-pull --repo=repo origin main
+ostree --repo=repo fsck
echo "ok pull"
cd ${test_tmpdir}
@@ -37,3 +38,21 @@ cd checkout-origin-main
assert_file_has_content firstfile '^first$'
assert_file_has_content baz/cow '^moo$'
echo "ok pull contents"
+
+cd ${test_tmpdir}
+ostree --repo=$(pwd)/ostree-srv/gnomerepo pack
+rm -rf repo
+mkdir repo
+ostree --repo=repo init
+ostree --repo=repo remote add origin $(cat httpd-address)/ostree/gnomerepo
+ostree-pull --repo=repo origin main
+ostree --repo=repo fsck
+echo "ok pull packed"
+
+cd ${test_tmpdir}
+rm -rf checkout-origin-main
+$OSTREE checkout origin/main checkout-origin-main
+cd checkout-origin-main
+assert_file_has_content firstfile '^first$'
+assert_file_has_content baz/cow '^moo$'
+echo "ok pull contents packed"
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
