[ostree/wip/packfile-rebase2] core: We can unpack <-> repack
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree/wip/packfile-rebase2] core: We can unpack <-> repack
- Date: Wed, 28 Mar 2012 00:49:07 +0000 (UTC)
commit 32d789f3fd18b20cf6b319d794fd2733131b28fc
Author: Colin Walters <walters verbum org>
Date: Tue Mar 27 20:46:27 2012 -0400
core: We can unpack <-> repack
Makefile-ostree.am | 1 +
src/libostree/ostree-core.c | 6 +-
src/libostree/ostree-repo.c | 44 ++++--
src/libostree/ostree-repo.h | 1 +
src/ostree/main.c | 1 +
src/ostree/ot-builtin-local-clone.c | 6 +-
src/ostree/ot-builtin-unpack.c | 294 +++++++++++++++++++++++++++++++++++
src/ostree/ot-builtins.h | 1 +
tests/t0001-archive.sh | 13 ++-
9 files changed, 344 insertions(+), 23 deletions(-)
---
diff --git a/Makefile-ostree.am b/Makefile-ostree.am
index 1b65f3c..0eedd14 100644
--- a/Makefile-ostree.am
+++ b/Makefile-ostree.am
@@ -35,6 +35,7 @@ ostree_SOURCES = src/ostree/main.c \
src/ostree/ot-builtin-prune.c \
src/ostree/ot-builtin-remote.c \
src/ostree/ot-builtin-pack.c \
+ src/ostree/ot-builtin-unpack.c \
src/ostree/ot-builtin-rev-parse.c \
src/ostree/ot-builtin-show.c \
src/ostree/ot-main.h \
diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c
index 390cf2e..24d1906 100644
--- a/src/libostree/ostree-core.c
+++ b/src/libostree/ostree-core.c
@@ -1226,7 +1226,7 @@ ostree_read_pack_entry_raw (guchar *pack_data,
guint64 entry_end;
guint32 entry_len;
- if (G_UNLIKELY (!(offset < pack_len)))
+ if (G_UNLIKELY (!(offset <= pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; out of range offset %" G_GUINT64_FORMAT,
@@ -1242,7 +1242,7 @@ ostree_read_pack_entry_raw (guchar *pack_data,
}
entry_start = ALIGN_VALUE (offset + 4, 8);
- if (G_UNLIKELY (!(entry_start < pack_len)))
+ if (G_UNLIKELY (!(entry_start <= pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; out of range data offset %" G_GUINT64_FORMAT,
@@ -1254,7 +1254,7 @@ ostree_read_pack_entry_raw (guchar *pack_data,
entry_len = GUINT32_FROM_BE (*((guint32*)(pack_data+offset)));
entry_end = entry_start + entry_len;
- if (G_UNLIKELY (!(entry_end < pack_len)))
+ if (G_UNLIKELY (!(entry_end <= pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted pack index; out of range entry length %u",
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 1d1f3be..277423f 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -795,6 +795,7 @@ dup_file_info_owned_by_me (GFileInfo *file_info)
static gboolean
stage_object_impl (OstreeRepo *self,
OstreeObjectType objtype,
+ gboolean store_if_packed,
GFileInfo *file_info,
GVariant *xattrs,
GInputStream *input,
@@ -912,7 +913,7 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self,
g_checksum_update (ret_checksum, (guint8*)g_variant_get_data (xattrs), g_variant_get_size (xattrs));
}
- if (expected_checksum)
+ if (expected_checksum && ret_checksum)
{
if (strcmp (g_checksum_get_string (ret_checksum), expected_checksum) != 0)
{
@@ -924,6 +925,8 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self,
}
actual_checksum = expected_checksum;
}
+ else if (expected_checksum)
+ actual_checksum = expected_checksum;
else
actual_checksum = g_checksum_get_string (ret_checksum);
@@ -951,6 +954,7 @@ impl_stage_archive_file_object_from_raw (OstreeRepo *self,
static gboolean
stage_object_impl (OstreeRepo *self,
OstreeObjectType objtype,
+ gboolean store_if_packed,
GFileInfo *file_info,
GVariant *xattrs,
GInputStream *input,
@@ -979,11 +983,22 @@ stage_object_impl (OstreeRepo *self,
if (expected_checksum)
{
- if (!ostree_repo_find_object (self, objtype, expected_checksum,
- &stored_path, &pending_path,
- &pack_checksum, &pack_offset,
- cancellable, error))
- goto out;
+ if (!store_if_packed)
+ {
+ if (!ostree_repo_find_object (self, objtype, expected_checksum,
+ &stored_path, &pending_path,
+ &pack_checksum, &pack_offset,
+ cancellable, error))
+ goto out;
+ }
+ else
+ {
+ if (!ostree_repo_find_object (self, objtype, expected_checksum,
+ &stored_path, &pending_path,
+ NULL, NULL,
+ cancellable, error))
+ goto out;
+ }
}
g_assert (objtype != OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT);
@@ -1195,7 +1210,7 @@ stage_gvariant_object (OstreeRepo *self,
g_variant_get_size (serialized),
NULL);
- if (!stage_object_impl (self, type,
+ if (!stage_object_impl (self, type, FALSE,
NULL, NULL, mem,
NULL, &ret_checksum, cancellable, error))
goto out;
@@ -1258,13 +1273,14 @@ gboolean
ostree_repo_stage_object_trusted (OstreeRepo *self,
OstreeObjectType objtype,
const char *checksum,
+ gboolean store_if_packed,
GFileInfo *file_info,
GVariant *xattrs,
GInputStream *input,
GCancellable *cancellable,
GError **error)
{
- return stage_object_impl (self, objtype,
+ return stage_object_impl (self, objtype, store_if_packed,
file_info, xattrs, input,
checksum, NULL, cancellable, error);
}
@@ -1282,7 +1298,7 @@ ostree_repo_stage_object (OstreeRepo *self,
gboolean ret = FALSE;
GChecksum *actual_checksum = NULL;
- if (!stage_object_impl (self, objtype,
+ if (!stage_object_impl (self, objtype, FALSE,
file_info, xattrs, input,
expected_checksum, &actual_checksum, cancellable, error))
goto out;
@@ -1768,7 +1784,7 @@ stage_directory_to_mtree_internal (OstreeRepo *self,
goto out;
}
- if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_RAW_FILE,
+ if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_RAW_FILE, FALSE,
modified_info, xattrs, file_input, NULL,
&child_file_checksum, cancellable, error))
goto out;
@@ -1970,7 +1986,7 @@ import_libarchive_entry_file (OstreeRepo *self,
if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
archive_stream = ostree_libarchive_input_stream_new (a);
- if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_RAW_FILE,
+ if (!stage_object_impl (self, OSTREE_OBJECT_TYPE_RAW_FILE, FALSE,
file_info, NULL, archive_stream,
NULL, &ret_checksum,
cancellable, error))
@@ -3046,7 +3062,6 @@ ostree_repo_load_variant (OstreeRepo *self,
gboolean ret = FALSE;
GFile *object_path = NULL;
GVariant *packed_object = NULL;
- GVariant *container_variant = NULL;
GVariant *ret_variant = NULL;
char *pack_checksum = NULL;
guchar *pack_data;
@@ -3069,8 +3084,6 @@ ostree_repo_load_variant (OstreeRepo *self,
}
else if (pack_checksum != NULL)
{
- guint32 actual_type;
-
if (!ostree_repo_map_pack_file (self, pack_checksum, &pack_data, &pack_len,
cancellable, error))
goto out;
@@ -3080,7 +3093,7 @@ ostree_repo_load_variant (OstreeRepo *self,
goto out;
if (!ostree_read_pack_entry_variant (packed_object, objtype, TRUE,
- &container_variant, cancellable, error))
+ &ret_variant, cancellable, error))
goto out;
}
else
@@ -3098,7 +3111,6 @@ ostree_repo_load_variant (OstreeRepo *self,
g_free (pack_checksum);
ot_clear_gvariant (&ret_variant);
ot_clear_gvariant (&packed_object);
- ot_clear_gvariant (&container_variant);
return ret;
}
/**
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 7c38176..f4c9915 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -114,6 +114,7 @@ gboolean ostree_repo_stage_object (OstreeRepo *self,
gboolean ostree_repo_stage_object_trusted (OstreeRepo *self,
OstreeObjectType objtype,
const char *checksum,
+ gboolean store_if_packed,
GFileInfo *file_info,
GVariant *xattrs,
GInputStream *content,
diff --git a/src/ostree/main.c b/src/ostree/main.c
index 9be3b8f..1b434b6 100644
--- a/src/ostree/main.c
+++ b/src/ostree/main.c
@@ -47,6 +47,7 @@ static OstreeBuiltin builtins[] = {
{ "rev-parse", ostree_builtin_rev_parse, 0 },
{ "remote", ostree_builtin_remote, 0 },
{ "show", ostree_builtin_show, 0 },
+ { "unpack", ostree_builtin_unpack, 0 },
{ NULL }
};
diff --git a/src/ostree/ot-builtin-local-clone.c b/src/ostree/ot-builtin-local-clone.c
index 840bf9d..1c9a3d0 100644
--- a/src/ostree/ot-builtin-local-clone.c
+++ b/src/ostree/ot-builtin-local-clone.c
@@ -142,8 +142,8 @@ import_loose_object (OtLocalCloneData *data,
goto out;
}
- if (!ostree_repo_stage_object_trusted (data->dest_repo, OSTREE_OBJECT_TYPE_RAW_FILE, checksum,
- archive_info, xattrs, input,
+ if (!ostree_repo_stage_object_trusted (data->dest_repo, OSTREE_OBJECT_TYPE_RAW_FILE,
+ checksum, FALSE, archive_info, xattrs, input,
NULL, error))
goto out;
}
@@ -157,7 +157,7 @@ import_loose_object (OtLocalCloneData *data,
}
if (!ostree_repo_stage_object_trusted (data->dest_repo, objtype, checksum,
- file_info, xattrs, input,
+ FALSE, file_info, xattrs, input,
NULL, error))
goto out;
}
diff --git a/src/ostree/ot-builtin-unpack.c b/src/ostree/ot-builtin-unpack.c
new file mode 100644
index 0000000..781a6c0
--- /dev/null
+++ b/src/ostree/ot-builtin-unpack.c
@@ -0,0 +1,294 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2012 Colin Walters <walters verbum org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ *
+ * Author: Colin Walters <walters verbum org>
+ */
+
+#include "config.h"
+
+#include "ot-builtins.h"
+#include "ostree.h"
+
+#include <glib/gi18n.h>
+#include <glib/gprintf.h>
+
+#include <gio/gunixinputstream.h>
+#include <gio/gunixoutputstream.h>
+
+static GOptionEntry options[] = {
+ { NULL }
+};
+
+typedef struct {
+ OstreeRepo *repo;
+} OtUnpackData;
+
+static gboolean
+gather_packed (OtUnpackData *data,
+ GHashTable *objects,
+ GHashTable **out_packed,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GHashTable *ret_packed = NULL;
+ GHashTableIter hash_iter;
+ gpointer key, value;
+ GVariant *pack_array = NULL;
+
+ ret_packed = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
+ (GDestroyNotify) g_variant_unref,
+ NULL);
+
+ g_hash_table_iter_init (&hash_iter, objects);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ GVariant *serialized_key = key;
+ GVariant *key_copy;
+ GVariant *objdata = value;
+ const char *checksum;
+ OstreeObjectType objtype;
+ gboolean is_loose;
+ gboolean is_packed;
+
+ ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+
+ ot_clear_gvariant (&pack_array);
+ g_variant_get (objdata, "(b as)", &is_loose, &pack_array);
+
+ is_packed = g_variant_n_children (pack_array) > 0;
+
+ if (is_loose)
+ continue;
+
+ g_assert (is_packed);
+
+ key_copy = g_variant_ref (serialized_key);
+ g_hash_table_replace (ret_packed, key_copy, key_copy);
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_packed, &ret_packed);
+ /* out: */
+ ot_clear_gvariant (&pack_array);
+ if (ret_packed)
+ g_hash_table_unref (ret_packed);
+ return ret;
+}
+
+static gboolean
+unpack_one_object (OstreeRepo *repo,
+ const char *checksum,
+ OstreeObjectType objtype,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GInputStream *input = NULL;
+ GFileInfo *file_info = NULL;
+ GVariant *xattrs = NULL;
+ GVariant *meta = NULL;
+ GVariant *serialized_meta = NULL;
+
+ g_assert (objtype != OSTREE_OBJECT_TYPE_RAW_FILE);
+
+ if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META)
+ {
+ if (!ostree_repo_load_file (repo, checksum,
+ &input, &file_info, &xattrs,
+ cancellable, error))
+ goto out;
+
+ if (!ostree_repo_stage_object_trusted (repo, OSTREE_OBJECT_TYPE_RAW_FILE,
+ checksum, TRUE, file_info, xattrs, input,
+ cancellable, error))
+ goto out;
+ }
+ else if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT)
+ {
+ /* nothing; handled in META case */
+ }
+ else
+ {
+ if (!ostree_repo_load_variant (repo, objtype, checksum, &meta, error))
+ goto out;
+
+ serialized_meta = ostree_wrap_metadata_variant (objtype, meta);
+
+ input = g_memory_input_stream_new_from_data (g_variant_get_data (serialized_meta),
+ g_variant_get_size (serialized_meta), NULL);
+
+ if (!ostree_repo_stage_object_trusted (repo, objtype, checksum, TRUE,
+ NULL, NULL, input, cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ g_clear_object (&input);
+ g_clear_object (&file_info);
+ ot_clear_gvariant (&xattrs);
+ ot_clear_gvariant (&meta);
+ ot_clear_gvariant (&serialized_meta);
+ return ret;
+}
+
+static gboolean
+delete_one_packfile (OstreeRepo *repo,
+ const char *pack_checksum,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *data_path = NULL;
+ GFile *index_path = NULL;
+
+ index_path = ostree_repo_get_pack_index_path (repo, pack_checksum);
+ data_path = ostree_repo_get_pack_data_path (repo, pack_checksum);
+
+ if (!ot_gfile_unlink (index_path, cancellable, error))
+ goto out;
+ if (!ot_gfile_unlink (data_path, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ g_clear_object (&index_path);
+ g_clear_object (&data_path);
+ return ret;
+}
+
+gboolean
+ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error)
+{
+ gboolean ret = FALSE;
+ GOptionContext *context;
+ gboolean in_transaction = FALSE;
+ OtUnpackData data;
+ OstreeRepo *repo = NULL;
+ GHashTable *objects = NULL;
+ GCancellable *cancellable = NULL;
+ GPtrArray *clusters = NULL;
+ GHashTable *packed_objects = NULL;
+ GHashTableIter hash_iter;
+ GHashTable *packfiles_to_delete = NULL;
+ gpointer key, value;
+ GFile *objpath = NULL;
+ guint64 unpacked_object_count = 0;
+
+ memset (&data, 0, sizeof (data));
+
+ context = g_option_context_new ("- Uncompress objects");
+ g_option_context_add_main_entries (context, options, NULL);
+
+ if (!g_option_context_parse (context, &argc, &argv, error))
+ goto out;
+
+ repo = ostree_repo_new (repo_path);
+ if (!ostree_repo_check (repo, error))
+ goto out;
+
+ if (ostree_repo_get_mode (repo) != OSTREE_REPO_MODE_ARCHIVE)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
+ "Can't unpack bare repositories yet");
+ goto out;
+ }
+
+ data.repo = repo;
+
+ if (!ostree_repo_list_objects (repo, OSTREE_REPO_LIST_OBJECTS_ALL, &objects, cancellable, error))
+ goto out;
+
+ if (!gather_packed (&data, objects, &packed_objects, cancellable, error))
+ goto out;
+
+ if (!ostree_repo_prepare_transaction (repo, cancellable, error))
+ goto out;
+
+ in_transaction = TRUE;
+
+ packfiles_to_delete = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
+
+ g_hash_table_iter_init (&hash_iter, packed_objects);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ GVariant *objkey = key;
+ GVariant *objdata;
+ const char *checksum;
+ const char *pack_checksum;
+ OstreeObjectType objtype;
+ gboolean is_loose;
+ GVariantIter *pack_array_iter;
+
+ objdata = g_hash_table_lookup (objects, objkey);
+ g_assert (objdata);
+
+ g_variant_get (objdata, "(bas)", &is_loose, &pack_array_iter);
+
+ g_assert (!is_loose);
+
+ while (g_variant_iter_loop (pack_array_iter, "&s", &pack_checksum))
+ {
+ if (!g_hash_table_lookup (packfiles_to_delete, pack_checksum))
+ {
+ gchar *duped_checksum = g_strdup (pack_checksum);
+ g_hash_table_replace (packfiles_to_delete, duped_checksum, duped_checksum);
+ }
+ }
+ g_variant_iter_free (pack_array_iter);
+
+ ostree_object_name_deserialize (objkey, &checksum, &objtype);
+
+ if (!unpack_one_object (repo, checksum, objtype, cancellable, error))
+ goto out;
+
+ unpacked_object_count++;
+ }
+
+ if (!ostree_repo_commit_transaction (repo, cancellable, error))
+ goto out;
+
+ g_hash_table_iter_init (&hash_iter, packfiles_to_delete);
+ while (g_hash_table_iter_next (&hash_iter, &key, &value))
+ {
+ const char *pack_checksum = key;
+
+ if (!delete_one_packfile (repo, pack_checksum, cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ if (in_transaction)
+ (void) ostree_repo_abort_transaction (repo, cancellable, NULL);
+ g_clear_object (&objpath);
+ if (context)
+ g_option_context_free (context);
+ g_clear_object (&repo);
+ if (clusters)
+ g_ptr_array_unref (clusters);
+ if (packfiles_to_delete)
+ g_hash_table_unref (packfiles_to_delete);
+ if (packed_objects)
+ g_hash_table_unref (packed_objects);
+ if (objects)
+ g_hash_table_unref (objects);
+ return ret;
+}
diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h
index d230cba..73afccc 100644
--- a/src/ostree/ot-builtins.h
+++ b/src/ostree/ot-builtins.h
@@ -43,6 +43,7 @@ gboolean ostree_builtin_show (int argc, char **argv, GFile *repo_path, GError **
gboolean ostree_builtin_pack (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_rev_parse (int argc, char **argv, GFile *repo_path, GError **error);
gboolean ostree_builtin_remote (int argc, char **argv, GFile *repo_path, GError **error);
+gboolean ostree_builtin_unpack (int argc, char **argv, GFile *repo_path, GError **error);
G_END_DECLS
diff --git a/tests/t0001-archive.sh b/tests/t0001-archive.sh
index ec3ab2c..4712732 100755
--- a/tests/t0001-archive.sh
+++ b/tests/t0001-archive.sh
@@ -21,7 +21,7 @@ set -e
. libtest.sh
-echo '1..16'
+echo '1..19'
setup_test_repository "archive"
echo "ok setup"
@@ -89,3 +89,14 @@ echo "ok fsck"
$OSTREE pack --analyze-only
echo "ok pack analyze"
+
+$OSTREE unpack
+echo "ok unpack"
+
+cd ${test_tmpdir}
+$OSTREE fsck
+echo "ok fsck"
+
+cd ${test_tmpdir}
+$OSTREE checkout test2 checkout-test2-from-unpacked
+echo "ok checkout union 2"
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
