[ostree/wip/packfile-rebase2: 7/11] wip
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree/wip/packfile-rebase2: 7/11] wip
- Date: Thu, 22 Mar 2012 03:14:30 +0000 (UTC)
commit 2d216442000a4824abee01b76f47d96c19af936c
Author: Colin Walters <walters verbum org>
Date: Tue Mar 20 13:49:31 2012 -0400
wip
src/libostree/ostree-core.h | 11 +-
src/libostree/ostree-repo.c | 329 +++++++++++++++++++++++++++++++++++++++-
src/libostree/ostree-repo.h | 15 ++-
src/ostree/ot-builtin-init.c | 2 +-
src/ostree/ot-builtin-repack.c | 296 +++++++++++++++++++++---------------
5 files changed, 514 insertions(+), 139 deletions(-)
---
diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h
index 398c4aa..e11392c 100644
--- a/src/libostree/ostree-core.h
+++ b/src/libostree/ostree-core.h
@@ -109,9 +109,9 @@ typedef enum {
* s - OSTPACKINDEX
* u - Version
* a{sv} - Metadata
- * a(st) - (checksum, offset into packfile)
+ * a(sut) - (checksum, objtype, offset into packfile)
*/
-#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sua{sv}a(st))")
+#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sua{sv}a(sut))")
typedef enum {
OSTREE_PACK_FILE_ENTRY_FLAG_NONE = 0,
@@ -125,13 +125,14 @@ typedef enum {
* u - number of entries
*
* Repeating tuple of:
+ * <padding to alignment of 4>
+ * <32 bit BE integer containing variant length>
* <padding to alignment of 8>
- * (yst) - Contents (flags, checksum, objtype, length)
- * <raw data>
+ * ( tuys ) - content_length, objtype, flags, checksum
*/
#define OSTREE_PACK_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(sua{sv}t)")
-#define OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT G_VARIANT_TYPE ("(yst)")
+#define OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT G_VARIANT_TYPE ("(ysut)")
gboolean ostree_validate_checksum_string (const char *sha256,
GError **error);
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 83489e7..bff33f9 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -60,6 +60,7 @@ struct _OstreeRepoPrivate {
GFile *local_heads_dir;
GFile *remote_heads_dir;
GFile *objects_dir;
+ GFile *pack_dir;
GFile *config_file;
gboolean inited;
@@ -68,6 +69,8 @@ struct _OstreeRepoPrivate {
GKeyFile *config;
OstreeRepoMode mode;
+ GHashTable *pack_index_mappings;
+
GHashTable *pending_transaction;
};
@@ -83,7 +86,9 @@ ostree_repo_finalize (GObject *object)
g_clear_object (&priv->local_heads_dir);
g_clear_object (&priv->remote_heads_dir);
g_clear_object (&priv->objects_dir);
+ g_clear_object (&priv->pack_dir);
g_clear_object (&priv->config_file);
+ g_hash_table_destroy (priv->pack_index_mappings);
g_hash_table_destroy (priv->pending_transaction);
if (priv->config)
g_key_file_free (priv->config);
@@ -154,6 +159,7 @@ ostree_repo_constructor (GType gtype,
priv->remote_heads_dir = g_file_resolve_relative_path (priv->repodir, "refs/remotes");
priv->objects_dir = g_file_get_child (priv->repodir, "objects");
+ priv->pack_dir = g_file_get_child (priv->objects_dir, "pack");
priv->config_file = g_file_get_child (priv->repodir, "config");
return object;
@@ -185,6 +191,9 @@ ostree_repo_init (OstreeRepo *self)
{
OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ priv->pack_index_mappings = g_hash_table_new_full (g_str_hash, g_str_equal,
+ g_free,
+ (GDestroyNotify)g_mapped_file_unref);
priv->pending_transaction = g_hash_table_new_full (g_str_hash, g_str_equal,
g_free,
NULL);
@@ -2324,7 +2333,8 @@ list_loose_object_dir (OstreeRepo *self,
dirname = ot_gfile_get_basename_cached (dir);
- enumerator = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO,
+ /* We're only querying name */
+ enumerator = g_file_enumerate_children (dir, "standard::name,standard::type",
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable,
error);
@@ -2371,12 +2381,8 @@ list_loose_object_dir (OstreeRepo *self,
g_string_append_len (checksum, name, 62);
key = ostree_object_name_serialize (checksum->str, objtype);
- value = g_variant_new ("(b astt)",
- TRUE, g_variant_new_strv (NULL, 0),
- g_file_info_get_attribute_uint64 (file_info,
- G_FILE_ATTRIBUTE_TIME_CHANGED),
- g_file_info_get_attribute_uint64 (file_info,
- G_FILE_ATTRIBUTE_STANDARD_SIZE));
+ value = g_variant_new ("(b as)",
+ TRUE, g_variant_new_strv (NULL, 0));
/* transfer ownership */
g_hash_table_replace (inout_objects, g_variant_ref_sink (key),
g_variant_ref_sink (value));
@@ -2402,6 +2408,76 @@ list_loose_object_dir (OstreeRepo *self,
}
static gboolean
+list_files_in_dir_matching (GFile *dir,
+ const char *prefix,
+ const char *suffix,
+ GPtrArray **out_files,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GError *temp_error = NULL;
+ GFileEnumerator *enumerator = NULL;
+ GFileInfo *file_info = NULL;
+ GPtrArray *ret_files = NULL;
+
+ g_return_val_if_fail (prefix != NULL || suffix != NULL, FALSE);
+
+ ret_files = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref);
+
+ enumerator = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO,
+ G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+ cancellable,
+ error);
+ if (!enumerator)
+ goto out;
+
+ while ((file_info = g_file_enumerator_next_file (enumerator, cancellable, &temp_error)) != NULL)
+ {
+ const char *name;
+ guint32 type;
+
+ name = g_file_info_get_attribute_byte_string (file_info, "standard::name");
+ type = g_file_info_get_attribute_uint32 (file_info, "standard::type");
+
+ if (type != G_FILE_TYPE_REGULAR)
+ goto loop_next;
+
+ if (prefix)
+ {
+ if (!g_str_has_prefix (name, prefix))
+ goto loop_next;
+ }
+ if (suffix)
+ {
+ if (!g_str_has_suffix (name, suffix))
+ goto loop_next;
+ }
+
+ g_ptr_array_add (ret_files, g_file_get_child (dir, name));
+
+ loop_next:
+ g_clear_object (&file_info);
+ }
+ if (temp_error != NULL)
+ {
+ g_propagate_error (error, temp_error);
+ goto out;
+ }
+ if (!g_file_enumerator_close (enumerator, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ ot_transfer_out_value (out_files, &ret_files);
+ out:
+ if (ret_files)
+ g_ptr_array_unref (ret_files);
+ g_clear_object (&file_info);
+ g_clear_object (&enumerator);
+ return ret;
+}
+
+static gboolean
list_loose_objects (OstreeRepo *self,
GHashTable *inout_objects,
GCancellable *cancellable,
@@ -2454,6 +2530,239 @@ list_loose_objects (OstreeRepo *self,
return ret;
}
+static GFile *
+get_pack_data_for_index (GFile *index)
+{
+ const char *basename;
+ GString *name = g_string_new ("");
+ GFile *ret;
+ GFile *parent;
+
+ basename = ot_gfile_get_basename_cached (index);
+ g_assert (g_str_has_suffix (basename, ".index"));
+ g_string_append_len (name, basename, strlen (basename) - 5);
+ g_string_append (name, "data");
+
+ parent = g_file_get_parent (index);
+ ret = g_file_get_child (parent, name->str);
+ g_object_unref (parent);
+ g_string_free (name, TRUE);
+ return ret;
+}
+
+static char *
+get_checksum_from_pack_name (const char *name)
+{
+ const char *dash;
+ const char *dot;
+
+ dash = strchr (name, '-');
+ g_assert (dash);
+ dot = strrchr (name, '.');
+ g_assert (dot);
+
+ g_assert_cmpint (dot - dash, ==, 64);
+
+ return g_strndup (dash + 1, 64);
+}
+
+static GFile *
+get_pack_index_path (OstreeRepo *self,
+ const char *checksum)
+{
+ char *name;
+ GFile *ret;
+
+ name = g_strconcat ("ostpack-", checksum, ".index", NULL);
+ ret = g_file_get_child (GET_PRIVATE (self)->pack_dir, name);
+ g_free (name);
+
+ return ret;
+}
+
+static GFile *
+get_pack_content_path (OstreeRepo *self,
+ const char *checksum)
+{
+ char *name;
+ GFile *ret;
+
+ name = g_strconcat ("ostpack-", checksum, ".data", NULL);
+ ret = g_file_get_child (GET_PRIVATE (self)->pack_dir, name);
+ g_free (name);
+
+ return ret;
+}
+
+gboolean
+ostree_repo_load_pack_index (OstreeRepo *self,
+ const char *sha256,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ GVariant *ret_variant = NULL;
+ GFile *path = NULL;
+
+ ret_variant = g_hash_table_lookup (priv->pack_index_mappings, sha256);
+ if (ret_variant)
+ {
+ g_variant_ref (ret_variant);
+ }
+ else
+ {
+ path = get_pack_index_path (self, sha256);
+ if (!ot_util_variant_map (path,
+ OSTREE_PACK_INDEX_VARIANT_FORMAT,
+ &ret_variant, error))
+ goto out;
+ g_hash_table_insert (priv->pack_index_mappings, g_strdup (sha256),
+ g_variant_ref (ret_variant));
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_variant, &ret_variant);
+ out:
+ g_clear_object (&path);
+ ot_clear_gvariant (&ret_variant);
+ return ret;
+}
+
+static gboolean
+list_objects_in_index (OstreeRepo *self,
+ const char *pack_checksum,
+ GHashTable *inout_objects,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GFile *index_path = NULL;
+ GVariant *index_variant = NULL;
+ GVariant *contents;
+ GVariantIter content_iter;
+ const char *checksum;
+ guint32 objtype_u32;
+ guint64 offset;
+
+ index_path = get_pack_index_path (self, pack_checksum);
+
+ if (!ostree_repo_load_pack_index (self, pack_checksum, &index_variant, cancellable, error))
+ goto out;
+
+ contents = g_variant_get_child_value (index_variant, 3);
+ g_variant_iter_init (&content_iter, contents);
+
+ while (g_variant_iter_loop (&content_iter, "(&sut)", &checksum, &objtype_u32, &offset))
+ {
+ GVariant *obj_key;
+ GVariant *objdata;
+ OstreeObjectType objtype = (OstreeObjectType)objtype_u32;
+ GVariantBuilder pack_contents_builder;
+ gboolean is_loose;
+
+ g_variant_builder_init (&pack_contents_builder,
+ G_VARIANT_TYPE_STRING_ARRAY);
+
+ obj_key = ostree_object_name_serialize (checksum, objtype);
+
+ objdata = g_hash_table_lookup (inout_objects, obj_key);
+ if (!objdata)
+ {
+ is_loose = FALSE;
+ }
+ else
+ {
+ GVariantIter *current_packs_iter;
+ const char *current_pack_checksum;
+
+ g_variant_get (objdata, "bas", &is_loose, ¤t_packs_iter);
+
+ while (g_variant_iter_loop (current_packs_iter, "&s", ¤t_pack_checksum))
+ {
+ g_variant_builder_add (&pack_contents_builder, current_pack_checksum);
+ }
+ g_variant_iter_free (current_packs_iter);
+ }
+ g_variant_builder_add (&pack_contents_builder, pack_checksum);
+ objdata = g_variant_new ("(b as)", is_loose,
+ g_variant_builder_end (&pack_contents_builder));
+ g_hash_table_replace (inout_objects,
+ ot_util_variant_take_ref (obj_key),
+ ot_util_variant_take_ref (objdata));
+ }
+
+ ret = TRUE;
+ out:
+ g_clear_object (&index_path);
+ ot_clear_gvariant (&index_variant);
+ ot_clear_gvariant (&contents);
+ return ret;
+}
+
+static gboolean
+list_packed_objects (OstreeRepo *self,
+ GHashTable *inout_objects,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GPtrArray *index_checksums = NULL;
+ guint i;
+
+ if (!ostree_repo_list_pack_indexes (self, &index_checksums, cancellable, error))
+ goto out;
+
+ for (i = 0; i < index_checksums->len; i++)
+ {
+ const char *checksum = index_checksums->pdata[i];
+
+ if (!list_objects_in_index (self, checksum, inout_objects, cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ if (index_checksums)
+ g_ptr_array_unref (index_checksums);
+ return ret;
+}
+
+gboolean
+ostree_repo_list_pack_indexes (OstreeRepo *self,
+ GPtrArray **out_indexes,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GPtrArray *index_files = NULL;
+ GPtrArray *ret_indexes = NULL;
+ OstreeRepoPrivate *priv = GET_PRIVATE (self);
+ guint i;
+
+ if (!list_files_in_dir_matching (priv->pack_dir,
+ "ostpack-", ".index",
+ &index_files,
+ cancellable, error))
+ goto out;
+
+ ret_indexes = g_ptr_array_new_with_free_func ((GDestroyNotify)g_free);
+ for (i = 0; i < index_files->len; i++)
+ {
+ GFile *index_path = index_files->pdata[i];
+ const char *basename = ot_gfile_get_basename_cached (index_path);
+ g_ptr_array_add (ret_indexes, get_checksum_from_pack_name (basename));
+ }
+
+ ret = TRUE;
+ ot_transfer_out_value (out_indexes, &ret_indexes);
+ out:
+ if (ret_indexes)
+ g_ptr_array_unref (ret_indexes);
+ return ret;
+}
+
/**
* ostree_repo_list_objects:
* @self:
@@ -2494,6 +2803,12 @@ ostree_repo_list_objects (OstreeRepo *self,
goto out;
}
+ if (flags & OSTREE_REPO_LIST_OBJECTS_PACKED)
+ {
+ if (!list_packed_objects (self, ret_objects, cancellable, error))
+ goto out;
+ }
+
ret = TRUE;
ot_transfer_out_value (out_objects, &ret_objects);
out:
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 0d3ae1b..09ec1ae 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -141,6 +141,12 @@ gboolean ostree_repo_load_variant (OstreeRepo *self,
GVariant **out_variant,
GError **error);
+gboolean ostree_repo_load_pack_index (OstreeRepo *self,
+ const char *sha256,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error);
+
typedef enum {
OSTREE_REPO_COMMIT_FILTER_ALLOW,
OSTREE_REPO_COMMIT_FILTER_SKIP
@@ -237,10 +243,8 @@ typedef enum {
*
* b - %TRUE if object is available "loose"
* as - List of pack file checksums in which this object appears
- * t - For loose objects, Unix ctime (seconds)
- * t - object size
*/
-#define OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE (G_VARIANT_TYPE ("(bastt)")
+#define OSTREE_REPO_LIST_OBJECTS_VARIANT_TYPE (G_VARIANT_TYPE ("(bas)")
gboolean ostree_repo_list_objects (OstreeRepo *self,
OstreeRepoListObjectsFlags flags,
@@ -248,6 +252,11 @@ gboolean ostree_repo_list_objects (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
+gboolean ostree_repo_list_pack_indexes (OstreeRepo *self,
+ GPtrArray **out_indexes,
+ GCancellable *cancellable,
+ GError **error);
+
G_END_DECLS
#endif /* _OSTREE_REPO */
diff --git a/src/ostree/ot-builtin-init.c b/src/ostree/ot-builtin-init.c
index ba16254..1bc5392 100644
--- a/src/ostree/ot-builtin-init.c
+++ b/src/ostree/ot-builtin-init.c
@@ -70,7 +70,7 @@ ostree_builtin_init (int argc, char **argv, GFile *repo_path, GError **error)
goto out;
g_clear_object (&grandchild);
- grandchild = g_file_get_child (child, "packs");
+ grandchild = g_file_get_child (child, "pack");
if (!g_file_make_directory (grandchild, NULL, error))
goto out;
diff --git a/src/ostree/ot-builtin-repack.c b/src/ostree/ot-builtin-repack.c
index cbf92ff..3bde4be 100644
--- a/src/ostree/ot-builtin-repack.c
+++ b/src/ostree/ot-builtin-repack.c
@@ -35,6 +35,7 @@
#define OT_GZIP_COMPRESSION_LEVEL (8)
static gboolean opt_analyze_only;
+static gboolean opt_ls;
static char* opt_pack_size;
static char* opt_int_compression;
static char* opt_ext_compression;
@@ -50,6 +51,7 @@ static GOptionEntry options[] = {
{ "internal-compression", 0, 0, G_OPTION_ARG_STRING, &opt_int_compression, "Compress objects using COMPRESSION", "COMPRESSION" },
{ "external-compression", 0, 0, G_OPTION_ARG_STRING, &opt_ext_compression, "Compress entire packfiles using COMPRESSION", "COMPRESSION" },
{ "analyze-only", 0, 0, G_OPTION_ARG_NONE, &opt_analyze_only, "Just analyze current state", NULL },
+ { "ls", 0, 0, G_OPTION_ARG_NONE, &opt_ls, "Print packfiles", NULL },
{ NULL }
};
@@ -74,64 +76,6 @@ typedef struct {
GPid compress_child_pid;
} OtBuildRepackFile;
-static GPtrArray *
-get_xz_args (void)
-{
- GPtrArray *ret = g_ptr_array_new ();
-
- g_ptr_array_add (ret, "xz");
- g_ptr_array_add (ret, "--memlimit-compress=512M");
-
- return ret;
-}
-
-static void
-compressor_child_setup (gpointer user_data)
-{
- int stdout_fd = GPOINTER_TO_INT (user_data);
-
- if (dup2 (stdout_fd, 1) < 0)
- g_assert_not_reached ();
- (void) close (stdout_fd);
-}
-
-static gboolean
-create_compressor_subprocess (OtBuildRepackFile *self,
- GPtrArray *compressor_argv,
- GFile *destfile,
- GOutputStream **out_output,
- GCancellable *cancellable,
- GError **error)
-{
- gboolean ret = FALSE;
- GOutputStream *ret_output = NULL;
- int stdin_pipe_fd;
- int target_stdout_fd;
-
- target_stdout_fd = open (ot_gfile_get_path_cached (destfile), O_WRONLY);
- if (target_stdout_fd < 0)
- {
- ot_util_set_error_from_errno (error, errno);
- goto out;
- }
-
- if (!g_spawn_async_with_pipes (NULL, (char**)compressor_argv->pdata, NULL,
- G_SPAWN_SEARCH_PATH, NULL, NULL,
- &self->compress_child_pid, &stdin_pipe_fd,
- compressor_child_setup, GINT_TO_POINTER (target_stdout_fd), error))
- goto out;
-
- (void) close (target_stdout_fd);
-
- ret_output = g_unix_output_stream_new (stdin_pipe_fd, TRUE);
-
- ret = TRUE;
- ot_transfer_out_value (out_output, &ret_output);
- out:
- g_clear_object (&ret_output);
- return ret;
-}
-
static gint
compare_object_data_by_size (gconstpointer ap,
gconstpointer bp)
@@ -152,37 +96,88 @@ compare_object_data_by_size (gconstpointer ap,
}
static gboolean
-write_aligned_variant (GOutputStream *output,
- GVariant *variant,
- GChecksum *checksum,
- guint64 *inout_offset,
- GCancellable *cancellable,
- GError **error)
+write_bytes_update_checksum (GOutputStream *output,
+ gconstpointer bytes,
+ gsize len,
+ GChecksum *checksum,
+ guint64 *inout_offset,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
- guint padding;
gsize bytes_written;
- char padding_nuls[7] = {0, 0, 0, 0, 0, 0, 0};
- padding = 8 - ((*inout_offset) & 7);
-
- if (padding > 0)
+ if (len > 0)
{
- g_checksum_update (checksum, (guchar*) padding_nuls, padding);
- if (!g_output_stream_write_all (output, padding_nuls, padding, &bytes_written,
+ g_checksum_update (checksum, (guchar*) bytes, len);
+ if (!g_output_stream_write_all (output, bytes, len, &bytes_written,
cancellable, error))
goto out;
- g_assert (bytes_written == padding);
- *inout_offset += padding;
+ g_assert (bytes_written == len);
+ *inout_offset += len;
}
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+static gboolean
+write_padding (GOutputStream *output,
+ guint alignment,
+ GChecksum *checksum,
+ guint64 *inout_offset,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ guint padding;
+ char padding_nuls[7] = {0, 0, 0, 0, 0, 0, 0};
+
+ if (alignment == 8)
+ padding = 8 - ((*inout_offset) & 7);
+ else
+ padding = 4 - ((*inout_offset) & 3);
+
+ if (!write_bytes_update_checksum (output, (guchar*)padding_nuls, padding,
+ checksum, inout_offset, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+static gboolean
+write_variant_with_size (GOutputStream *output,
+ GVariant *variant,
+ GChecksum *checksum,
+ guint64 *inout_offset,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ guint64 variant_size;
+ guint32 variant_size_u32_be;
+
+ g_assert ((*inout_offset & 7) == 0);
+
+ /* Write variant size */
+ variant_size = g_variant_get_size (variant);
+ g_assert (variant_size < G_MAXUINT32);
+ variant_size_u32_be = GUINT32_TO_BE((guint32) variant_size);
- g_checksum_update (checksum, (guchar*) g_variant_get_data (variant),
- g_variant_get_size (variant));
- if (!g_output_stream_write_all (output, g_variant_get_data (variant),
- g_variant_get_size (variant), &bytes_written,
- cancellable, error))
+ if (!write_bytes_update_checksum (output, (guchar*)&variant_size_u32_be, 4,
+ checksum, inout_offset, cancellable, error))
+ goto out;
+
+ /* Pad to offset of 8, write variant */
+ if (!write_padding (output, 8, checksum, inout_offset, cancellable, error))
+ goto out;
+ if (!write_bytes_update_checksum (output, g_variant_get_data (variant),
+ variant_size, checksum,
+ inout_offset, cancellable, error))
goto out;
- *inout_offset += bytes_written;
ret = TRUE;
out:
@@ -240,7 +235,7 @@ create_pack_file (OtRepackData *data,
offset = 0;
pack_checksum = g_checksum_new (G_CHECKSUM_SHA256);
- g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(st)"));
+ g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(sut)"));
index_content_builder_initialized = TRUE;
pack_header = g_variant_new ("(su a{sv}u)",
@@ -248,26 +243,33 @@ create_pack_file (OtRepackData *data,
g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
objects->len);
- if (!write_aligned_variant (pack_out, pack_header, pack_checksum, &offset,
- cancellable, error))
+ if (!write_variant_with_size (pack_out, pack_header, pack_checksum, &offset,
+ cancellable, error))
goto out;
for (i = 0; i < objects->len; i++)
{
GVariant *object_data = objects->pdata[i];
- guint64 objsize;
const char *checksum;
guint32 objtype_u32;
OstreeObjectType objtype;
char buf[4096];
guint64 obj_bytes_written;
+ guint64 expected_objsize;
+ guint64 objsize;
GOutputStream *write_pack_out;
guchar entry_flags;
- g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &objsize);
+ g_variant_get (object_data, "(&sut)", &checksum, &objtype_u32, &expected_objsize);
objtype = (OstreeObjectType) objtype_u32;
+ if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error))
+ goto out;
+
+ /* offset points to aligned header size */
+ g_variant_builder_add (&index_content_builder, "(sut)", checksum, (guint32)objtype, offset);
+
ot_clear_gvariant (&object_header);
entry_flags = 0;
if (data->int_compression != OT_COMPRESSION_NONE)
@@ -284,12 +286,6 @@ create_pack_file (OtRepackData *data,
}
}
- object_header = g_variant_new ("(yst)", GUINT32_TO_BE (entry_flags), checksum, (guint32)objtype, objsize);
-
- if (!write_aligned_variant (pack_out, object_header, pack_checksum,
- &offset, cancellable, error))
- goto out;
-
g_clear_object (&object_path);
object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
@@ -298,6 +294,25 @@ create_pack_file (OtRepackData *data,
if (!object_input)
goto out;
+ g_clear_object (&object_file_info);
+ object_file_info = g_file_input_stream_query_info (object_input, OSTREE_GIO_FAST_QUERYINFO, cancellable, error);
+ if (!object_file_info)
+ goto out;
+
+ objsize = g_file_info_get_attribute_uint64 (object_file_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
+
+ g_assert_cmpint (objsize, ==, expected_objsize);
+
+ ot_clear_gvariant (&object_header);
+ object_header = g_variant_new ("(tuys)", GUINT64_TO_BE (objsize),
+ GUINT32_TO_BE ((guint32)objtype),
+ GUINT32_TO_BE (entry_flags),
+ checksum);
+
+ if (!write_variant_with_size (pack_out, object_header, pack_checksum,
+ &offset, cancellable, error))
+ goto out;
+
if (data->int_compression != OT_COMPRESSION_NONE)
{
g_clear_object (&compressor);
@@ -345,7 +360,6 @@ create_pack_file (OtRepackData *data,
g_assert_cmpint (obj_bytes_written, ==, objsize);
- g_variant_builder_add (&index_content_builder, "(st)", checksum, offset);
}
if (!g_output_stream_close (pack_out, cancellable, error))
@@ -370,7 +384,7 @@ create_pack_file (OtRepackData *data,
}
g_clear_object (&pack_temppath);
- index_content = g_variant_new ("(su a{sv}@a(st))",
+ index_content = g_variant_new ("(su a{sv}@a(sut))",
"OSTPACKINDEX", GUINT32_TO_BE(0),
g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
g_variant_builder_end (&index_content_builder));
@@ -432,22 +446,29 @@ create_pack_file (OtRepackData *data,
/**
* cluster_objects_stupidly:
+ * @objects: Map from serialized object name to objdata
+ * @out_clusters: (out): [Array of [Array of object data]]. Free with g_ptr_array_unref().
*
- * Just sorts by size currently.
- *
- * Returns: [Array of [Array of object data]]. Free with g_ptr_array_unref().
+ * Just sorts by size currently. Also filters out non-regular object
+ * content.
*/
-static GPtrArray *
+static gboolean
cluster_objects_stupidly (OtRepackData *data,
- GHashTable *objects)
+ GHashTable *objects,
+ GPtrArray **out_clusters,
+ GCancellable *cancellable,
+ GError **error)
{
- GPtrArray *ret = NULL;
+ gboolean ret = FALSE;
+ GPtrArray *ret_clusters = NULL;
GPtrArray *object_list = NULL;
guint i;
guint64 current_size;
guint current_offset;
GHashTableIter hash_iter;
gpointer key, value;
+ GFile *object_path = NULL;
+ GFileInfo *object_info = NULL;
object_list = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
@@ -456,22 +477,40 @@ cluster_objects_stupidly (OtRepackData *data,
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
- GVariant *objdata = value;
const char *checksum;
OstreeObjectType objtype;
guint64 size;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
- g_variant_get_child (objdata, 3, "t", &size);
+ g_clear_object (&object_path);
+ object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
+
+ if (objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_META)
+ {
+ /* Counted under files */
+ continue;
+ }
+
+ g_clear_object (&object_info);
+ object_info = g_file_query_info (object_path, OSTREE_GIO_FAST_QUERYINFO,
+ G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+ cancellable, error);
+ if (!object_info)
+ goto out;
+
+ if (g_file_info_get_file_type (object_info) != G_FILE_TYPE_REGULAR)
+ continue;
+
+ size = g_file_info_get_attribute_uint64 (object_info, G_FILE_ATTRIBUTE_STANDARD_SIZE);
g_ptr_array_add (object_list,
- g_variant_new ("(sut)", checksum, objtype, size));
+ g_variant_ref_sink (g_variant_new ("(sut)", checksum, objtype, size)));
}
g_ptr_array_sort (object_list, compare_object_data_by_size);
- ret = g_ptr_array_new ();
+ ret_clusters = g_ptr_array_new ();
current_size = 0;
current_offset = 0;
@@ -488,9 +527,9 @@ cluster_objects_stupidly (OtRepackData *data,
GPtrArray *current = g_ptr_array_new ();
for (j = current_offset; j < i; j++)
{
- g_ptr_array_add (current, object_list->pdata[j]);
+ g_ptr_array_add (current, g_variant_ref (object_list->pdata[j]));
}
- g_ptr_array_add (ret, current);
+ g_ptr_array_add (ret_clusters, current);
current_size = objsize;
current_offset = i;
}
@@ -504,6 +543,9 @@ cluster_objects_stupidly (OtRepackData *data,
}
}
+ ret = TRUE;
+ ot_transfer_out_value (out_clusters, &ret_clusters);
+ out:
if (object_list)
g_ptr_array_unref (object_list);
return ret;
@@ -593,6 +635,14 @@ parse_compression_string (const char *compstr,
return ret;
}
+static gboolean
+do_ls (OtRepackData *data,
+ GCancellable *cancellable,
+ GError **error)
+{
+ return FALSE;
+}
+
gboolean
ostree_builtin_repack (int argc, char **argv, GFile *repo_path, GError **error)
{
@@ -603,7 +653,6 @@ ostree_builtin_repack (int argc, char **argv, GFile *repo_path, GError **error)
GHashTable *objects = NULL;
GCancellable *cancellable = NULL;
guint i;
- guint64 total_size;
GPtrArray *clusters = NULL;
GHashTableIter hash_iter;
gpointer key, value;
@@ -636,14 +685,11 @@ ostree_builtin_repack (int argc, char **argv, GFile *repo_path, GError **error)
g_hash_table_iter_init (&hash_iter, objects);
- total_size = 0;
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
GVariant *serialized_key = key;
- GVariant *objdata = value;
const char *checksum;
OstreeObjectType objtype;
- guint64 size;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
@@ -666,35 +712,39 @@ ostree_builtin_repack (int argc, char **argv, GFile *repo_path, GError **error)
/* Counted under files */
break;
}
-
- g_variant_get_child (objdata, 3, "t", &size);
-
- total_size += size;
}
g_print ("Commits: %u\n", data.n_commits);
g_print ("Tree contents: %u\n", data.n_dirtree);
g_print ("Tree meta: %u\n", data.n_dirmeta);
g_print ("Files: %u\n", data.n_files);
- g_print ("Total size: %" G_GUINT64_FORMAT "\n", total_size);
g_print ("\n");
g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data.pack_size);
- clusters = cluster_objects_stupidly (&data, objects);
+ if (opt_ls)
+ {
+ if (!do_ls (&data, cancellable, error))
+ goto out;
+ }
+ else
+ {
+ if (!cluster_objects_stupidly (&data, objects, &clusters, cancellable, error))
+ goto out;
- g_print ("Going to create %u packfiles\n", clusters->len);
+ g_print ("Going to create %u packfiles\n", clusters->len);
- for (i = 0; i < clusters->len; i++)
- {
- GPtrArray *cluster = clusters->pdata[i];
+ for (i = 0; i < clusters->len; i++)
+ {
+ GPtrArray *cluster = clusters->pdata[i];
- g_print ("%u: %u objects\n", i, cluster->len);
+ g_print ("%u: %u objects\n", i, cluster->len);
- if (!opt_analyze_only)
- {
- if (!create_pack_file (&data, cluster, cancellable, error))
- goto out;
+ if (!opt_analyze_only)
+ {
+ if (!create_pack_file (&data, cluster, cancellable, error))
+ goto out;
+ }
}
}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
