[ostree/wip/packfile-rebase2] checkout from pack
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree/wip/packfile-rebase2] checkout from pack
- Date: Wed, 28 Mar 2012 00:48:31 +0000 (UTC)
commit d10d87288e30ec24693a08a96a845d9ca0444676
Author: Colin Walters <walters verbum org>
Date: Tue Mar 27 09:59:29 2012 -0400
checkout from pack
src/libostree/ostree-repo-file.c | 31 ++++++----
src/libostree/ostree-repo.c | 116 +++++++++++++++++++++++++-------------
src/ostree/ot-builtin-repack.c | 107 +++++++++++++++++------------------
tests/t0001-archive.sh | 13 +++--
4 files changed, 156 insertions(+), 111 deletions(-)
---
diff --git a/src/libostree/ostree-repo-file.c b/src/libostree/ostree-repo-file.c
index 0370bcc..ac4b129 100644
--- a/src/libostree/ostree-repo-file.c
+++ b/src/libostree/ostree-repo-file.c
@@ -724,39 +724,46 @@ bsearch_in_file_variant (GVariant *variant,
const char *name,
int *out_pos)
{
- int i, n;
- int m;
+ gsize imax, imin;
+ gsize imid;
+ gsize n;
- i = 0;
- n = g_variant_n_children (variant) - 1;
- m = 0;
+ n = g_variant_n_children (variant);
+ if (n == 0)
+ return FALSE;
- while (i <= n)
+ imax = n - 1;
+ imin = 0;
+ while (imax >= imin)
{
GVariant *child;
const char *cur;
int cmp;
- m = i + ((n - i) / 2);
+ imid = (imin + imax) / 2;
- child = g_variant_get_child_value (variant, m);
+ child = g_variant_get_child_value (variant, imid);
g_variant_get_child (child, 0, "&s", &cur, NULL);
cmp = strcmp (cur, name);
if (cmp < 0)
- i = m + 1;
+ imin = imid + 1;
else if (cmp > 0)
- n = m - 1;
+ {
+ if (imid == 0)
+ break;
+ imax = imid - 1;
+ }
else
{
ot_clear_gvariant (&child);
- *out_pos = m;
+ *out_pos = imid;
return TRUE;
}
ot_clear_gvariant (&child);
}
- *out_pos = m;
+ *out_pos = imid;
return FALSE;
}
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 51b0508..a12ba6f 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -40,6 +40,9 @@
#include "ostree-libarchive-input-stream.h"
#endif
+#define ALIGN_VALUE(this, boundary) \
+ (( ((unsigned long)(this)) + (((unsigned long)(boundary)) -1)) & (~(((unsigned long)(boundary))-1)))
+
enum {
PROP_0,
@@ -2590,13 +2593,12 @@ ostree_repo_map_pack_file (OstreeRepo *self,
ret_len = (guint64)g_mapped_file_get_length (map);
ret = TRUE;
- ot_transfer_out_value (out_data, &ret_data);
+ if (out_data)
+ *out_data = ret_data;
if (out_len)
*out_len = ret_len;
out:
g_clear_object (&path);
- if (ret_data)
- g_mapped_file_unref (ret_data);
return ret;
}
@@ -2606,25 +2608,32 @@ bsearch_in_pack_index (GVariant *index_contents,
OstreeObjectType objtype,
guint64 *out_offset)
{
+ gsize imax, imin;
gsize n;
- gsize i;
- gsize m;
guint32 target_objtype = (guint32) objtype;
- i = 0;
- n = g_variant_n_children (index_contents) - 1;
- m = 0;
+ n = g_variant_n_children (index_contents);
+
+ if (n == 0)
+ return FALSE;
- while (i <= n)
+ imax = n - 1;
+ imin = 0;
+ while (imax >= imin)
{
GVariant *cur_csum_bytes;
guint32 cur_objtype;
guint64 cur_offset;
+ gsize imid;
int c;
- m = i + ((n - i) / 2);
+ imid = (imin + imax) / 2;
+
+ g_variant_get_child (index_contents, imid, "(u ayt)", &cur_objtype,
+ &cur_csum_bytes, &cur_offset);
+ cur_objtype = GUINT32_FROM_BE (cur_objtype);
+ cur_offset = GUINT64_FROM_BE (cur_offset);
- g_variant_get_child (index_contents, m, "(u ayt)", &cur_objtype, &cur_csum_bytes, &cur_offset);
c = ostree_cmp_checksum_bytes (cur_csum_bytes, csum_bytes);
if (c == 0)
{
@@ -2636,9 +2645,13 @@ bsearch_in_pack_index (GVariant *index_contents,
g_variant_unref (cur_csum_bytes);
if (c < 0)
- i = m + 1;
+ imin = imid + 1;
else if (c > 0)
- n = m - 1;
+ {
+ if (imid == 0)
+ break;
+ imax = imid - 1;
+ }
else
{
*out_offset = cur_offset;
@@ -2679,23 +2692,24 @@ read_pack_entry (gboolean trusted,
goto out;
}
- g_assert ((((guint64)pack_data+offset) & 0x3) == 0);
- entry_len = GUINT32_FROM_BE (*((guint32*)(pack_data+offset)));
- entry_end = offset + entry_len;
- if (G_UNLIKELY (!(entry_end < pack_len)))
+ entry_start = ALIGN_VALUE (offset + 4, 8);
+ if (G_UNLIKELY (!(entry_start < pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Corrupted pack index; out of range entry length %u",
- entry_len);
+ "Corrupted pack index; out of range data offset %" G_GUINT64_FORMAT,
+ entry_start);
goto out;
}
- entry_start = offset + 4;
- if (G_UNLIKELY (!(entry_start < pack_len)))
+ g_assert ((((guint64)pack_data+offset) & 0x3) == 0);
+ entry_len = GUINT32_FROM_BE (*((guint32*)(pack_data+offset)));
+
+ entry_end = entry_start + entry_len;
+ if (G_UNLIKELY (!(entry_end < pack_len)))
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Corrupted pack index; out of range data offset %" G_GUINT64_FORMAT,
- entry_start);
+ "Corrupted pack index; out of range entry length %u",
+ entry_len);
goto out;
}
@@ -2721,7 +2735,6 @@ get_pack_entry_stream (GVariant *pack_entry)
g_variant_get_child (pack_entry, 1, "y", &entry_flags);
g_variant_get_child (pack_entry, 3, "@ay", &pack_data);
- g_variant_ref_sink (pack_data);
data_ptr = g_variant_get_fixed_array (pack_data, &data_len, 1);
memory_input = g_memory_input_stream_new_from_data (data_ptr, data_len, NULL);
@@ -2749,22 +2762,37 @@ get_pack_entry_stream (GVariant *pack_entry)
return ret_input;
}
-static GVariant *
+static gboolean
get_pack_entry_as_variant (GVariant *pack_entry,
const GVariantType *type,
- gboolean trusted)
+ gboolean trusted,
+ GVariant **out_variant,
+ GCancellable *cancellable,
+ GError **error)
{
- GVariant *pack_data;
+ gboolean ret = FALSE;
+ GInputStream *stream;
+ GMemoryOutputStream *data_stream;
GVariant *ret_variant;
- gconstpointer data_ptr;
- gsize data_len;
- g_variant_get_child (pack_entry, 3, "@ay", &pack_data);
- data_ptr = g_variant_get_fixed_array (pack_data, &data_len, 1);
- ret_variant = g_variant_new_from_data (type, data_ptr, data_len, trusted,
- (GDestroyNotify)g_variant_unref,
- pack_data);
- return ret_variant;
+ stream = get_pack_entry_stream (pack_entry);
+
+ data_stream = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
+
+ if (!g_output_stream_splice ((GOutputStream*)data_stream, stream,
+ G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
+ cancellable, error))
+ goto out;
+
+ ret_variant = g_variant_new_from_data (type, g_memory_output_stream_get_data (data_stream),
+ g_memory_output_stream_get_data_size (data_stream),
+ trusted, (GDestroyNotify) g_object_unref, data_stream);
+
+ ret = TRUE;
+ g_variant_ref_sink (ret_variant);
+ ot_transfer_out_value (out_variant, &ret_variant);
+ out:
+ return ret;
}
gboolean
@@ -3056,12 +3084,12 @@ find_object_in_packs (OstreeRepo *self,
goto out;
ot_clear_gvariant (&index_contents);
- index_contents = g_variant_get_child_value (index_variant, 3);
+ index_contents = g_variant_get_child_value (index_variant, 2);
if (!bsearch_in_pack_index (index_contents, csum_bytes, objtype, &offset))
continue;
- ret_pack_checksum = g_strdup (checksum);
+ ret_pack_checksum = g_strdup (pack_checksum);
ret_pack_offset = offset;
break;
}
@@ -3174,7 +3202,7 @@ ostree_repo_load_variant (OstreeRepo *self,
if (!ostree_map_metadata_file (object_path, expected_type, &ret_variant, error))
goto out;
}
- else
+ else if (pack_checksum != NULL)
{
guint32 actual_type;
@@ -3186,12 +3214,13 @@ ostree_repo_load_variant (OstreeRepo *self,
cancellable, error))
goto out;
- container_variant = get_pack_entry_as_variant (packed_object, OSTREE_SERIALIZED_VARIANT_FORMAT, TRUE);
+ if (!get_pack_entry_as_variant (packed_object, OSTREE_SERIALIZED_VARIANT_FORMAT, TRUE, &container_variant,
+ cancellable, error))
+ goto out;
g_variant_get (container_variant, "(uv)",
&actual_type, &ret_variant);
actual_type = GUINT32_FROM_BE (actual_type);
- ot_util_variant_take_ref (ret_variant);
if (actual_type != expected_type)
{
@@ -3201,6 +3230,13 @@ ostree_repo_load_variant (OstreeRepo *self,
goto out;
}
}
+ else
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "No such metadata object %s.%s",
+ sha256, ostree_object_type_to_string (expected_type));
+ goto out;
+ }
ret = TRUE;
ot_transfer_out_value (out_variant, &ret_variant);
diff --git a/src/ostree/ot-builtin-repack.c b/src/ostree/ot-builtin-repack.c
index f2e7ce5..9bbbce6 100644
--- a/src/ostree/ot-builtin-repack.c
+++ b/src/ostree/ot-builtin-repack.c
@@ -109,8 +109,8 @@ write_bytes_update_checksum (GOutputStream *output,
if (!g_output_stream_write_all (output, bytes, len, &bytes_written,
cancellable, error))
goto out;
- g_assert (bytes_written == len);
- *inout_offset += len;
+ g_assert_cmpint (bytes_written, ==, len);
+ *inout_offset += bytes_written;
}
ret = TRUE;
@@ -128,7 +128,8 @@ write_padding (GOutputStream *output,
{
gboolean ret = FALSE;
guint bits;
- char padding_nuls[7] = {0, 0, 0, 0, 0, 0, 0};
+ guint padding_len;
+ guchar padding_nuls[8] = {0, 0, 0, 0, 0, 0, 0, 0};
if (alignment == 8)
bits = ((*inout_offset) & 7);
@@ -137,7 +138,8 @@ write_padding (GOutputStream *output,
if (bits > 0)
{
- if (!write_bytes_update_checksum (output, (guchar*)padding_nuls, alignment - bits,
+ padding_len = alignment - bits;
+ if (!write_bytes_update_checksum (output, (guchar*)padding_nuls, padding_len,
checksum, inout_offset, cancellable, error))
goto out;
}
@@ -173,6 +175,8 @@ write_variant_with_size (GOutputStream *output,
/* Pad to offset of 8, write variant */
if (!write_padding (output, 8, checksum, inout_offset, cancellable, error))
goto out;
+ g_assert ((*inout_offset & 7) == 0);
+
if (!write_bytes_update_checksum (output, g_variant_get_data (variant),
variant_size, checksum,
inout_offset, cancellable, error))
@@ -201,6 +205,8 @@ compare_index_content (gconstpointer ap,
g_variant_get (a_v, "(u ayt)", &a_objtype, &a_csum_bytes, &a_offset);
g_variant_get (b_v, "(u ayt)", &b_objtype, &b_csum_bytes, &b_offset);
+ a_objtype = GUINT32_FROM_BE (a_objtype);
+ b_objtype = GUINT32_FROM_BE (b_objtype);
c = ostree_cmp_checksum_bytes (a_csum_bytes, b_csum_bytes);
if (c == 0)
{
@@ -292,36 +298,17 @@ create_pack_file (OtRepackData *data,
objtype = (OstreeObjectType) objtype_u32;
- if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error))
- goto out;
-
- /* offset points to aligned header size */
- index_entry = g_variant_new ("(u ayt)",
- GUINT32_TO_BE ((guint32)objtype),
- ostree_checksum_to_bytes (checksum),
- offset);
- g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry));
-
- if (objtype == OSTREE_OBJECT_TYPE_DIR_TREE
- || objtype == OSTREE_OBJECT_TYPE_DIR_META
- || objtype == OSTREE_OBJECT_TYPE_COMMIT)
+ switch (data->int_compression)
{
- ;
- }
- else
- {
- switch (data->int_compression)
- {
- case OT_COMPRESSION_GZIP:
- {
- entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP;
- break;
- }
- default:
- {
- g_assert_not_reached ();
- }
- }
+ case OT_COMPRESSION_GZIP:
+ {
+ entry_flags |= OSTREE_PACK_FILE_ENTRY_FLAG_GZIP;
+ break;
+ }
+ default:
+ {
+ g_assert_not_reached ();
+ }
}
g_clear_object (&object_path);
@@ -369,7 +356,7 @@ create_pack_file (OtRepackData *data,
if (bytes_read > 0)
{
g_checksum_update (pack_checksum, (guint8*)buf, bytes_read);
- if (!g_output_stream_write_all (pack_out, buf, bytes_read, &bytes_written, cancellable, error))
+ if (!g_output_stream_write_all ((GOutputStream*)object_data_stream, buf, bytes_read, &bytes_written, cancellable, error))
goto out;
}
}
@@ -379,14 +366,27 @@ create_pack_file (OtRepackData *data,
goto out;
ot_clear_gvariant (&packed_object);
- packed_object = g_variant_new ("(uy ay@ay)", GUINT32_TO_BE ((guint32)objtype),
- entry_flags,
- g_variant_new_fixed_array (G_VARIANT_TYPE ("y"),
- g_memory_output_stream_get_data (object_data_stream),
- g_memory_output_stream_get_data_size (object_data_stream),
- 1),
- ostree_checksum_to_bytes (checksum));
- g_clear_object (&object_data_stream);
+ {
+ guchar *data = g_memory_output_stream_get_data (object_data_stream);
+ gsize data_len = g_memory_output_stream_get_data_size (object_data_stream);
+ packed_object = g_variant_new ("(uy ay@ay)", GUINT32_TO_BE ((guint32)objtype),
+ entry_flags,
+ ostree_checksum_to_bytes (checksum),
+ g_variant_new_fixed_array (G_VARIANT_TYPE ("y"),
+ data, data_len,
+ 1));
+ g_clear_object (&object_data_stream);
+ }
+
+ if (!write_padding (pack_out, 4, pack_checksum, &offset, cancellable, error))
+ goto out;
+
+ /* offset points to aligned header size */
+ index_entry = g_variant_new ("(u ayt)",
+ GUINT32_TO_BE ((guint32)objtype),
+ ostree_checksum_to_bytes (checksum),
+ GUINT64_TO_BE (offset));
+ g_ptr_array_add (index_content_list, g_variant_ref_sink (index_entry));
if (!write_variant_with_size (pack_out, packed_object, pack_checksum,
&offset, cancellable, error))
@@ -554,14 +554,19 @@ cluster_objects_stupidly (OtRepackData *data,
if (current_size + objsize > data->pack_size || i == (object_list->len - 1))
{
guint j;
- GPtrArray *current = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
- for (j = current_offset; j < i; j++)
+ GPtrArray *current;
+
+ if (current_offset < i)
{
- g_ptr_array_add (current, g_variant_ref (object_list->pdata[j]));
+ current = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
+ for (j = current_offset; j < i; j++)
+ {
+ g_ptr_array_add (current, g_variant_ref (object_list->pdata[j]));
+ }
+ g_ptr_array_add (ret_clusters, current);
+ current_size = objsize;
+ current_offset = i;
}
- g_ptr_array_add (ret_clusters, current);
- current_size = objsize;
- current_offset = i;
}
else if (objsize > data->pack_size)
{
@@ -708,24 +713,20 @@ do_stats_gather_loose (OtRepackData *data,
if (is_loose && is_packed)
{
- g_print ("loose+packed: %s.%u\n", checksum, objtype);
n_loose_and_packed++;
}
else if (is_loose)
{
GVariant *copy = g_variant_ref (serialized_key);
g_hash_table_replace (ret_loose, copy, copy);
- g_print ("loose: %s.%u\n", checksum, objtype);
n_loose++;
}
else if (g_variant_n_children (pack_array) > 1)
{
- g_print ("dup-packed: %s.%u\n", checksum, objtype);
n_dup_packed++;
}
else
{
- g_print ("packed: %s.%u\n", checksum, objtype);
n_packed++;
}
@@ -835,8 +836,6 @@ ostree_builtin_repack (int argc, char **argv, GFile *repo_path, GError **error)
{
GPtrArray *cluster = clusters->pdata[i];
- g_print ("%u: %u objects\n", i, cluster->len);
-
if (!opt_analyze_only)
{
if (!create_pack_file (&data, cluster, cancellable, error))
diff --git a/tests/t0001-archive.sh b/tests/t0001-archive.sh
index b07b988..61dec40 100755
--- a/tests/t0001-archive.sh
+++ b/tests/t0001-archive.sh
@@ -21,7 +21,7 @@ set -e
. libtest.sh
-echo '1..13'
+echo '1..14'
setup_test_repository "archive"
echo "ok setup"
@@ -69,12 +69,15 @@ assert_file_has_content cow-contents "moo"
echo "ok cat-file"
cd ${test_tmpdir}
-$OSTREE repack
+$OSTREE repack --keep-loose
echo "ok repack"
+$OSTREE checkout test2 checkout-test2-from-packed
+echo "ok checkout union 1"
+
cd ${test_tmpdir}
$OSTREE repack
-echo "ok repack again"
+echo "ok repack delete loose"
-$OSTREE checkout test2 checkout-test2-from-packed
-echo "ok checkout union 1"
+$OSTREE repack --analyze-only
+echo "ok repack analyze"
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
