[ostree/wip/packfile-rebase2: 4/11] repack: creating raw packfiles

[Colin Walters <walters src gnome org>]

commit 4d262b23a495fb130742377dc76107b15006136d
Author: Colin Walters <walters verbum org>
Date:   Sun Mar 18 12:13:37 2012 -0400

    repack: creating raw packfiles

 src/libostree/README.md        |   15 +-
 src/libostree/ostree-core.h    |   28 ++-
 src/ostree/ot-builtin-repack.c |  540 ++++++++++++++++++++++++++++++++++++----
 3 files changed, 525 insertions(+), 58 deletions(-)
---
diff --git a/src/libostree/README.md b/src/libostree/README.md
index 04d1462..3ab4730 100644
--- a/src/libostree/README.md
+++ b/src/libostree/README.md
@@ -15,7 +15,7 @@ overhead.  In contrast, a git repository stores copies of
 zlib-compressed data.
 
 Key differences versus git
-----------------------------
+--------------------------
 
  * As mentioned above, extended attributes and owner uid/gid are versioned
  * SHA256 instead of SHA1
@@ -28,3 +28,16 @@ While this is still in planning, I plan to heavily optimize OSTree for
 versioning ELF operating systems.  In industry jargon, this would be
 "content-aware storage".
 
+Related work in storage
+-----------------------
+
+git: http://git-scm.com/
+Venti: http://plan9.bell-labs.com/magic/man2html/6/venti
+Elephant FS: http://www.hpl.hp.com/personal/Alistair_Veitch/papers/elephant-hotos/index.html
+
+Compression
+-----------
+
+xdelta: http://xdelta.org/
+Bsdiff: http://www.daemonology.net/bsdiff/
+xz: http://tukaani.org/xz/
diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h
index 062648e..ab1b3a1 100644
--- a/src/libostree/ostree-core.h
+++ b/src/libostree/ostree-core.h
@@ -97,12 +97,36 @@ typedef enum {
  */
 #define OSTREE_ARCHIVED_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(uuuuusa(ayay))")
 
+/* Pack super index
+ * s - OSTSUPERPACKINDEX
+ * u - Version
+ * a{sv} - Metadata
+ * a(say) - (pack file checksum, bloom filter)
+ */
+#define OSTREE_PACK_SUPER_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sua{sv}a(say))")
+
+/* Pack index
+ * s - OSTPACKINDEX
+ * u - Version
+ * a{sv} - Metadata
+ * a(st) - (checksum, offset into packfile)
+ */
+#define OSTREE_PACK_INDEX_VARIANT_FORMAT G_VARIANT_TYPE ("(sua{sv}a(st))")
+
 /* Pack files
+ * s - OSTPACKFILE
  * u - Version
  * a{sv} - Metadata
- * a(ysay) - Contents (flags, checksum, content)
+ * u - number of entries
+ *
+ * Repeating tuple of:
+ * <padding to alignment of 8>
+ * (yst) - Contents (flags, checksum, objtype, length)
+ * <raw data>
  */
-#define OSTREE_PACK_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(ua{sv}a(ysay))")
+#define OSTREE_PACK_FILE_VARIANT_FORMAT G_VARIANT_TYPE ("(sua{sv}t)")
+
+#define OSTREE_PACK_FILE_CONTENT_VARIANT_FORMAT G_VARIANT_TYPE ("(yst)")
 
 gboolean ostree_validate_checksum_string (const char *sha256,
                                           GError    **error);
diff --git a/src/ostree/ot-builtin-repack.c b/src/ostree/ot-builtin-repack.c
index dad2c7d..09e935c 100644
--- a/src/ostree/ot-builtin-repack.c
+++ b/src/ostree/ot-builtin-repack.c
@@ -28,27 +28,38 @@
 #include <glib/gi18n.h>
 #include <glib/gprintf.h>
 
+#include <gio/gunixinputstream.h>
 #include <gio/gunixoutputstream.h>
 
 #define DEFAULT_PACK_SIZE_BYTES (50*1024*1024)
 
+static gboolean opt_analyze_only;
 static char* opt_pack_size;
-static char* opt_compression;
+static char* opt_int_compression;
+
+typedef enum {
+  OT_INTERNAL_COMPRESSION_NONE,
+  OT_INTERNAL_COMPRESSION_GZIP
+} OtInternalCompressionType;
 
 static GOptionEntry options[] = {
   { "pack-size", 0, 0, G_OPTION_ARG_STRING, &opt_pack_size, "Maximum uncompressed size of packfiles in bytes; may be suffixed with k, m, or g", "BYTES" },
-  { "compression", 0, 0, G_OPTION_ARG_STRING, &opt_compression, "Compress generated packfiles using COMPRESSION; may be one of 'gzip', 'xz'", "COMPRESSION" },
+  { "internal-compression", 0, 0, G_OPTION_ARG_STRING, &opt_int_compression, "Compress objects using COMPRESSION; may be one of 'gzip', 'xz'", "COMPRESSION" },
+  { "analyze-only", 0, 0, G_OPTION_ARG_NONE, &opt_analyze_only, "Just analyze current state", NULL },
   { NULL }
 };
 
 typedef struct {
   OstreeRepo *repo;
+
+  guint64 pack_size;
+  OtInternalCompressionType int_compression;
+
   guint n_commits;
   guint n_dirmeta;
   guint n_dirtree;
   guint n_files;
   GPtrArray *objects;
-  GArray *object_sizes;
   gboolean had_error;
   GError **error;
 } OtRepackData;
@@ -59,26 +70,41 @@ typedef struct {
   GPid compress_child_pid;
 } OtBuildRepackFile;
 
+static GPtrArray *
+get_xz_args (void)
+{
+  GPtrArray *ret = g_ptr_array_new ();
+  
+  g_ptr_array_add (ret, "xz");
+  g_ptr_array_add (ret, "--memlimit-compress=512M");
+
+  return ret;
+}
+
 static void
 compressor_child_setup (gpointer user_data)
 {
   int stdout_fd = GPOINTER_TO_INT (user_data);
 
-  dup2 (stdout_fd, 1);
+  if (dup2 (stdout_fd, 1) < 0)
+    g_assert_not_reached ();
+  (void) close (stdout_fd);
 }
 
 static gboolean
-build_repack_file_init (OtBuildRepackFile *self,
-                        GFile             *destfile,
-                        GPtrArray         *compressor_argv,
-                        GCancellable      *cancellable,
-                        GError           **error)
+create_compressor_subprocess (OtBuildRepackFile *self,
+                              GPtrArray         *compressor_argv,
+                              GFile             *destfile,
+                              GOutputStream    **out_output,
+                              GCancellable      *cancellable,
+                              GError           **error)
 {
   gboolean ret = FALSE;
+  GOutputStream *ret_output = NULL;
   int stdin_pipe_fd;
   int target_stdout_fd;
 
-  target_stdout_fd = open (ot_gfile_get_path_cached (destfile), O_WRONLY | O_CREAT);
+  target_stdout_fd = open (ot_gfile_get_path_cached (destfile), O_WRONLY);
   if (target_stdout_fd < 0)
     {
       ot_util_set_error_from_errno (error, errno);
@@ -93,32 +119,15 @@ build_repack_file_init (OtBuildRepackFile *self,
   
   (void) close (target_stdout_fd);
 
-  self->out = g_unix_output_stream_new (stdin_pipe_fd, TRUE);
+  ret_output = g_unix_input_stream_new (stdin_pipe_fd, TRUE);
 
   ret = TRUE;
+  ot_transfer_out_value (out_output, &ret_output);
  out:
+  g_clear_object (&ret_output);
   return ret;
 }
 
-static inline int
-size_to_bucket (guint64 objsize)
-{
-  int off;
-  if (objsize < 128)
-    return 0;
-
-  objsize >>= 7;
-  off = 0;
-  while (objsize > 0)
-    {
-      off++;
-      objsize >>= 1;
-    }
-  if (off > 23)
-    return 23;
-  return off;
-}
-
 static void
 object_iter_callback (OstreeRepo    *repo,
                       const char    *checksum,
@@ -129,12 +138,8 @@ object_iter_callback (OstreeRepo    *repo,
 {
   gboolean ret = FALSE;
   OtRepackData *data = user_data;
-  char *key;
-  GError **error = data->error;
   guint64 objsize;
-  int bucket;
-  GVariant *archive_meta = NULL;
-  GFileInfo *archive_info = NULL;
+  GVariant *objdata = NULL;
 
   switch (objtype)
     {
@@ -156,21 +161,404 @@ object_iter_callback (OstreeRepo    *repo,
       break;
     }
 
-  objsize = g_file_info_get_size (file_info);
+  /* For archived content, only count regular files */
+  if (!(objtype == OSTREE_OBJECT_TYPE_ARCHIVED_FILE_CONTENT
+        && g_file_info_get_file_type (file_info) != G_FILE_TYPE_REGULAR))
+    {
+      objsize = g_file_info_get_size (file_info);
 
-  g_ptr_array_add (data->objects, 
+      objdata = g_variant_new ("(tsu)", objsize, checksum, (guint32)objtype);
+      g_ptr_array_add (data->objects, g_variant_ref_sink (objdata));
+      objdata = NULL; /* Transfer ownership */
+    }
 
-  bucket = size_to_bucket (objsize);
-  data->object_bucket_count[bucket]++;
-  data->object_bucket_size[bucket] += objsize;
-  
   ret = TRUE;
   /* out: */
-  ot_clear_gvariant (&archive_meta);
-  g_clear_object (&archive_info);
+  ot_clear_gvariant (&objdata);
   data->had_error = !ret;
 }
 
+static gint
+compare_object_data_by_size (gconstpointer    ap,
+                             gconstpointer    bp)
+{
+  GVariant *a = *(void **)ap;
+  GVariant *b = *(void **)bp;
+  guint64 a_size;
+  guint64 b_size;
+
+  g_variant_get_child (a, 0, "t", &a_size);
+  g_variant_get_child (b, 0, "t", &b_size);
+  if (a == b)
+    return 0;
+  else if (a > b)
+    return 1;
+  else
+    return -1;
+}
+
+static gboolean
+write_aligned_variant (GOutputStream      *output,
+                       GVariant           *variant,
+                       GChecksum          *checksum,
+                       guint64            *inout_offset,
+                       GCancellable       *cancellable,
+                       GError            **error)
+{
+  gboolean ret = FALSE;
+  guint padding;
+  gsize bytes_written;
+  char padding_nuls[7] = {0, 0, 0, 0, 0, 0, 0};
+
+  padding = 8 - ((*inout_offset) & 7);
+  
+  if (padding > 0)
+    {
+      g_checksum_update (checksum, (guchar*) padding_nuls, padding);
+      if (!g_output_stream_write_all (output, padding_nuls, padding, &bytes_written,
+                                      cancellable, error))
+        goto out;
+      g_assert (bytes_written == padding);
+      *inout_offset += padding;
+    }
+
+  g_checksum_update (checksum, (guchar*) g_variant_get_data (variant),
+                     g_variant_get_size (variant));
+  if (!g_output_stream_write_all (output, g_variant_get_data (variant),
+                                  g_variant_get_size (variant), &bytes_written,
+                                  cancellable, error))
+    goto out;
+  *inout_offset += bytes_written;
+
+  ret = TRUE;
+ out:
+  return ret;
+}
+
+static gboolean
+create_pack_file (OtRepackData        *data,
+                  GPtrArray           *objects,
+                  GCancellable        *cancellable,
+                  GError             **error)
+{
+  gboolean ret = FALSE;
+  GFile *pack_dir = NULL;
+  GFile *index_temppath = NULL;
+  GOutputStream *index_out = NULL;
+  GFile *pack_temppath = NULL;
+  GOutputStream *pack_out = NULL;
+  GFile *object_path = NULL;
+  GFileInfo *object_file_info = NULL;
+  GFileInputStream *object_input = NULL;
+  GConverter *compressor = NULL;
+  GConverterOutputStream *compressed_object_output = NULL;
+  guint i;
+  guint64 offset;
+  gsize bytes_read;
+  gsize bytes_written;
+  GVariantBuilder index_content_builder;
+  gboolean index_content_builder_initialized = FALSE;
+  GVariant *pack_header = NULL;
+  GVariant *object_header = NULL;
+  GVariant *index_content = NULL;
+  GChecksum *pack_checksum = NULL;
+  char *pack_name = NULL;
+  GFile *pack_file_path = NULL;
+  GFile *pack_index_path = NULL;
+
+  if (g_cancellable_set_error_if_cancelled (cancellable, error))
+    return FALSE;
+
+  if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo),
+                                        "pack-index", NULL,
+                                        &index_temppath,
+                                        &index_out,
+                                        cancellable, error))
+    goto out;
+  
+  if (!ostree_create_temp_regular_file (ostree_repo_get_tmpdir (data->repo),
+                                        "pack-content", NULL,
+                                        &pack_temppath,
+                                        &pack_out,
+                                        cancellable, error))
+    goto out;
+
+  offset = 0;
+  pack_checksum = g_checksum_new (G_CHECKSUM_SHA256);
+
+  g_variant_builder_init (&index_content_builder, G_VARIANT_TYPE ("a(st)"));
+  index_content_builder_initialized = TRUE;
+
+  pack_header = g_variant_new ("(su a{sv}u)",
+                               "OSTPACK", GUINT32_TO_BE (0),
+                               g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
+                               objects->len);
+
+  if (!write_aligned_variant (pack_out, pack_header, pack_checksum, &offset,
+                              cancellable, error))
+    goto out;
+  
+  for (i = 0; i < objects->len; i++)
+    {
+      GVariant *object_data = objects->pdata[i];
+      guint64 objsize;
+      const char *checksum;
+      guint32 objtype_u32;
+      OstreeObjectType objtype;
+      char buf[4096];
+      guint64 obj_bytes_written;
+      GOutputStream *write_pack_out;
+
+      g_variant_get (object_data, "(t&su)", &objsize, &checksum, &objtype_u32);
+                     
+      objtype = (OstreeObjectType) objtype_u32;
+
+      ot_clear_gvariant (&object_header);
+      object_header = g_variant_new ("(yst)", GUINT32_TO_BE (0), checksum, (guint32)objtype, objsize);
+
+      if (!write_aligned_variant (pack_out, object_header, pack_checksum,
+                                  &offset, cancellable, error))
+        goto out;
+      
+      g_clear_object (&object_path);
+      object_path = ostree_repo_get_object_path (data->repo, checksum, objtype);
+      
+      g_clear_object (&object_input);
+      object_input = g_file_read (object_path, cancellable, error);
+      if (!object_input)
+        goto out;
+
+      if (data->int_compression != OT_INTERNAL_COMPRESSION_NONE)
+        {
+          g_clear_object (&compressor);
+          switch (data->int_compression)
+            {
+            case OT_INTERNAL_COMPRESSION_GZIP:
+              compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_GZIP, 8);
+              break;
+            default:
+              g_assert_not_reached ();
+            }
+          
+          g_clear_object (&compressed_object_output);
+          compressed_object_output = (GConverterOutputStream*)g_object_new (G_TYPE_CONVERTER_OUTPUT_STREAM,
+                                                                            "converter", compressor,
+                                                                            "base-stream", pack_out,
+                                                                            "close-base-stream", FALSE,
+                                                                            NULL);
+          write_pack_out = (GOutputStream*)compressed_object_output;
+        }
+      else
+         write_pack_out = (GOutputStream*)pack_out;
+
+      obj_bytes_written = 0;
+      do
+        {
+          if (!g_input_stream_read_all ((GInputStream*)object_input, buf, sizeof(buf), &bytes_read, cancellable, error))
+            goto out;
+          g_checksum_update (pack_checksum, (guint8*)buf, bytes_read);
+          if (bytes_read > 0)
+            {
+              if (!g_output_stream_write_all (write_pack_out, buf, bytes_read, &bytes_written, cancellable, error))
+                goto out;
+              offset += bytes_written;
+              obj_bytes_written += bytes_written;
+            }
+        }
+      while (bytes_read > 0);
+
+      if (compressed_object_output)
+        {
+          if (!g_output_stream_flush ((GOutputStream*)compressed_object_output, cancellable, error))
+            goto out;
+        }
+
+      g_assert_cmpint (obj_bytes_written, ==, objsize);
+
+      g_variant_builder_add (&index_content_builder, "(st)", checksum, offset);
+    }
+  
+  if (!g_output_stream_close (pack_out, cancellable, error))
+    goto out;
+
+  pack_dir = g_file_resolve_relative_path (ostree_repo_get_path (data->repo),
+                                           "objects/pack");
+
+  pack_name = g_strconcat ("ostpack-", g_checksum_get_string (pack_checksum), ".data", NULL);
+  pack_file_path = g_file_get_child (pack_dir, pack_name);
+
+  if (rename (ot_gfile_get_path_cached (pack_temppath),
+              ot_gfile_get_path_cached (pack_file_path)) < 0)
+    {
+      ot_util_set_error_from_errno (error, errno);
+      g_prefix_error (error, "Failed to rename pack file '%s' to '%s': ",
+                      ot_gfile_get_path_cached (pack_temppath),
+                      ot_gfile_get_path_cached (pack_file_path));
+      goto out;
+    }
+  g_clear_object (&pack_temppath);
+
+  index_content = g_variant_new ("(su a{sv}@a(st))",
+                                 "OSTPACKINDEX", GUINT32_TO_BE(0),
+                                 g_variant_new_array (G_VARIANT_TYPE ("{sv}"), NULL, 0),
+                                 g_variant_builder_end (&index_content_builder));
+  index_content_builder_initialized = FALSE;
+
+  if (!g_output_stream_write_all (index_out,
+                                  g_variant_get_data (index_content),
+                                  g_variant_get_size (index_content),
+                                  &bytes_written,
+                                  cancellable,
+                                  error))
+    goto out;
+
+  if (!g_output_stream_close (index_out, cancellable, error))
+    goto out;
+
+  g_free (pack_name);
+  pack_name = g_strconcat ("ostpack-", g_checksum_get_string (pack_checksum), ".index", NULL);
+  pack_index_path = g_file_get_child (pack_dir, pack_name);
+
+  if (rename (ot_gfile_get_path_cached (index_temppath),
+              ot_gfile_get_path_cached (pack_index_path)) < 0)
+    {
+      ot_util_set_error_from_errno (error, errno);
+      g_prefix_error (error, "Failed to rename pack file '%s' to '%s': ",
+                      ot_gfile_get_path_cached (index_temppath),
+                      ot_gfile_get_path_cached (pack_index_path));
+      goto out;
+    }
+  g_clear_object (&index_temppath);
+
+  ret = TRUE;
+ out:
+  if (index_temppath)
+    (void) unlink (ot_gfile_get_path_cached (index_temppath));
+  g_clear_object (&index_temppath);
+  g_clear_object (&index_out);
+  if (pack_temppath)
+    (void) unlink (ot_gfile_get_path_cached (pack_temppath));
+  g_clear_object (&pack_temppath);
+  g_clear_object (&pack_out);
+  g_clear_object (&object_path);
+  g_clear_object (&object_input);
+  g_clear_object (&compressor);
+  g_clear_object (&compressed_object_output);
+  g_clear_object (&object_file_info);
+  ot_clear_gvariant (&object_header);
+  if (pack_checksum)
+    g_checksum_free (pack_checksum);
+  g_clear_object (&pack_dir);
+  ot_clear_gvariant (&index_content);
+  g_free (pack_name);
+  g_clear_object (&pack_file_path);
+  g_clear_object (&pack_index_path);
+  if (index_content_builder_initialized)
+    g_variant_builder_clear (&index_content_builder);
+  return ret;
+}
+
+/**
+ * cluster_objects_stupidly:
+ *
+ * Just sorts by size currently.
+ *
+ * Returns: [Array of [Array of object data]].  Free with g_ptr_array_unref().
+ */
+static GPtrArray *
+cluster_objects_stupidly (OtRepackData      *data)
+{
+  GPtrArray *ret = NULL;
+  GPtrArray *objects = data->objects;
+  guint i;
+  guint64 current_size;
+  guint current_offset;
+
+  g_ptr_array_sort (data->objects, compare_object_data_by_size);
+
+  ret = g_ptr_array_new ();
+
+  current_size = 0;
+  current_offset = 0;
+  for (i = 0; i < objects->len; i++)
+    { 
+      GVariant *objdata = objects->pdata[i];
+      guint64 objsize;
+
+      g_variant_get_child (objdata, 0, "t", &objsize);
+
+      if (current_size + objsize > data->pack_size)
+        {
+          guint j;
+          GPtrArray *current = g_ptr_array_new ();
+          for (j = current_offset; j < i; j++)
+            {
+              g_ptr_array_add (current, objects->pdata[j]);
+            }
+          g_ptr_array_add (ret, current);
+          current_size = objsize;
+          current_offset = i;
+        }
+      else if (objsize > data->pack_size)
+        {
+          break;
+        }
+      else
+        {
+          current_size += objsize;
+        }
+    }
+
+  return ret;
+}
+
+static gboolean
+parse_size_spec_with_suffix (const char *spec,
+                             guint64    *out_size,
+                             GError    **error)
+{
+  gboolean ret = FALSE;
+  char *endptr = NULL;
+  guint64 ret_size;
+
+  ret_size = g_ascii_strtoull (spec, &endptr, 10);
+  
+  if (endptr && *endptr)
+    {
+      char suffix = *endptr;
+      
+      switch (suffix)
+        {
+        case 'k':
+        case 'K':
+          {
+            ret_size *= 1024;
+            break;
+          }
+        case 'm':
+        case 'M':
+          {
+            ret_size *= (1024 * 1024);
+            break;
+          }
+        case 'g':
+        case 'G':
+          {
+            ret_size *= (1024 * 1024 * 1024);
+            break;
+          }
+        default:
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Invalid size suffix '%c'", suffix);
+          goto out;
+        }
+    }
+
+  ret = TRUE;
+  *out_size = ret_size;
+ out:
+  return ret;
+}
 
 gboolean
 ostree_builtin_repack (int argc, char **argv, GFile *repo_path, GError **error)
@@ -180,8 +568,9 @@ ostree_builtin_repack (int argc, char **argv, GFile *repo_path, GError **error)
   gboolean ret = FALSE;
   OstreeRepo *repo = NULL;
   GCancellable *cancellable = NULL;
-  int i;
+  guint i;
   guint64 total_size;
+  GPtrArray *clusters = NULL;
 
   memset (&data, 0, sizeof (data));
 
@@ -197,8 +586,29 @@ ostree_builtin_repack (int argc, char **argv, GFile *repo_path, GError **error)
 
   data.repo = repo;
   data.error = error;
-  data.objects = g_ptr_array_new_with_free_func (g_free);
-  data.object_sizes = g_array_new (FALSE, FALSE, sizeof (guint64));
+  data.objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
+  if (opt_pack_size)
+    {
+      if (!parse_size_spec_with_suffix (opt_pack_size, &data.pack_size, error))
+        goto out;
+    }
+  else
+    {
+      data.pack_size = DEFAULT_PACK_SIZE_BYTES;
+    }
+  if (opt_int_compression)
+    {
+      if (strcmp (opt_int_compression, "gzip") == 0)
+        data.int_compression = OT_INTERNAL_COMPRESSION_GZIP;
+      else
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Invalid internal compression '%s'", opt_int_compression);
+          goto out;
+        }
+    }
+  else
+    data.int_compression = OT_INTERNAL_COMPRESSION_NONE;
 
   if (!ostree_repo_iter_objects (repo, object_iter_callback, &data, error))
     goto out;
@@ -212,23 +622,43 @@ ostree_builtin_repack (int argc, char **argv, GFile *repo_path, GError **error)
   g_print ("Files: %u\n", data.n_files);
 
   total_size = 0;
-  for (i = 0; i < G_N_ELEMENTS(data.object_bucket_size); i++)
+  for (i = 0; i < data.objects->len; i++)
     {
-      int size;
-      if (i == 0)
-        size = 128;
-      else
-        size = 1 << (i + 7);
-      g_print ("%d: %" G_GUINT64_FORMAT " objects, %" G_GUINT64_FORMAT " bytes\n",
-               size, data.object_bucket_count[i], data.object_bucket_size[i]);
-      total_size += data.object_bucket_size[i];
+      GVariant *objdata = data.objects->pdata[i];
+      guint64 size;
+      
+      g_variant_get_child (objdata, 0, "t", &size);
+      
+      total_size += size;
     }
   g_print ("Total size: %" G_GUINT64_FORMAT "\n", total_size);
 
+  g_print ("\n");
+  g_print ("Using pack size: %" G_GUINT64_FORMAT "\n", data.pack_size);
+
+  clusters = cluster_objects_stupidly (&data);
+
+  g_print ("Going to create %u packfiles\n", clusters->len);
+
+  for (i = 0; i < clusters->len; i++)
+    {
+      GPtrArray *cluster = clusters->pdata[i];
+      
+      g_print ("%u: %u objects\n", i, cluster->len);
+
+      if (!opt_analyze_only)
+        {
+          if (!create_pack_file (&data, cluster, cancellable, error))
+            goto out;
+        }
+    }
+
   ret = TRUE;
  out:
   if (context)
     g_option_context_free (context);
   g_clear_object (&repo);
+  if (clusters)
+    g_ptr_array_unref (clusters);
   return ret;
 }