[ostree] Initial basic static delta code drop
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree] Initial basic static delta code drop
- Date: Tue, 4 Feb 2014 20:04:40 +0000 (UTC)
commit 2d6374822b481e885aa5f74ef9e6f993d5c792c5
Author: Colin Walters <walters verbum org>
Date: Thu Aug 15 09:17:37 2013 -0400
Initial basic static delta code drop
This has a very basic level of functionality (deltas can be generated,
and applied offline). There is only some stubbed out pull code to
fetch them via HTTP.
But, better to commit this now and improve it from a known starting
point, rather than have it languish in a branch.
Makefile-libostree.am | 4 +
Makefile-ostree.am | 1 +
Makefile-tests.am | 1 +
src/libostree/README-deltas.md | 159 ++++++++
src/libostree/ostree-core-private.h | 10 +
src/libostree/ostree-core.c | 59 +++-
src/libostree/ostree-core.h | 2 +
src/libostree/ostree-repo-private.h | 1 +
src/libostree/ostree-repo-pull.c | 172 ++++++++--
.../ostree-repo-static-delta-compilation.c | 380 +++++++++++++++++++
src/libostree/ostree-repo-static-delta-core.c | 333 +++++++++++++++++
src/libostree/ostree-repo-static-delta-private.h | 96 +++++
.../ostree-repo-static-delta-processing.c | 381 ++++++++++++++++++++
src/libostree/ostree-repo.c | 3 +
src/libostree/ostree-repo.h | 31 ++
src/libostree/ostree-varint.c | 9 +-
src/libostree/ostree-varint.h | 7 +-
src/libotutil/ot-variant-utils.c | 65 ++++
src/libotutil/ot-variant-utils.h | 12 +
src/ostree/main.c | 1 +
src/ostree/ot-builtin-static-delta.c | 129 +++++++
src/ostree/ot-builtins.h | 1 +
tests/test-delta.sh | 74 ++++
tests/test-varint.c | 2 +-
24 files changed, 1890 insertions(+), 43 deletions(-)
---
diff --git a/Makefile-libostree.am b/Makefile-libostree.am
index 060c058..8e9aaca 100644
--- a/Makefile-libostree.am
+++ b/Makefile-libostree.am
@@ -70,6 +70,10 @@ libostree_1_la_SOURCES = \
src/libostree/ostree-bootloader-uboot.c \
src/libostree/ostree-gpg-verifier.c \
src/libostree/ostree-gpg-verifier.h \
+ src/libostree/ostree-repo-static-delta-core.c \
+ src/libostree/ostree-repo-static-delta-processing.c \
+ src/libostree/ostree-repo-static-delta-compilation.c \
+ src/libostree/ostree-repo-static-delta-private.h \
$(NULL)
if USE_LIBARCHIVE
libostree_1_la_SOURCES += src/libostree/ostree-libarchive-input-stream.h \
diff --git a/Makefile-ostree.am b/Makefile-ostree.am
index fa12055..52f50e2 100644
--- a/Makefile-ostree.am
+++ b/Makefile-ostree.am
@@ -41,6 +41,7 @@ ostree_SOURCES = src/ostree/main.c \
src/ostree/ot-builtin-reset.c \
src/ostree/ot-builtin-rev-parse.c \
src/ostree/ot-builtin-show.c \
+ src/ostree/ot-builtin-static-delta.c \
src/ostree/ot-main.h \
src/ostree/ot-main.c \
src/ostree/ot-dump.h \
diff --git a/Makefile-tests.am b/Makefile-tests.am
index e9f45f3..e281883 100644
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -38,6 +38,7 @@ testfiles = test-basic \
test-admin-deploy-etcmerge-cornercases \
test-admin-deploy-uboot \
test-setuid \
+ test-delta \
test-xattrs \
$(NULL)
insttest_SCRIPTS = $(addprefix tests/,$(testfiles:=.sh))
diff --git a/src/libostree/README-deltas.md b/src/libostree/README-deltas.md
new file mode 100644
index 0000000..28ebcc1
--- /dev/null
+++ b/src/libostree/README-deltas.md
@@ -0,0 +1,159 @@
+OSTree Static Object Deltas
+===========================
+
+Currently, OSTree's "archive-z2" mode stores both metadata and content
+objects as individual files in the filesystem. Content objects are
+zlib-compressed.
+
+The advantage of this is model are:
+
+0) It's easy to understand and implement
+1) Can be served directly over plain HTTP by a static webserver
+2) Space efficient on the server
+
+However, it can be inefficient both for large updates and small ones:
+
+0) For large tree changes (such as going from -runtime to
+ -devel-debug, or major version upgrades), this can mean thousands
+ and thousands of HTTP requests. The overhead for that is very
+ large (until SPDY/HTTP2.0), and will be catastrophically bad if the
+ webserver is not configured with KeepAlive.
+1) Small changes (typo in gnome-shell .js file) still require around
+ 5 metadata HTTP requests, plus a redownload of the whole file.
+
+Why not smart servers?
+======================
+
+Smart servers (custom daemons, or just CGI scripts) as git has are not
+under consideration for this proposal. OSTree is designed for the
+same use case as GNU/Linux distribution package systems are, where
+content is served by a network of volunteer mirrors that will
+generally not run custom code.
+
+In particular, Amazon S3 style dumb content servers is a very
+important use case, as is being able to apply updates from static
+media like DVD-ROM.
+
+Finding Static Deltas
+=====================
+
+Since static deltas may not exist, the client first needs to attempt
+to locate one. Suppose a client wants to retrieve commit ${new} while
+currently running ${current}. The first thing to fetch is the delta
+metadata, called "meta". It can be found at
+${repo}/deltas/${current}-${new}/meta.
+
+FIXME: GPG signatures (.metameta?) Or include commit object in meta?
+But we would then be forced to verify the commit only after processing
+the entirety of the delta, which is dangerous. I think we need to
+require signing deltas.
+
+Delta Bytecode Format
+=====================
+
+A delta-part has the following form:
+
+byte compression-type (0 = none, 'g' = gzip')
+REPEAT[(varint size, delta-part-content)]
+
+delta-part-content:
+ byte[] payload
+ ARRAY[operation]
+
+The rationale for having delta-part is that it allows easy incremental
+resumption of downloads. The client can look at the delta descriptor
+and skip downloading delta-parts for which it already has the
+contained objects. This is better than simply resuming a gigantic
+file because if the client decides to fetch a slightly newer version,
+it's very probable that some of the downloading we've already done is
+still useful.
+
+For the actual delta payload, it comes as a stream of pair of
+(payload, operation) so that it can be processed while being
+decompressed.
+
+Finally, the delta-part-content is effectively a high level bytecode
+for a stack-oriented machine. It iterates on the array of objects in
+order. The following operations are available:
+
+FETCH
+ Fall back to fetching the current object individually. Move
+ to the next object.
+
+WRITE(array[(varint offset, varint length)])
+ Write from current input target (default payload) to output.
+
+GUNZIP(array[(varint offset, varint length)])
+ gunzip from current input target (default payload) to output.
+
+CLOSE
+ Close the current output target, and proceed to the next; if the
+ output object was a temporary, the output resets to the current
+ object.
+
+# Change the input source to an object
+READOBJECT(csum object)
+ Set object as current input target
+
+# Change the input source to payload
+READPAYLOAD
+ Set payload as current input target
+
+Compiling Deltas
+================
+
+After reading the above, you may be wondering how we actually *make*
+these deltas. I envison a strategy similar to that employed by
+Chromium autoupdate:
+http://www.chromium.org/chromium-os/chromiumos-design-docs/autoupdate-details
+
+Something like this would be a useful initial algorithm:
+1) Compute the set of added objects NEW
+2) For each object in NEW:
+ - Look for a the set of "superficially similar" objects in the
+ previous tree, using heuristics based first on filename (including
+ prefix), then on size. Call this set CANDIDATES.
+ For each entry in CANDIDATES:
+ - Try doing a bup/librsync style rolling checksum, and compute the
+ list of changed blocks.
+ - Try gzip-compressing it
+3) Choose the lowest cost method for each NEW object, and partition
+ the program for each method into deltapart-sized chunks.
+
+However, there are many other possibilities, that could be used in a
+hybrid mode with the above. For example, we could try to find similar
+objects, and gzip them together. This would be a *very* useful
+strategy for things like the 9000 Boost headers which have massive
+amounts of redundant data.
+
+Notice too that the delta format supports falling back to retrieving
+individual objects. For cases like the initramfs which is compressed
+inside the tree with gzip, we're not going to find an efficient way to
+sync it, so the delta compiler should just fall back to fetching it
+individually.
+
+Which Deltas To Create?
+=======================
+
+Going back to the start, there are two cases to optimize for:
+
+1) Incremental upgrades between builds
+2) Major version upgrades
+
+A command line operation would look something like this:
+
+$ ostree --repo=/path/to/repo gendelta --ref-prefix=gnome-ostree/buildmaster/ --strategy=latest --depth=5
+
+This would tell ostree to generate deltas from each of the last 4
+commits to each ref (e.g. gnome-ostree/buildmaster/x86_64-runtime) to
+the latest commit. It might also be possible of course to have
+--strategy=incremental where we generate a delta between each commit.
+I suspect that'd be something to do if one has a *lot* of disk space
+to spend, and there's a reason for clients to be fetching individual
+refs.
+
+$ ostree --repo=/path/to/repo gendelta --from=gnome-ostree/3.10/x86_64-runtime
--to=gnome-ostree/buildmaster/x86_64-runtime
+
+This is an obvious one - generate a delta from the last stable release
+to the current development head.
+
diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h
index 84356e7..851e74b 100644
--- a/src/libostree/ostree-core-private.h
+++ b/src/libostree/ostree-core-private.h
@@ -91,6 +91,16 @@ _ostree_get_relative_object_path (const char *checksum,
OstreeObjectType type,
gboolean compressed);
+
+char *
+_ostree_get_relative_static_delta_path (const char *from,
+ const char *to);
+
+char *
+_ostree_get_relative_static_delta_part_path (const char *from,
+ const char *to,
+ guint i);
+
void
_ostree_loose_path (char *buf,
const char *checksum,
diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c
index fc3d274..9b7a237 100644
--- a/src/libostree/ostree-core.c
+++ b/src/libostree/ostree-core.c
@@ -1208,13 +1208,41 @@ ostree_checksum_from_bytes_v (GVariant *csum_v)
* ostree_checksum_bytes_peek:
* @bytes: #GVariant of type ay
*
- * Returns: (transfer none): Binary checksum data in @bytes; do not free
+ * Returns: (transfer none) (array fixed-size=32) (element-type guint8): Binary checksum data in @bytes; do
not free. If @bytes does not have the correct length, return %NULL.
*/
const guchar *
ostree_checksum_bytes_peek (GVariant *bytes)
{
gsize n_elts;
- return g_variant_get_fixed_array (bytes, &n_elts, 1);
+ const guchar *ret;
+ ret = g_variant_get_fixed_array (bytes, &n_elts, 1);
+ if (G_UNLIKELY (n_elts != 32))
+ return NULL;
+ return ret;
+}
+
+/**
+ * ostree_checksum_bytes_peek_validate:
+ * @bytes: #GVariant of type ay
+ * @error: Errror
+ *
+ * Like ostree_checksum_bytes_peek(), but also throws @error.
+ *
+ * Returns: (transfer none) (array fixed-size=32) (element-type guint8): Binary checksum data
+ */
+const guchar *
+ostree_checksum_bytes_peek_validate (GVariant *bytes,
+ GError **error)
+{
+ const guchar *ret = ostree_checksum_bytes_peek (bytes);
+ if (G_UNLIKELY (!ret))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid checksum of length %" G_GUINT64_FORMAT
+ " expected 32", (guint64) g_variant_n_children (bytes));
+ return NULL;
+ }
+ return ret;
}
/*
@@ -1293,8 +1321,23 @@ _ostree_get_relative_object_path (const char *checksum,
return g_string_free (path, FALSE);
}
+char *
+_ostree_get_relative_static_delta_path (const char *from,
+ const char *to)
+{
+ return g_strdup_printf ("deltas/%s-%s/meta", from, to);
+}
+
+char *
+_ostree_get_relative_static_delta_part_path (const char *from,
+ const char *to,
+ guint i)
+{
+ return g_strdup_printf ("deltas/%s-%s/%u", from, to, i);
+}
+
/*
- * ostree_file_header_parse:
+ * file_header_parse:
* @metadata: A metadata variant of type %OSTREE_FILE_HEADER_GVARIANT_FORMAT
* @out_file_info: (out): Parsed file information
* @out_xattrs: (out): Parsed extended attribute set
@@ -1449,15 +1492,7 @@ gboolean
ostree_validate_structureof_csum_v (GVariant *checksum,
GError **error)
{
- gsize n_children = g_variant_n_children (checksum);
- if (n_children != 32)
- {
- g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
- "Invalid checksum of length %" G_GUINT64_FORMAT
- " expected 32", (guint64) n_children);
- return FALSE;
- }
- return TRUE;
+ return ostree_checksum_bytes_peek_validate (checksum, error) != NULL;
}
/**
diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h
index 5c76a4a..bd8e68f 100644
--- a/src/libostree/ostree-core.h
+++ b/src/libostree/ostree-core.h
@@ -136,6 +136,8 @@ void ostree_checksum_inplace_to_bytes (const char *checksum,
const guchar *ostree_checksum_bytes_peek (GVariant *bytes);
+const guchar *ostree_checksum_bytes_peek_validate (GVariant *bytes, GError **error);
+
int ostree_cmp_checksum_bytes (const guchar *a, const guchar *b);
gboolean ostree_validate_rev (const char *rev, GError **error);
diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h
index 711bfea..b12e116 100644
--- a/src/libostree/ostree-repo-private.h
+++ b/src/libostree/ostree-repo-private.h
@@ -41,6 +41,7 @@ struct OstreeRepo {
GFile *remote_heads_dir;
GFile *objects_dir;
int objects_dir_fd;
+ GFile *deltas_dir;
GFile *uncompressed_objects_dir;
int uncompressed_objects_dir_fd;
GFile *remote_cache_dir;
diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c
index efdf174..b98d132 100644
--- a/src/libostree/ostree-repo-pull.c
+++ b/src/libostree/ostree-repo-pull.c
@@ -58,6 +58,7 @@
#include "ostree.h"
#include "ostree-core-private.h"
#include "ostree-repo-private.h"
+#include "ostree-repo-static-delta-private.h"
#include "ostree-fetcher.h"
#include "otutil.h"
@@ -97,6 +98,7 @@ typedef struct {
GThread *metadata_thread;
GMainContext *metadata_thread_context;
GMainLoop *metadata_thread_loop;
+ GPtrArray *static_delta_metas;
OtWaitableQueue *metadata_objects_to_scan;
OtWaitableQueue *metadata_objects_to_fetch;
GHashTable *scanned_metadata; /* Maps object name to itself */
@@ -348,11 +350,13 @@ fetch_uri_sync_on_complete (GObject *object,
}
static gboolean
-fetch_uri_contents_utf8_sync (OtPullData *pull_data,
- SoupURI *uri,
- char **out_contents,
- GCancellable *cancellable,
- GError **error)
+fetch_uri_contents_membuf_sync (OtPullData *pull_data,
+ SoupURI *uri,
+ gboolean add_nul,
+ gboolean allow_noent,
+ GBytes **out_contents,
+ GCancellable *cancellable,
+ GError **error)
{
gboolean ret = FALSE;
const guint8 nulchar = 0;
@@ -360,6 +364,8 @@ fetch_uri_contents_utf8_sync (OtPullData *pull_data,
gs_unref_object GMemoryOutputStream *buf = NULL;
OstreeFetchUriSyncData fetch_data = { 0, };
+ g_assert (error != NULL);
+
if (g_cancellable_set_error_if_cancelled (cancellable, error))
return FALSE;
@@ -371,7 +377,15 @@ fetch_uri_contents_utf8_sync (OtPullData *pull_data,
run_mainloop_monitor_fetcher (pull_data);
if (!fetch_data.result_stream)
- goto out;
+ {
+ if (g_error_matches (*error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND))
+ {
+ g_clear_error (error);
+ ret = TRUE;
+ *out_contents = NULL;
+ }
+ goto out;
+ }
buf = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
if (g_output_stream_splice ((GOutputStream*)buf, fetch_data.result_stream,
@@ -379,14 +393,40 @@ fetch_uri_contents_utf8_sync (OtPullData *pull_data,
cancellable, error) < 0)
goto out;
- /* Add trailing NUL */
- if (!g_output_stream_write ((GOutputStream*)buf, &nulchar, 1, cancellable, error))
- goto out;
+ if (add_nul)
+ {
+ if (!g_output_stream_write ((GOutputStream*)buf, &nulchar, 1, cancellable, error))
+ goto out;
+ }
if (!g_output_stream_close ((GOutputStream*)buf, cancellable, error))
goto out;
- ret_contents = g_memory_output_stream_steal_data (buf);
+ ret = TRUE;
+ *out_contents = g_memory_output_stream_steal_as_bytes (buf);
+ out:
+ g_clear_object (&(fetch_data.result_stream));
+ return ret;
+}
+
+static gboolean
+fetch_uri_contents_utf8_sync (OtPullData *pull_data,
+ SoupURI *uri,
+ char **out_contents,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_unref_bytes GBytes *bytes = NULL;
+ gs_free char *ret_contents = NULL;
+ gsize len;
+
+ if (!fetch_uri_contents_membuf_sync (pull_data, uri, TRUE, FALSE,
+ &bytes, cancellable, error))
+ goto out;
+
+ ret_contents = g_bytes_unref_to_data (bytes, &len);
+ bytes = NULL;
if (!g_utf8_validate (ret_contents, -1, NULL))
{
@@ -398,7 +438,6 @@ fetch_uri_contents_utf8_sync (OtPullData *pull_data,
ret = TRUE;
ot_transfer_out_value (out_contents, &ret_contents);
out:
- g_clear_object (&(fetch_data.result_stream));
return ret;
}
@@ -1193,6 +1232,70 @@ load_remote_repo_config (OtPullData *pull_data,
return ret;
}
+static void
+initiate_commit_scan (OtPullData *pull_data,
+ const char *checksum)
+{
+ ot_waitable_queue_push (pull_data->metadata_objects_to_scan,
+ pull_worker_message_new (PULL_MSG_SCAN,
+ ostree_object_name_serialize (checksum,
OSTREE_OBJECT_TYPE_COMMIT)));
+}
+
+#if 0
+static gboolean
+request_static_delta_meta_sync (OtPullData *pull_data,
+ const char *ref,
+ const char *checksum,
+ GVariant **out_delta_meta,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_free char *from_revision = NULL;
+ SoupURI *target_uri = NULL;
+ gs_unref_variant GVariant *ret_delta_meta = NULL;
+
+ if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE, &from_revision, error))
+ goto out;
+
+ if (from_revision == NULL)
+ {
+ initiate_commit_scan (pull_data, checksum);
+ }
+ else
+ {
+ gs_free char *delta_name = _ostree_get_relative_static_delta_path (from_revision, checksum);
+ gs_unref_bytes GBytes *delta_meta_data = NULL;
+ gs_unref_variant GVariant *delta_meta = NULL;
+
+ target_uri = suburi_new (pull_data->base_uri, delta_name, NULL);
+
+ if (!fetch_uri_contents_membuf_sync (pull_data, target_uri, FALSE, TRUE,
+ &delta_meta_data,
+ pull_data->cancellable, error))
+ goto out;
+
+ if (delta_meta_data)
+ {
+ g_print ("Using static delta\n");
+ ret_delta_meta = ot_variant_new_from_bytes ((GVariantType*)OSTREE_STATIC_DELTA_META_FORMAT,
+ delta_meta_data, FALSE);
+ }
+ }
+
+ ret = TRUE;
+ gs_transfer_out_value (out_delta_meta, &ret_delta_meta);
+ out:
+ return ret;
+}
+#endif
+
+static void
+process_one_static_delta_meta (OtPullData *pull_data,
+ GVariant *delta_meta)
+{
+}
+
gboolean
ostree_repo_pull (OstreeRepo *self,
const char *remote_name,
@@ -1207,6 +1310,7 @@ ostree_repo_pull (OstreeRepo *self,
gpointer key, value;
gboolean tls_permissive = FALSE;
OstreeFetcherConfigFlags fetcher_flags = 0;
+ guint i;
gs_free char *remote_key = NULL;
gs_free char *path = NULL;
gs_free char *baseurl = NULL;
@@ -1293,6 +1397,8 @@ ostree_repo_pull (OstreeRepo *self,
goto out;
}
+ pull_data->static_delta_metas = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
+
requested_refs_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
updated_refs = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free);
commits_to_fetch = g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
@@ -1364,12 +1470,29 @@ ostree_repo_pull (OstreeRepo *self,
{
const char *branch = *branches_iter;
char *contents;
+ GVariant *descriptor_data = NULL;
if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
goto out;
-
- /* Transfer ownership of contents */
- g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents);
+
+ initiate_commit_scan (pull_data, contents);
+#if 0
+ if (!request_static_delta_meta_sync (pull_data, branch, contents,
+ &descriptor_data, cancellable, error))
+ goto out;
+#endif
+
+ if (!descriptor_data)
+ {
+ /* Transfer ownership of contents */
+ g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents);
+ }
+ else
+ {
+ /* Transfer ownership of delta descriptor */
+ g_ptr_array_add (pull_data->static_delta_metas, descriptor_data);
+ g_free (contents);
+ }
}
}
}
@@ -1388,24 +1511,24 @@ ostree_repo_pull (OstreeRepo *self,
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *commit = value;
-
- ot_waitable_queue_push (pull_data->metadata_objects_to_scan,
- pull_worker_message_new (PULL_MSG_SCAN,
- ostree_object_name_serialize (commit,
OSTREE_OBJECT_TYPE_COMMIT)));
+ initiate_commit_scan (pull_data, commit);
}
g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *ref = key;
- const char *sha256 = value;
+ const char *checksum = value;
- ot_waitable_queue_push (pull_data->metadata_objects_to_scan,
- pull_worker_message_new (PULL_MSG_SCAN,
- ostree_object_name_serialize (sha256,
OSTREE_OBJECT_TYPE_COMMIT)));
- g_hash_table_insert (updated_refs, g_strdup (ref), g_strdup (sha256));
+ initiate_commit_scan (pull_data, checksum);
+ g_hash_table_insert (updated_refs, g_strdup (ref), g_strdup (checksum));
}
-
+
+ for (i = 0; i < pull_data->static_delta_metas->len; i++)
+ {
+ process_one_static_delta_meta (pull_data, pull_data->static_delta_metas->pdata[i]);
+ }
+
{
queue_src = ot_waitable_queue_create_source (pull_data->metadata_objects_to_fetch);
g_source_set_callback (queue_src, (GSourceFunc)on_metadata_objects_to_fetch_ready, pull_data, NULL);
@@ -1486,6 +1609,7 @@ ostree_repo_pull (OstreeRepo *self,
pull_worker_message_new (PULL_MSG_QUIT, NULL));
g_thread_join (pull_data->metadata_thread);
}
+ g_clear_pointer (&pull_data->static_delta_metas, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&pull_data->metadata_objects_to_scan, (GDestroyNotify) ot_waitable_queue_unref);
g_clear_pointer (&pull_data->metadata_objects_to_fetch, (GDestroyNotify) ot_waitable_queue_unref);
g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref);
diff --git a/src/libostree/ostree-repo-static-delta-compilation.c
b/src/libostree/ostree-repo-static-delta-compilation.c
new file mode 100644
index 0000000..cfcf1f4
--- /dev/null
+++ b/src/libostree/ostree-repo-static-delta-compilation.c
@@ -0,0 +1,380 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2013,2014 Colin Walters <walters verbum org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include <string.h>
+
+#include "ostree-core-private.h"
+#include "ostree-repo-private.h"
+#include "ostree-repo-static-delta-private.h"
+#include "ostree-diff.h"
+#include "otutil.h"
+#include "ostree-varint.h"
+
+typedef struct {
+ guint64 uncompressed_size;
+ GPtrArray *objects;
+ GString *payload;
+ GString *operations;
+} OstreeStaticDeltaPartBuilder;
+
+typedef struct {
+ GPtrArray *parts;
+} OstreeStaticDeltaBuilder;
+
+static void
+ostree_static_delta_part_builder_unref (OstreeStaticDeltaPartBuilder *part_builder)
+{
+ if (part_builder->objects)
+ g_ptr_array_unref (part_builder->objects);
+ if (part_builder->payload)
+ g_string_free (part_builder->payload, TRUE);
+ if (part_builder->operations)
+ g_string_free (part_builder->operations, TRUE);
+ g_free (part_builder);
+}
+
+static OstreeStaticDeltaPartBuilder *
+allocate_part (OstreeStaticDeltaBuilder *builder)
+{
+ OstreeStaticDeltaPartBuilder *part = g_new0 (OstreeStaticDeltaPartBuilder, 1);
+ part->objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
+ part->payload = g_string_new (NULL);
+ part->operations = g_string_new (NULL);
+ part->uncompressed_size = 0;
+ g_ptr_array_add (builder->parts, part);
+ return part;
+}
+
+static GBytes *
+objtype_checksum_array_new (GPtrArray *objects)
+{
+ guint i;
+ GByteArray *ret = g_byte_array_new ();
+
+ g_assert (objects->len > 0);
+ for (i = 0; i < objects->len; i++)
+ {
+ GVariant *serialized_key = objects->pdata[i];
+ OstreeObjectType objtype;
+ const char *checksum;
+ guint8 csum[32];
+ guint8 objtype_v;
+
+ ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+ objtype_v = (guint8) objtype;
+
+ ostree_checksum_inplace_to_bytes (checksum, csum);
+
+ g_byte_array_append (ret, &objtype_v, 1);
+ g_byte_array_append (ret, csum, sizeof (csum));
+ }
+ return g_byte_array_free_to_bytes (ret);
+}
+
+static gboolean
+generate_delta_lowlatency (OstreeRepo *repo,
+ const char *from,
+ const char *to,
+ OstreeStaticDeltaBuilder *builder,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ GHashTableIter hashiter;
+ gpointer key, value;
+ OstreeStaticDeltaPartBuilder *current_part = NULL;
+ gs_unref_object GFile *root_from = NULL;
+ gs_unref_object GFile *root_to = NULL;
+ gs_unref_ptrarray GPtrArray *modified = NULL;
+ gs_unref_ptrarray GPtrArray *removed = NULL;
+ gs_unref_ptrarray GPtrArray *added = NULL;
+ gs_unref_hashtable GHashTable *to_reachable_objects = NULL;
+ gs_unref_hashtable GHashTable *from_reachable_objects = NULL;
+ gs_unref_hashtable GHashTable *new_reachable_objects = NULL;
+
+ if (!ostree_repo_read_commit (repo, from, &root_from, NULL,
+ cancellable, error))
+ goto out;
+ if (!ostree_repo_read_commit (repo, to, &root_to, NULL,
+ cancellable, error))
+ goto out;
+
+ /* Gather a filesystem level diff; when we do heuristics to ship
+ * just parts of changed files, we can make use of this data.
+ */
+ modified = g_ptr_array_new_with_free_func ((GDestroyNotify) ostree_diff_item_unref);
+ removed = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref);
+ added = g_ptr_array_new_with_free_func ((GDestroyNotify) g_object_unref);
+ if (!ostree_diff_dirs (OSTREE_DIFF_FLAGS_NONE, root_from, root_to, modified, removed, added,
+ cancellable, error))
+ goto out;
+
+ if (!ostree_repo_traverse_commit (repo, from, -1, &from_reachable_objects,
+ cancellable, error))
+ goto out;
+
+ if (!ostree_repo_traverse_commit (repo, to, -1, &to_reachable_objects,
+ cancellable, error))
+ goto out;
+
+ new_reachable_objects = ostree_repo_traverse_new_reachable ();
+
+ g_hash_table_iter_init (&hashiter, to_reachable_objects);
+ while (g_hash_table_iter_next (&hashiter, &key, &value))
+ {
+ GVariant *serialized_key = key;
+
+ if (g_hash_table_contains (from_reachable_objects, serialized_key))
+ continue;
+
+ g_hash_table_insert (new_reachable_objects, g_variant_ref (serialized_key), serialized_key);
+ }
+
+ current_part = allocate_part (builder);
+
+ g_hash_table_iter_init (&hashiter, new_reachable_objects);
+ while (g_hash_table_iter_next (&hashiter, &key, &value))
+ {
+ GVariant *serialized_key = key;
+ const char *checksum;
+ OstreeObjectType objtype;
+ guint64 content_size;
+ gsize object_payload_start;
+ gs_unref_object GInputStream *content_stream = NULL;
+ gsize bytes_read;
+ const guint readlen = 4096;
+
+ ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
+
+ if (!ostree_repo_load_object_stream (repo, objtype, checksum,
+ &content_stream, &content_size,
+ cancellable, error))
+ goto out;
+
+ current_part->uncompressed_size += content_size;
+
+ /* Ensure we have at least one object per delta, even if a given
+ * object is larger.
+ */
+ if (current_part->objects->len > 0 &&
+ current_part->payload->len + content_size > OSTREE_STATIC_DELTA_PART_MAX_SIZE_BYTES)
+ {
+ current_part = allocate_part (builder);
+ }
+
+ g_ptr_array_add (current_part->objects, g_variant_ref (serialized_key));
+
+ object_payload_start = current_part->payload->len;
+
+ while (TRUE)
+ {
+ gsize empty_space;
+
+ empty_space = current_part->payload->allocated_len - current_part->payload->len;
+ if (empty_space < readlen)
+ {
+ gsize origlen;
+ origlen = current_part->payload->len;
+ g_string_set_size (current_part->payload, current_part->payload->allocated_len + (readlen -
empty_space));
+ current_part->payload->len = origlen;
+ }
+
+ if (!g_input_stream_read_all (content_stream,
+ current_part->payload->str + current_part->payload->len,
+ readlen,
+ &bytes_read,
+ cancellable, error))
+ goto out;
+ if (bytes_read == 0)
+ break;
+
+ current_part->payload->len += bytes_read;
+ }
+
+ g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_WRITE);
+ _ostree_write_varuint64 (current_part->operations, object_payload_start);
+ _ostree_write_varuint64 (current_part->operations, content_size);
+ g_printerr ("write %" G_GUINT64_FORMAT " %" G_GUINT64_FORMAT "\n", (guint64) object_payload_start,
(guint64)(content_size));
+ g_string_append_c (current_part->operations, (gchar)OSTREE_STATIC_DELTA_OP_CLOSE);
+ }
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+/**
+ * ostree_repo_static_delta_generate:
+ * @self: Repo
+ * @opt: High level optimization choice
+ * @from: ASCII SHA256 checksum of origin
+ * @to: ASCII SHA256 checksum of target
+ * @metadata: (allow-none): Optional metadata
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Generate a lookaside "static delta" from @from which can generate
+ * the objects in @to. This delta is an optimization over fetching
+ * individual objects, and can be conveniently stored and applied
+ * offline.
+ */
+gboolean
+ostree_repo_static_delta_generate (OstreeRepo *self,
+ OstreeStaticDeltaGenerateOpt opt,
+ const char *from,
+ const char *to,
+ GVariant *metadata,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ OstreeStaticDeltaBuilder builder = { 0, };
+ guint i;
+ GVariant *metadata_source;
+ gs_unref_variant_builder GVariantBuilder *part_headers = NULL;
+ gs_unref_ptrarray GPtrArray *part_tempfiles = NULL;
+ gs_unref_variant GVariant *delta_descriptor = NULL;
+ gs_free char *descriptor_relpath = NULL;
+ gs_unref_object GFile *descriptor_path = NULL;
+ gs_unref_object GFile *descriptor_dir = NULL;
+ gs_unref_variant GVariant *tmp_metadata = NULL;
+
+ builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref);
+
+ /* Ignore optimization flags */
+ if (!generate_delta_lowlatency (self, from, to, &builder,
+ cancellable, error))
+ goto out;
+
+ part_headers = g_variant_builder_new (G_VARIANT_TYPE ("a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT));
+ part_tempfiles = g_ptr_array_new_with_free_func (g_object_unref);
+ for (i = 0; i < builder.parts->len; i++)
+ {
+ OstreeStaticDeltaPartBuilder *part_builder = builder.parts->pdata[i];
+ GBytes *payload_b;
+ GBytes *operations_b;
+ gs_free guchar *part_checksum = NULL;
+ gs_free_checksum GChecksum *checksum = NULL;
+ gs_unref_bytes GBytes *objtype_checksum_array = NULL;
+ gs_unref_bytes GBytes *checksum_bytes = NULL;
+ gs_unref_object GFile *part_tempfile = NULL;
+ gs_unref_object GOutputStream *part_temp_outstream = NULL;
+ gs_unref_object GInputStream *part_in = NULL;
+ gs_unref_object GInputStream *part_payload_in = NULL;
+ gs_unref_object GMemoryOutputStream *part_payload_out = NULL;
+ gs_unref_object GConverterOutputStream *part_payload_compressor = NULL;
+ gs_unref_object GConverter *zlib_compressor = NULL;
+ gs_unref_variant GVariant *delta_part_content = NULL;
+ gs_unref_variant GVariant *delta_part = NULL;
+ gs_unref_variant GVariant *delta_part_header = NULL;
+
+ payload_b = g_string_free_to_bytes (part_builder->payload);
+ part_builder->payload = NULL;
+
+ operations_b = g_string_free_to_bytes (part_builder->operations);
+ part_builder->operations = NULL;
+ /* FIXME - avoid duplicating memory here */
+ delta_part_content = g_variant_new ("(@ay ay)",
+ ot_gvariant_new_ay_bytes (payload_b),
+ ot_gvariant_new_ay_bytes (operations_b));
+ g_variant_ref_sink (delta_part_content);
+
+ /* Hardcode gzip for now */
+ zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9);
+ part_payload_in = ot_variant_read (delta_part_content);
+ part_payload_out = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
+ part_payload_compressor = (GConverterOutputStream*)g_converter_output_stream_new
((GOutputStream*)part_payload_out, zlib_compressor);
+
+ if (0 > g_output_stream_splice ((GOutputStream*)part_payload_compressor, part_payload_in,
+ G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET |
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
+ cancellable, error))
+ goto out;
+
+ /* FIXME - avoid duplicating memory here */
+ delta_part = g_variant_new ("(y ay)",
+ (guint8)'g',
+ ot_gvariant_new_ay_bytes (g_memory_output_stream_steal_as_bytes
(part_payload_out)));
+
+ if (!gs_file_open_in_tmpdir (self->tmp_dir, 0644,
+ &part_tempfile, &part_temp_outstream,
+ cancellable, error))
+ goto out;
+ part_in = ot_variant_read (delta_part);
+ if (!ot_gio_splice_get_checksum (part_temp_outstream, part_in,
+ &part_checksum,
+ cancellable, error))
+ goto out;
+
+ checksum_bytes = g_bytes_new (part_checksum, 32);
+ objtype_checksum_array = objtype_checksum_array_new (part_builder->objects);
+ delta_part_header = g_variant_new ("(@aytt ay)",
+ ot_gvariant_new_ay_bytes (checksum_bytes),
+ g_variant_get_size (delta_part),
+ part_builder->uncompressed_size,
+ ot_gvariant_new_ay_bytes (objtype_checksum_array));
+ g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header));
+ g_ptr_array_add (part_tempfiles, g_object_ref (part_tempfile));
+ }
+
+ descriptor_relpath = _ostree_get_relative_static_delta_path (from, to);
+ descriptor_path = g_file_resolve_relative_path (self->repodir, descriptor_relpath);
+ descriptor_dir = g_file_get_parent (descriptor_path);
+
+ if (!gs_file_ensure_directory (descriptor_dir, TRUE, cancellable, error))
+ goto out;
+
+ for (i = 0; i < builder.parts->len; i++)
+ {
+ GFile *tempfile = part_tempfiles->pdata[i];
+ gs_free char *part_relpath = _ostree_get_relative_static_delta_part_path (from, to, i);
+ gs_unref_object GFile *part_path = g_file_resolve_relative_path (self->repodir, part_relpath);
+
+ if (!gs_file_rename (tempfile, part_path, cancellable, error))
+ goto out;
+ }
+
+ if (metadata != NULL)
+ metadata_source = metadata;
+ else
+ {
+ GVariantBuilder tmpbuilder;
+ g_variant_builder_init (&tmpbuilder, G_VARIANT_TYPE ("(a(ss)a(say))"));
+ g_variant_builder_add (&tmpbuilder, "a(ss)", NULL);
+ g_variant_builder_add (&tmpbuilder, "a(say)", NULL);
+ tmp_metadata = g_variant_builder_end (&tmpbuilder);
+ g_variant_ref_sink (tmp_metadata);
+ metadata_source = tmp_metadata;
+ }
+
+ delta_descriptor = g_variant_new ("(@(a(ss)a(say))aya(ayttay))",
+ metadata_source,
+ g_variant_builder_new (G_VARIANT_TYPE ("ay")),
+ part_headers);
+
+ if (!ot_util_variant_save (descriptor_path, delta_descriptor, cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ g_clear_pointer (&builder.parts, g_ptr_array_unref);
+ return ret;
+}
diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c
new file mode 100644
index 0000000..3cd706e
--- /dev/null
+++ b/src/libostree/ostree-repo-static-delta-core.c
@@ -0,0 +1,333 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2013 Colin Walters <walters verbum org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include "ostree-repo-private.h"
+#include "ostree-repo-static-delta-private.h"
+#include "otutil.h"
+
+gboolean
+_ostree_static_delta_parse_checksum_array (GVariant *array,
+ guint8 **out_checksums_array,
+ guint *out_n_checksums,
+ GError **error)
+{
+ gsize n = g_variant_n_children (array);
+ guint n_checksums;
+
+ n_checksums = n / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN;
+
+ if (G_UNLIKELY(n == 0 ||
+ n > (G_MAXUINT32/OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN) ||
+ (n_checksums * OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN) != n))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Invalid checksum array length %" G_GSIZE_FORMAT, n);
+ return FALSE;
+ }
+
+ *out_checksums_array = (gpointer)g_variant_get_data (array);
+ *out_n_checksums = n_checksums;
+
+ return TRUE;
+}
+
+
+/**
+ * ostree_repo_list_static_delta_names:
+ * @self: Repo
+ * @out_deltas: (out) (element-type utf8): String name of deltas (checksum-checksum.delta)
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * This function synchronously enumerates all static deltas in the
+ * repository, returning its result in @out_deltas.
+ */
+gboolean
+ostree_repo_list_static_delta_names (OstreeRepo *self,
+ GPtrArray **out_deltas,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_unref_ptrarray GPtrArray *ret_deltas = NULL;
+ gs_unref_object GFileEnumerator *dir_enum = NULL;
+
+ ret_deltas = g_ptr_array_new_with_free_func (g_free);
+
+ if (g_file_query_exists (self->deltas_dir, NULL))
+ {
+ dir_enum = g_file_enumerate_children (self->deltas_dir, OSTREE_GIO_FAST_QUERYINFO,
+ G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
+ NULL, error);
+ if (!dir_enum)
+ goto out;
+
+ while (TRUE)
+ {
+ GFileInfo *file_info;
+ GFile *child;
+ const char *name;
+
+ if (!gs_file_enumerator_iterate (dir_enum, &file_info, &child,
+ NULL, error))
+ goto out;
+ if (file_info == NULL)
+ break;
+
+ if (g_file_info_get_file_type (file_info) != G_FILE_TYPE_DIRECTORY)
+ continue;
+
+ name = gs_file_get_basename_cached (child);
+
+ {
+ gs_unref_object GFile *meta_path = g_file_get_child (child, "meta");
+
+ if (g_file_query_exists (meta_path, NULL))
+ {
+ g_ptr_array_add (ret_deltas, g_strdup (name));
+ }
+ }
+ }
+ }
+
+ ret = TRUE;
+ gs_transfer_out_value (out_deltas, &ret_deltas);
+ out:
+ return ret;
+}
+
+static gboolean
+have_all_objects (OstreeRepo *repo,
+ GVariant *checksum_array,
+ gboolean *out_have_all,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ guint8 *checksums_data;
+ guint i,n_checksums;
+ gboolean have_object = TRUE;
+
+ if (!_ostree_static_delta_parse_checksum_array (checksum_array,
+ &checksums_data,
+ &n_checksums,
+ error))
+ goto out;
+
+ for (i = 0; i < n_checksums; i++)
+ {
+ guint8 objtype = *checksums_data;
+ const guint8 *csum = checksums_data + 1;
+ char tmp_checksum[65];
+
+ if (G_UNLIKELY(!ostree_validate_structureof_objtype (objtype, error)))
+ goto out;
+
+ ostree_checksum_inplace_from_bytes (csum, tmp_checksum);
+
+ if (!ostree_repo_has_object (repo, (OstreeObjectType) objtype, tmp_checksum,
+ &have_object, cancellable, error))
+ goto out;
+
+ if (!have_object)
+ break;
+
+ checksums_data += OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN;
+ }
+
+ ret = TRUE;
+ *out_have_all = have_object;
+ out:
+ return ret;
+}
+
+static gboolean
+zlib_uncompress_data (GBytes *data,
+ GBytes **out_uncompressed,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_unref_object GMemoryInputStream *memin = (GMemoryInputStream*)g_memory_input_stream_new_from_bytes
(data);
+ gs_unref_object GMemoryOutputStream *memout = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0,
g_realloc, g_free);
+ gs_unref_object GConverter *zlib_decomp =
+ (GConverter*) g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW);
+ gs_unref_object GInputStream *convin = g_converter_input_stream_new ((GInputStream*)memin, zlib_decomp);
+
+ if (0 > g_output_stream_splice ((GOutputStream*)memout, convin,
+ G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE |
+ G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
+ cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ *out_uncompressed = g_memory_output_stream_steal_as_bytes (memout);
+ out:
+ return ret;
+}
+
+/**
+ * ostree_repo_static_delta_execute_offline:
+ * @self: Repo
+ * @dir: Path to a directory containing static delta data
+ * @skip_validation: If %TRUE, assume data integrity
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Given a directory representing an already-downloaded static delta
+ * on disk, apply it, generating a new commit. The directory must be
+ * named with the form "FROM-TO", where both are checksums, and it
+ * must contain a file named "meta", along with at least one part.
+ */
+gboolean
+ostree_repo_static_delta_execute_offline (OstreeRepo *self,
+ GFile *dir,
+ gboolean skip_validation,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ guint i, n;
+ gs_unref_object GFile *meta_file = g_file_get_child (dir, "meta");
+ gs_unref_variant GVariant *meta = NULL;
+ gs_unref_variant GVariant *headers = NULL;
+
+ if (!ot_util_variant_map (meta_file, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_META_FORMAT),
+ FALSE, &meta, error))
+ goto out;
+
+ headers = g_variant_get_child_value (meta, 2);
+ n = g_variant_n_children (headers);
+ for (i = 0; i < n; i++)
+ {
+ guint64 size;
+ guint64 usize;
+ const guchar *csum;
+ gboolean have_all;
+ gs_unref_variant GVariant *header = NULL;
+ gs_unref_variant GVariant *csum_v = NULL;
+ gs_unref_variant GVariant *objects = NULL;
+ gs_unref_object GFile *part_path = NULL;
+ gs_unref_variant GVariant *part = NULL;
+ gs_unref_object GInputStream *raw_in = NULL;
+ gs_unref_object GInputStream *in = NULL;
+
+ header = g_variant_get_child_value (headers, i);
+ g_variant_get (header, "(@aytt ay)", &csum_v, &size, &usize, &objects);
+
+ if (!have_all_objects (self, objects, &have_all, cancellable, error))
+ goto out;
+
+ /* If we already have these objects, don't bother executing the
+ * static delta.
+ */
+ if (have_all)
+ continue;
+
+ csum = ostree_checksum_bytes_peek_validate (csum_v, error);
+ if (!csum)
+ goto out;
+
+ part_path = ot_gfile_resolve_path_printf (dir, "%u", i);
+
+ in = (GInputStream*)g_file_read (part_path, cancellable, error);
+ if (!in)
+ goto out;
+
+ if (!skip_validation)
+ {
+ gs_unref_object GInputStream *tmp_in = NULL;
+ gs_free guchar *actual_checksum = NULL;
+
+ tmp_in = (GInputStream*)g_file_read (part_path, cancellable, error);
+ if (!tmp_in)
+ goto out;
+
+ if (!ot_gio_checksum_stream (tmp_in, &actual_checksum,
+ cancellable, error))
+ goto out;
+
+ if (ostree_cmp_checksum_bytes (csum, actual_checksum) != 0)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Checksum mismatch in static delta %s part %u",
+ gs_file_get_path_cached (dir), i);
+ goto out;
+ }
+ }
+
+ {
+ GMappedFile *mfile = gs_file_map_noatime (part_path, cancellable, error);
+ gs_unref_bytes GBytes *bytes = NULL;
+ gs_unref_bytes GBytes *payload = NULL;
+ gsize partlen;
+ const guint8*partdata;
+
+ if (!mfile)
+ goto out;
+
+ bytes = g_mapped_file_get_bytes (mfile);
+ g_mapped_file_unref (mfile);
+
+ partdata = g_bytes_get_data (bytes, &partlen);
+
+ if (partlen < 1)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Corrupted 0 length byte part %s/%i",
+ gs_file_get_basename_cached (dir),
+ i);
+ goto out;
+ }
+
+ switch (partdata[0])
+ {
+ case 0:
+ payload = g_bytes_new_from_bytes (bytes, 1, partlen - 1);
+ break;
+ case 'g':
+ {
+ gs_unref_bytes GBytes *subbytes = g_bytes_new_from_bytes (bytes, 1, partlen - 1);
+ if (!zlib_uncompress_data (subbytes, &payload,
+ cancellable, error))
+ goto out;
+ }
+ break;
+ }
+
+ part = ot_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT),
+ payload, FALSE);
+
+
+ if (!_ostree_static_delta_part_execute (self, objects, part, cancellable, error))
+ {
+ g_prefix_error (error, "executing delta part %i: ", i);
+ goto out;
+ }
+ }
+ }
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
diff --git a/src/libostree/ostree-repo-static-delta-private.h
b/src/libostree/ostree-repo-static-delta-private.h
new file mode 100644
index 0000000..772a501
--- /dev/null
+++ b/src/libostree/ostree-repo-static-delta-private.h
@@ -0,0 +1,96 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2013 Colin Walters <walters verbum org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#pragma once
+
+#include "ostree-core.h"
+
+G_BEGIN_DECLS
+
+/* Arbitrarily chosen */
+#define OSTREE_STATIC_DELTA_PART_MAX_SIZE_BYTES (16*1024*1024)
+/* 1 byte for object type, 32 bytes for checksum */
+#define OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN 33
+
+/**
+ * OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT:
+ *
+ * ay data source
+ * ay operations
+ */
+#define OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT "(ayay)"
+
+/**
+ * OSTREE_STATIC_DELTA_META_ENTRY_FORMAT:
+ *
+ * ay checksum
+ * guint64 size: Total size of delta (sum of parts)
+ * guint64 usize: Uncompressed size of resulting objects on disk
+ * ARRAY[(guint8 objtype, csum object)]
+ *
+ * The checksum is of the delta payload, and each entry in the array
+ * represents an OSTree object which will be created by the deltapart.
+ */
+
+#define OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "(ayttay)"
+
+/**
+ * OSTREE_STATIC_DELTA_META_FORMAT:
+ *
+ * A .delta object is a custom binary format. It has the following high
+ * level form:
+ *
+ * delta-descriptor:
+ * metadata: a{sv}
+ * ARRAY[(csum from, csum to)]: ay
+ * ARRAY[delta-part-header]
+ *
+ * The metadata would include things like a version number, as well as
+ * extended verification data like a GPG signature.
+ *
+ * The second array is an array of delta objects that should be
+ * fetched and applied before this one. This is a fairly generic
+ * recursion mechanism that would potentially allow saving significant
+ * storage space on the server.
+ */
+#define OSTREE_STATIC_DELTA_META_FORMAT "(a{sv}aya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT ")"
+
+gboolean _ostree_static_delta_part_execute (OstreeRepo *repo,
+ GVariant *header,
+ GVariant *part,
+ GCancellable *cancellable,
+ GError **error);
+
+typedef enum {
+ OSTREE_STATIC_DELTA_OP_FETCH = 1,
+ OSTREE_STATIC_DELTA_OP_WRITE = 2,
+ OSTREE_STATIC_DELTA_OP_GUNZIP = 3,
+ OSTREE_STATIC_DELTA_OP_CLOSE = 4,
+ OSTREE_STATIC_DELTA_OP_READOBJECT = 5,
+ OSTREE_STATIC_DELTA_OP_READPAYLOAD = 6
+} OstreeStaticDeltaOpCode;
+
+gboolean
+_ostree_static_delta_parse_checksum_array (GVariant *array,
+ guint8 **out_checksums_array,
+ guint *out_n_checksums,
+ GError **error);
+G_END_DECLS
+
diff --git a/src/libostree/ostree-repo-static-delta-processing.c
b/src/libostree/ostree-repo-static-delta-processing.c
new file mode 100644
index 0000000..a373f00
--- /dev/null
+++ b/src/libostree/ostree-repo-static-delta-processing.c
@@ -0,0 +1,381 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2013,2014 Colin Walters <walters verbum org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include <string.h>
+
+#include "ostree-repo-private.h"
+#include "ostree-repo-static-delta-private.h"
+#include "otutil.h"
+#include "ostree-varint.h"
+
+/* This should really always be true, but hey, let's just assert it */
+G_STATIC_ASSERT (sizeof (guint) >= sizeof (guint32));
+
+typedef struct {
+ guint checksum_index;
+ const guint8 *checksums;
+ guint n_checksums;
+
+ const guint8 *opdata;
+ guint oplen;
+
+ OstreeObjectType output_objtype;
+ const guint8 *output_target;
+ GFile *output_tmp_path;
+ GOutputStream *output_tmp_stream;
+ const guint8 *input_target_csum;
+
+ const guint8 *payload_data;
+ guint64 payload_size;
+} StaticDeltaExecutionState;
+
+typedef gboolean (*DispatchOpFunc) (OstreeRepo *repo,
+ StaticDeltaExecutionState *state,
+ GCancellable *cancellable,
+ GError **error);
+
+typedef struct {
+ const char *name;
+ DispatchOpFunc func;
+} OstreeStaticDeltaOperation;
+
+#define OPPROTO(name) \
+ static gboolean dispatch_##name (OstreeRepo *repo, \
+ StaticDeltaExecutionState *state, \
+ GCancellable *cancellable, \
+ GError **error);
+
+OPPROTO(fetch)
+OPPROTO(write)
+OPPROTO(gunzip)
+OPPROTO(close)
+#undef OPPROTO
+
+static OstreeStaticDeltaOperation op_dispatch_table[] = {
+ { "fetch", dispatch_fetch },
+ { "write", dispatch_write },
+ { "gunzip", dispatch_gunzip },
+ { "close", dispatch_close },
+ { NULL }
+};
+
+static gboolean
+open_output_target_csum (OstreeRepo *repo,
+ StaticDeltaExecutionState *state,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ guint8 *objcsum;
+
+ g_assert (state->checksums != NULL);
+ g_assert (state->output_target == NULL);
+ g_assert (state->output_tmp_path == NULL);
+ g_assert (state->output_tmp_stream == NULL);
+ g_assert (state->checksum_index < state->n_checksums);
+
+ objcsum = (guint8*)state->checksums + (state->checksum_index * OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN);
+
+ if (G_UNLIKELY(!ostree_validate_structureof_objtype (*objcsum, error)))
+ goto out;
+
+ state->output_objtype = (OstreeObjectType) *objcsum;
+ state->output_target = objcsum + 1;
+ if (!gs_file_open_in_tmpdir (repo->tmp_dir, 0644,
+ &state->output_tmp_path, &state->output_tmp_stream,
+ cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+
+gboolean
+_ostree_static_delta_part_execute (OstreeRepo *repo,
+ GVariant *objects,
+ GVariant *part,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ guint8 *checksums_data;
+ gs_unref_variant GVariant *checksums = NULL;
+ gs_unref_variant GVariant *payload = NULL;
+ gs_unref_variant GVariant *ops = NULL;
+ StaticDeltaExecutionState statedata = { 0, };
+ StaticDeltaExecutionState *state = &statedata;
+ guint n_executed = 0;
+
+ if (!_ostree_static_delta_parse_checksum_array (objects,
+ &checksums_data,
+ &state->n_checksums,
+ error))
+ goto out;
+
+ state->checksums = checksums_data;
+ g_assert (state->n_checksums > 0);
+ if (!open_output_target_csum (repo, state, cancellable, error))
+ goto out;
+
+ g_variant_get (part, "(@ay ay)", &payload, &ops);
+
+ state->payload_data = g_variant_get_data (payload);
+ state->payload_size = g_variant_get_size (payload);
+
+ state->oplen = g_variant_n_children (ops);
+ state->opdata = g_variant_get_data (ops);
+ while (state->oplen > 0)
+ {
+ guint8 opcode = state->opdata[0];
+ OstreeStaticDeltaOperation *op;
+
+ if (G_UNLIKELY (opcode == 0 || opcode >= G_N_ELEMENTS (op_dispatch_table)))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
+ "Out of range opcode %u at offset %u", opcode, n_executed);
+ goto out;
+ }
+ op = &op_dispatch_table[opcode-1];
+ g_printerr ("dispatch %u\n", opcode-1);
+ state->oplen--;
+ state->opdata++;
+ if (!op->func (repo, state, cancellable, error))
+ goto out;
+
+ n_executed++;
+ }
+
+ ret = TRUE;
+ out:
+ return ret;
+}
+
+static gboolean
+dispatch_fetch (OstreeRepo *repo,
+ StaticDeltaExecutionState *state,
+ GCancellable *cancellable,
+ GError **error)
+{
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
+ "Static delta fetch opcode is not implemented in this version");
+ return FALSE;
+}
+
+static gboolean
+read_varuint64 (StaticDeltaExecutionState *state,
+ guint64 *out_value,
+ GError **error)
+{
+ gsize bytes_read;
+ if (!_ostree_read_varuint64 (state->opdata, state->oplen, out_value, &bytes_read))
+ {
+ g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+ "Unexpected EOF reading varint");
+ return FALSE;
+ }
+ state->opdata += bytes_read;
+ state->oplen -= bytes_read;
+ return TRUE;
+}
+
+static gboolean
+validate_ofs (StaticDeltaExecutionState *state,
+ guint64 offset,
+ guint64 length,
+ GError **error)
+{
+ if (G_UNLIKELY (offset + length < offset ||
+ offset + length > state->payload_size))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
+ "Invalid offset/length %" G_GUINT64_FORMAT "/%" G_GUINT64_FORMAT,
+ offset, length);
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static gboolean
+dispatch_write (OstreeRepo *repo,
+ StaticDeltaExecutionState *state,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ guint64 offset;
+ guint64 length;
+ gsize bytes_written;
+
+ if (G_UNLIKELY(state->oplen < 2))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
+ "Expected at least 2 bytes for write op");
+ goto out;
+ }
+ if (!read_varuint64 (state, &offset, error))
+ goto out;
+ if (!read_varuint64 (state, &length, error))
+ goto out;
+
+ if (!validate_ofs (state, offset, length, error))
+ goto out;
+
+ if (!g_output_stream_write_all (state->output_tmp_stream,
+ state->payload_data + offset,
+ length,
+ &bytes_written,
+ cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ if (!ret)
+ g_prefix_error (error, "opcode write: ");
+ return ret;
+}
+
+static gboolean
+dispatch_gunzip (OstreeRepo *repo,
+ StaticDeltaExecutionState *state,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ guint64 offset;
+ guint64 length;
+ gs_unref_object GConverter *zlib_decomp = NULL;
+ gs_unref_object GInputStream *payload_in = NULL;
+ gs_unref_object GInputStream *zlib_in = NULL;
+
+ if (G_UNLIKELY(state->oplen < 2))
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
+ "Expected at least 2 bytes for gunzip op");
+ goto out;
+ }
+ if (!read_varuint64 (state, &offset, error))
+ goto out;
+ if (!read_varuint64 (state, &length, error))
+ goto out;
+
+ if (!validate_ofs (state, offset, length, error))
+ goto out;
+
+ payload_in = g_memory_input_stream_new_from_data (state->payload_data + offset, length, NULL);
+ zlib_decomp = (GConverter*)g_zlib_decompressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW);
+ zlib_in = g_converter_input_stream_new (payload_in, zlib_decomp);
+
+ if (0 > g_output_stream_splice (state->output_tmp_stream, zlib_in,
+ G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE,
+ cancellable, error))
+ goto out;
+
+ ret = TRUE;
+ out:
+ if (!ret)
+ g_prefix_error (error, "opcode gunzip: ");
+ return ret;
+}
+
+static gboolean
+dispatch_close (OstreeRepo *repo,
+ StaticDeltaExecutionState *state,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ char tmp_checksum[65];
+
+ if (state->checksum_index == state->n_checksums)
+ {
+ g_set_error (error, G_IO_ERROR, G_IO_ERROR_INVALID_ARGUMENT,
+ "Too many close operations");
+ goto out;
+ }
+
+ g_assert (state->output_tmp_stream);
+ g_assert (state->output_tmp_path);
+
+ if (!g_output_stream_close (state->output_tmp_stream, cancellable, error))
+ goto out;
+
+ g_clear_object (&state->output_tmp_stream);
+
+ ostree_checksum_inplace_from_bytes (state->output_target, tmp_checksum);
+
+ if (OSTREE_OBJECT_TYPE_IS_META (state->output_objtype))
+ {
+ gs_unref_variant GVariant *metadata = NULL;
+
+ if (!ot_util_variant_map (state->output_tmp_path,
+ ostree_metadata_variant_type (state->output_objtype),
+ TRUE, &metadata, error))
+ goto out;
+
+ if (!ostree_repo_write_metadata (repo, state->output_objtype, tmp_checksum,
+ metadata, NULL, cancellable, error))
+ goto out;
+
+ g_print ("Wrote metadata object '%s'\n",
+ tmp_checksum);
+ }
+ else
+ {
+ gs_unref_object GInputStream *in = NULL;
+ gs_unref_object GFileInfo *info = NULL;
+
+ in = (GInputStream*)g_file_read (state->output_tmp_path, cancellable, error);
+ if (!in)
+ goto out;
+
+ info = g_file_input_stream_query_info ((GFileInputStream*)in, G_FILE_ATTRIBUTE_STANDARD_SIZE,
+ cancellable, error);
+ if (!info)
+ goto out;
+
+ if (!ostree_repo_write_content (repo, tmp_checksum, in,
+ g_file_info_get_size (info), NULL,
+ cancellable, error))
+ goto out;
+
+ g_print ("Wrote content object '%s'\n",
+ tmp_checksum);
+ }
+
+ state->output_target = NULL;
+ g_clear_object (&state->output_tmp_path);
+
+ state->checksum_index++;
+ if (state->checksum_index < state->n_checksums)
+ {
+ if (!open_output_target_csum (repo, state, cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ if (!ret)
+ g_prefix_error (error, "opcode close: ");
+ return ret;
+}
diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c
index 65c2bdf..aab625a 100644
--- a/src/libostree/ostree-repo.c
+++ b/src/libostree/ostree-repo.c
@@ -99,6 +99,7 @@ ostree_repo_finalize (GObject *object)
g_clear_object (&self->objects_dir);
if (self->objects_dir_fd != -1)
(void) close (self->objects_dir_fd);
+ g_clear_object (&self->deltas_dir);
g_clear_object (&self->uncompressed_objects_dir);
if (self->uncompressed_objects_dir_fd != -1)
(void) close (self->uncompressed_objects_dir_fd);
@@ -175,6 +176,8 @@ ostree_repo_constructed (GObject *object)
self->objects_dir = g_file_get_child (self->repodir, "objects");
self->uncompressed_objects_dir = g_file_resolve_relative_path (self->repodir,
"uncompressed-objects-cache/objects");
+ self->deltas_dir = g_file_get_child (self->repodir, "deltas");
+ self->uncompressed_objects_dir = g_file_get_child (self->repodir, "uncompressed-objects-cache");
self->remote_cache_dir = g_file_get_child (self->repodir, "remote-cache");
self->config_file = g_file_get_child (self->repodir, "config");
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index b3f263e..df7f3e0 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -426,6 +426,37 @@ gboolean ostree_repo_list_objects (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
+gboolean ostree_repo_list_static_delta_names (OstreeRepo *self,
+ GPtrArray **out_deltas,
+ GCancellable *cancellable,
+ GError **error);
+
+/**
+ * OstreeStaticDeltaGenerateOpt:
+ * @OSTREE_STATIC_DELTA_GENERATE_OPT_LOWLATENCY: Optimize for speed of delta creation over space
+ * @OSTREE_STATIC_DELTA_GENERATE_OPT_MAJOR: Optimize for delta size (may be very slow)
+ *
+ * Parameters controlling optimization of static deltas.
+ */
+typedef enum {
+ OSTREE_STATIC_DELTA_GENERATE_OPT_LOWLATENCY,
+ OSTREE_STATIC_DELTA_GENERATE_OPT_MAJOR
+} OstreeStaticDeltaGenerateOpt;
+
+gboolean ostree_repo_static_delta_generate (OstreeRepo *self,
+ OstreeStaticDeltaGenerateOpt opt,
+ const char *from,
+ const char *to,
+ GVariant *metadata,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean ostree_repo_static_delta_execute_offline (OstreeRepo *self,
+ GFile *dir,
+ gboolean skip_validation,
+ GCancellable *cancellable,
+ GError **error);
+
GHashTable *ostree_repo_traverse_new_reachable (void);
gboolean ostree_repo_traverse_commit (OstreeRepo *repo,
diff --git a/src/libostree/ostree-varint.c b/src/libostree/ostree-varint.c
index 04ef6b8..800aaa6 100644
--- a/src/libostree/ostree-varint.c
+++ b/src/libostree/ostree-varint.c
@@ -62,13 +62,15 @@ static const int max_varint_bytes = 10;
* _ostree_read_varuint64:
* @buf: (array length=buflen): Byte buffer
* @buflen: Length of bytes in @buf
+ * @out_value: (out): Value
* @bytes_read: (out): Number of bytes read
*
- * Returns: An unsigned 64 bit integer value
+ * Returns: %TRUE on success, %FALSE on end of stream
*/
-guint64
+gboolean
_ostree_read_varuint64 (const guint8 *buf,
gsize buflen,
+ guint64 *out_value,
gsize *bytes_read)
{
guint64 result = 0;
@@ -92,8 +94,9 @@ _ostree_read_varuint64 (const guint8 *buf,
} while (b & 0x80);
*bytes_read = count;
+ *out_value = result;
- return result;
+ return TRUE;
}
/**
diff --git a/src/libostree/ostree-varint.h b/src/libostree/ostree-varint.h
index 1a47d4d..ae1152b 100644
--- a/src/libostree/ostree-varint.h
+++ b/src/libostree/ostree-varint.h
@@ -24,9 +24,10 @@
G_BEGIN_DECLS
-guint64 _ostree_read_varuint64 (const guint8 *buf,
- gsize buflen,
- gsize *bytes_read);
+gboolean _ostree_read_varuint64 (const guint8 *buf,
+ gsize buflen,
+ guint64 *out_value,
+ gsize *bytes_read);
void _ostree_write_varuint64 (GString *buf, guint64 n);
diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c
index 291f746..0f8e4be 100644
--- a/src/libotutil/ot-variant-utils.c
+++ b/src/libotutil/ot-variant-utils.c
@@ -23,8 +23,10 @@
#include "config.h"
#include <gio/gio.h>
+#include <gio/gfiledescriptorbased.h>
#include <string.h>
+#include <sys/mman.h>
#include "otutil.h"
@@ -149,6 +151,55 @@ ot_util_variant_map (GFile *src,
return ret;
}
+typedef struct {
+ gpointer addr;
+ gsize len;
+} VariantMapData;
+
+static void
+variant_map_data_destroy (gpointer data)
+{
+ VariantMapData *mdata = data;
+ (void) munmap (mdata->addr, mdata->len);
+}
+
+gboolean
+ot_util_variant_map_fd (GFileDescriptorBased *stream,
+ goffset start,
+ const GVariantType *type,
+ gboolean trusted,
+ GVariant **out_variant,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gpointer map;
+ struct stat stbuf;
+ VariantMapData *mdata = NULL;
+ gsize len;
+
+ if (!gs_stream_fstat (stream, &stbuf, NULL, error))
+ goto out;
+
+ len = stbuf.st_size - start;
+ map = mmap (NULL, len, PROT_READ, MAP_PRIVATE,
+ g_file_descriptor_based_get_fd (stream), start);
+ if (!map)
+ {
+ ot_util_set_error_from_errno (error, errno);
+ goto out;
+ }
+
+ mdata = g_new (VariantMapData, 1);
+ mdata->addr = map;
+ mdata->len = len;
+
+ ret = TRUE;
+ *out_variant = g_variant_new_from_data (type, map, len, trusted,
+ variant_map_data_destroy, mdata);
+ out:
+ return ret;
+}
+
/**
* Read all input from @src, allocating a new #GVariant from it into
* output variable @out_variant. @src will be closed as a result.
@@ -219,3 +270,17 @@ ot_util_variant_builder_from_variant (GVariant *variant,
return builder;
}
+GVariant *
+ot_variant_new_from_bytes (const GVariantType *type,
+ GBytes *bytes,
+ gboolean trusted)
+{
+#if GLIB_VERSION_MIN_REQUIRED >= GLIB_VERSION_2_36
+ return g_variant_new_from_bytes (type, bytes, trusted);
+#else
+ gsize size;
+ gconstpointer data = g_bytes_get_data (bytes, &size);
+ return g_variant_new_from_data (type, data, size, trusted,
+ (GDestroyNotify)g_bytes_unref, bytes);
+#endif
+}
diff --git a/src/libotutil/ot-variant-utils.h b/src/libotutil/ot-variant-utils.h
index 92746a2..218e543 100644
--- a/src/libotutil/ot-variant-utils.h
+++ b/src/libotutil/ot-variant-utils.h
@@ -46,6 +46,13 @@ gboolean ot_util_variant_map (GFile *src,
GVariant **out_variant,
GError **error);
+gboolean ot_util_variant_map_fd (GFileDescriptorBased *stream,
+ goffset offset,
+ const GVariantType *type,
+ gboolean trusted,
+ GVariant **out_variant,
+ GError **error);
+
gboolean ot_util_variant_from_stream (GInputStream *src,
const GVariantType *type,
gboolean trusted,
@@ -58,5 +65,10 @@ GInputStream *ot_variant_read (GVariant *variant);
GVariantBuilder *ot_util_variant_builder_from_variant (GVariant *variant,
const GVariantType *type);
+GVariant *
+ot_variant_new_from_bytes (const GVariantType *type,
+ GBytes *bytes,
+ gboolean trusted);
+
G_END_DECLS
diff --git a/src/ostree/main.c b/src/ostree/main.c
index c863252..5463ce1 100644
--- a/src/ostree/main.c
+++ b/src/ostree/main.c
@@ -54,6 +54,7 @@ static OstreeCommand commands[] = {
{ "remote", ostree_builtin_remote, 0 },
{ "rev-parse", ostree_builtin_rev_parse, 0 },
{ "show", ostree_builtin_show, 0 },
+ { "static-delta", ostree_builtin_static_delta, 0 },
#ifdef HAVE_LIBSOUP
{ "trivial-httpd", ostree_builtin_trivial_httpd, OSTREE_BUILTIN_FLAG_NO_REPO },
#endif
diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c
new file mode 100644
index 0000000..be535a6
--- /dev/null
+++ b/src/ostree/ot-builtin-static-delta.c
@@ -0,0 +1,129 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2013 Colin Walters <walters verbum org>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include "ot-builtins.h"
+#include "ostree.h"
+#include "otutil.h"
+
+static char *opt_from_rev;
+static char *opt_to_rev;
+static char *opt_apply;
+
+static GOptionEntry options[] = {
+ { "from", 0, 0, G_OPTION_ARG_STRING, &opt_from_rev, "Create delta from revision REV", "REV" },
+ { "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Create delta to revision REV", "REV" },
+ { "apply", 0, 0, G_OPTION_ARG_FILENAME, &opt_apply, "Apply delta from PATH", "PATH" },
+ { NULL }
+};
+
+gboolean
+ostree_builtin_static_delta (int argc, char **argv, OstreeRepo *repo, GCancellable *cancellable, GError
**error)
+{
+ gboolean ret = FALSE;
+ GOptionContext *context;
+ gs_unref_ptrarray GPtrArray *delta_names = NULL;
+
+ context = g_option_context_new ("Manage static delta files");
+ g_option_context_add_main_entries (context, options, NULL);
+
+ if (!g_option_context_parse (context, &argc, &argv, error))
+ goto out;
+
+ if (opt_apply)
+ {
+ gs_unref_object GFile *path = g_file_new_for_path (opt_apply);
+
+ if (!ostree_repo_prepare_transaction (repo, NULL, cancellable, error))
+ goto out;
+
+ if (!ostree_repo_static_delta_execute_offline (repo, path, TRUE, cancellable, error))
+ goto out;
+
+ if (!ostree_repo_commit_transaction (repo, NULL, cancellable, error))
+ goto out;
+ }
+ else
+ {
+ if (argc >= 2 && opt_to_rev == NULL)
+ opt_to_rev = argv[1];
+
+ if (argc < 2 && opt_to_rev == NULL)
+ {
+ guint i;
+ if (!ostree_repo_list_static_delta_names (repo, &delta_names, cancellable, error))
+ goto out;
+
+ if (delta_names->len == 0)
+ {
+ g_print ("(No static deltas)\n");
+ }
+ else
+ {
+ for (i = 0; i < delta_names->len; i++)
+ {
+ g_print ("%s\n", (char*)delta_names->pdata[i]);
+ }
+ }
+ }
+ else if (opt_to_rev != NULL)
+ {
+ const char *from_source;
+ gs_free char *from_resolved = NULL;
+ gs_free char *to_resolved = NULL;
+ gs_free char *from_parent_str = NULL;
+
+ if (opt_from_rev == NULL)
+ {
+ from_parent_str = g_strconcat (opt_to_rev, "^", NULL);
+ from_source = from_parent_str;
+ }
+ else
+ {
+ from_source = opt_from_rev;
+ }
+
+ if (!ostree_repo_resolve_rev (repo, from_source, FALSE, &from_resolved, error))
+ goto out;
+ if (!ostree_repo_resolve_rev (repo, opt_to_rev, FALSE, &to_resolved, error))
+ goto out;
+
+ g_print ("Generating static delta:\n");
+ g_print (" From: %s\n", from_resolved);
+ g_print (" To: %s\n", to_resolved);
+ if (!ostree_repo_static_delta_generate (repo, OSTREE_STATIC_DELTA_GENERATE_OPT_MAJOR,
+ from_resolved, to_resolved, NULL,
+ cancellable, error))
+ goto out;
+ }
+ else
+ {
+ ot_util_usage_error (context, "--from=REV must be specified", error);
+ goto out;
+ }
+ }
+
+ ret = TRUE;
+ out:
+ if (context)
+ g_option_context_free (context);
+ return ret;
+}
diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h
index cbd7847..fee66f2 100644
--- a/src/ostree/ot-builtins.h
+++ b/src/ostree/ot-builtins.h
@@ -45,6 +45,7 @@ BUILTINPROTO(refs);
BUILTINPROTO(reset);
BUILTINPROTO(fsck);
BUILTINPROTO(show);
+BUILTINPROTO(static_delta);
BUILTINPROTO(rev_parse);
BUILTINPROTO(remote);
BUILTINPROTO(write_refs);
diff --git a/tests/test-delta.sh b/tests/test-delta.sh
new file mode 100755
index 0000000..c5de5b4
--- /dev/null
+++ b/tests/test-delta.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+# Copyright (C) 2011,2013 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -e
+
+. $(dirname $0)/libtest.sh
+
+bindatafiles="bash true ostree"
+morebindatafiles="false ls"
+
+echo '1..2'
+
+mkdir repo
+ostree --repo=repo init --mode=archive-z2
+
+mkdir files
+for bin in ${bindatafiles}; do
+ cp $(which ${bin}) files
+done
+
+ostree --repo=repo commit -b test -s test --tree=dir=files
+
+function permuteFile() {
+ permutation=$(($1 % 2))
+ output=$2
+ case $permutation in
+ 0) dd if=/dev/zero count=40 bs=1 >> $output;;
+ 1) echo aheader | cat - $output >> $output.new && mv $output.new $output;;
+ esac
+}
+
+function permuteDirectory() {
+ permutation=$1
+ dir=$2
+ for x in ${dir}/*; do
+ for z in $(seq ${permutation}); do
+ permuteFile ${z} ${x}
+ done
+ done
+}
+
+permuteDirectory 1 files
+ostree --repo=repo commit -b test -s test --tree=dir=files
+ostree static-delta --repo=repo
+
+origrev=$(ostree --repo=repo rev-parse test^)
+newrev=$(ostree --repo=repo rev-parse test)
+ostree static-delta --repo=repo --from=${origrev} --to=${newrev}
+
+assert_has_dir repo/deltas/${origrev}-${newrev}
+
+mkdir repo2
+ostree --repo=repo2 init --mode=archive-z2
+ostree --repo=repo2 pull-local repo ${origrev}
+
+ostree --repo=repo2 static-delta --apply=repo/deltas/${origrev}-${newrev}
+ostree --repo=repo2 fsck
+ostree --repo=repo2 show ${newrev}
diff --git a/tests/test-varint.c b/tests/test-varint.c
index 9fbc7f3..1dc6ec1 100644
--- a/tests/test-varint.c
+++ b/tests/test-varint.c
@@ -38,7 +38,7 @@ check_one_roundtrip (guint64 val)
gs_free char *data = g_variant_print (v, FALSE);
g_test_message ("%" G_GUINT64_FORMAT " -> %s", val, data);
}
- newval = _ostree_read_varuint64 ((guint8*)buf->str, buf->len, &bytes_read);
+ g_assert (_ostree_read_varuint64 ((guint8*)buf->str, buf->len, &newval, &bytes_read));
g_assert_cmpint (bytes_read, <=, 10);
g_assert_cmpint (val, ==, newval);
}
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
