[ostree] Add an internal API to stream content objects
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [ostree] Add an internal API to stream content objects
- Date: Thu, 12 Feb 2015 16:43:50 +0000 (UTC)
commit 247866a9bca3557fb4a22a772a034eb89682b6a3
Author: Colin Walters <walters verbum org>
Date: Thu Jan 29 21:23:05 2015 -0500
Add an internal API to stream content objects
For future delta work where we do more interesting things than just
"tar of new objects", this lays the groundwork for doing streaming
writes into content objects.
It's also more efficient, as we avoid many intermediate allocations
and virtual calls. Just a single `g_output_stream_write_all` for the
splice case.
Conflicts:
src/libostree/ostree-repo-private.h
src/libostree/ostree-repo-static-delta-processing.c
src/libostree/ostree-repo-commit.c | 178 +++++++++++++++++++++++++----------
src/libostree/ostree-repo-private.h | 26 +++++
2 files changed, 155 insertions(+), 49 deletions(-)
---
diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c
index c6ac572..054ed65 100644
--- a/src/libostree/ostree-repo-commit.c
+++ b/src/libostree/ostree-repo-commit.c
@@ -74,7 +74,9 @@ _ostree_repo_get_tmpobject_path (OstreeRepo *repo,
}
static GVariant *
-create_file_metadata (GFileInfo *file_info,
+create_file_metadata (guint32 uid,
+ guint32 gid,
+ guint32 mode,
GVariant *xattrs)
{
GVariant *ret_metadata = NULL;
@@ -84,9 +86,9 @@ create_file_metadata (GFileInfo *file_info,
tmp_xattrs = g_variant_ref_sink (g_variant_new_array (G_VARIANT_TYPE ("(ayay)"), NULL, 0));
ret_metadata = g_variant_new ("(uuu a(ayay))",
- GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::uid")),
- GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::gid")),
- GUINT32_TO_BE (g_file_info_get_attribute_uint32 (file_info, "unix::mode")),
+ GUINT32_TO_BE (uid),
+ GUINT32_TO_BE (gid),
+ GUINT32_TO_BE (mode),
xattrs ? xattrs : tmp_xattrs);
g_variant_ref_sink (ret_metadata);
@@ -95,14 +97,16 @@ create_file_metadata (GFileInfo *file_info,
static gboolean
write_file_metadata_to_xattr (int fd,
- GFileInfo *file_info,
+ guint32 uid,
+ guint32 gid,
+ guint32 mode,
GVariant *xattrs,
GError **error)
{
gs_unref_variant GVariant *filemeta = NULL;
int res;
- filemeta = create_file_metadata (file_info, xattrs);
+ filemeta = create_file_metadata (uid, gid, mode, xattrs);
do
res = fsetxattr (fd, "user.ostreemeta",
@@ -125,12 +129,13 @@ commit_loose_object_trusted (OstreeRepo *self,
const char *checksum,
OstreeObjectType objtype,
const char *loose_path,
- GFile *temp_file,
const char *temp_filename,
gboolean object_is_symlink,
- GFileInfo *file_info,
+ guint32 uid,
+ guint32 gid,
+ guint32 mode,
GVariant *xattrs,
- GOutputStream *temp_out,
+ int fd,
GCancellable *cancellable,
GError **error)
{
@@ -162,8 +167,7 @@ commit_loose_object_trusted (OstreeRepo *self,
* as regular files.
*/
if (G_UNLIKELY (fchownat (self->tmp_dir_fd, temp_filename,
- g_file_info_get_attribute_uint32 (file_info, "unix::uid"),
- g_file_info_get_attribute_uint32 (file_info, "unix::gid"),
+ uid, gid,
AT_SYMLINK_NOFOLLOW) == -1))
{
gs_set_error_from_errno (error, errno);
@@ -179,22 +183,13 @@ commit_loose_object_trusted (OstreeRepo *self,
}
else
{
- int fd;
int res;
struct timespec times[2];
- g_assert (temp_out != NULL);
-
- fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)temp_out);
-
if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE)
{
- g_assert (file_info != NULL);
-
do
- res = fchown (fd,
- g_file_info_get_attribute_uint32 (file_info, "unix::uid"),
- g_file_info_get_attribute_uint32 (file_info, "unix::gid"));
+ res = fchown (fd, uid, gid);
while (G_UNLIKELY (res == -1 && errno == EINTR));
if (G_UNLIKELY (res == -1))
{
@@ -203,7 +198,7 @@ commit_loose_object_trusted (OstreeRepo *self,
}
do
- res = fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode"));
+ res = fchmod (fd, mode);
while (G_UNLIKELY (res == -1 && errno == EINTR));
if (G_UNLIKELY (res == -1))
{
@@ -220,9 +215,7 @@ commit_loose_object_trusted (OstreeRepo *self,
if (objtype == OSTREE_OBJECT_TYPE_FILE && self->mode == OSTREE_REPO_MODE_BARE_USER)
{
- g_assert (file_info != NULL);
-
- if (!write_file_metadata_to_xattr (fd, file_info, xattrs, error))
+ if (!write_file_metadata_to_xattr (fd, uid, gid, mode, xattrs, error))
goto out;
if (!object_is_symlink)
@@ -232,7 +225,7 @@ commit_loose_object_trusted (OstreeRepo *self,
checkout. To make this work we apply all user bits and the read bits for
group/other */
do
- res = fchmod (fd, g_file_info_get_attribute_uint32 (file_info, "unix::mode") | 0744);
+ res = fchmod (fd, mode | 0744);
while (G_UNLIKELY (res == -1 && errno == EINTR));
if (G_UNLIKELY (res == -1))
{
@@ -274,9 +267,6 @@ commit_loose_object_trusted (OstreeRepo *self,
goto out;
}
}
-
- if (!g_output_stream_close (temp_out, cancellable, error))
- goto out;
}
if (!_ostree_repo_ensure_loose_objdir_at (self->objects_dir_fd, loose_path,
@@ -467,6 +457,85 @@ fallocate_stream (GFileDescriptorBased *stream,
return ret;
}
+gboolean
+_ostree_repo_open_trusted_content_bare (OstreeRepo *self,
+ const char *checksum,
+ guint64 content_len,
+ OstreeRepoTrustedContentBareCommit *out_state,
+ GOutputStream **out_stream,
+ gboolean *out_have_object,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ gs_free char *temp_filename = NULL;
+ gs_unref_object GOutputStream *ret_stream = NULL;
+ gboolean have_obj;
+ char loose_objpath[_OSTREE_LOOSE_PATH_MAX];
+
+ if (!_ostree_repo_has_loose_object (self, checksum, OSTREE_OBJECT_TYPE_FILE,
+ &have_obj, loose_objpath,
+ NULL,
+ cancellable, error))
+ goto out;
+
+ if (!have_obj)
+ {
+ if (!gs_file_open_in_tmpdir_at (self->tmp_dir_fd, 0644, &temp_filename, &ret_stream,
+ cancellable, error))
+ goto out;
+
+ if (!fallocate_stream ((GFileDescriptorBased*)ret_stream, content_len,
+ cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ if (!have_obj)
+ {
+ out_state->temp_filename = temp_filename;
+ temp_filename = NULL;
+ out_state->fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)ret_stream);
+ gs_transfer_out_value (out_stream, &ret_stream);
+ }
+ *out_have_object = have_obj;
+ out:
+ return ret;
+}
+
+gboolean
+_ostree_repo_commit_trusted_content_bare (OstreeRepo *self,
+ const char *checksum,
+ OstreeRepoTrustedContentBareCommit *state,
+ guint32 uid,
+ guint32 gid,
+ guint32 mode,
+ GVariant *xattrs,
+ GCancellable *cancellable,
+ GError **error)
+{
+ gboolean ret = FALSE;
+ char loose_objpath[_OSTREE_LOOSE_PATH_MAX];
+
+ if (state->fd != -1)
+ {
+ _ostree_loose_path (loose_objpath, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode);
+
+ if (!commit_loose_object_trusted (self, checksum, OSTREE_OBJECT_TYPE_FILE,
+ loose_objpath,
+ state->temp_filename,
+ FALSE, uid, gid, mode,
+ xattrs, state->fd,
+ cancellable, error))
+ goto out;
+ }
+
+ ret = TRUE;
+ out:
+ g_free (state->temp_filename);
+ return ret;
+}
+
static gboolean
write_object (OstreeRepo *self,
OstreeObjectType objtype,
@@ -482,7 +551,6 @@ write_object (OstreeRepo *self,
gboolean do_commit;
OstreeRepoMode repo_mode;
gs_free char *temp_filename = NULL;
- gs_unref_object GFile *temp_file = NULL;
gs_unref_object GFile *stored_path = NULL;
gs_free guchar *ret_csum = NULL;
gs_unref_object OstreeChecksumInputStream *checksum_input = NULL;
@@ -583,7 +651,6 @@ write_object (OstreeRepo *self,
cancellable, error))
goto out;
- temp_file = g_file_get_child (self->tmp_dir, temp_filename);
if (g_output_stream_splice (temp_out, file_input, 0,
cancellable, error) < 0)
goto out;
@@ -595,7 +662,6 @@ write_object (OstreeRepo *self,
&temp_filename,
cancellable, error))
goto out;
- temp_file = g_file_get_child (self->tmp_dir, temp_filename);
}
else if (repo_mode == OSTREE_REPO_MODE_ARCHIVE_Z2)
{
@@ -610,7 +676,6 @@ write_object (OstreeRepo *self,
&temp_filename, &temp_out,
cancellable, error))
goto out;
- temp_file = g_file_get_child (self->tmp_dir, temp_filename);
temp_file_is_regular = TRUE;
file_meta = _ostree_zlib_file_header_new (file_info, xattrs);
@@ -645,7 +710,6 @@ write_object (OstreeRepo *self,
cancellable, error))
goto out;
- temp_file = g_file_get_child (self->tmp_dir, temp_filename);
if (g_output_stream_splice (temp_out, checksum_input ? (GInputStream*)checksum_input : input,
0,
cancellable, error) < 0)
@@ -676,16 +740,17 @@ write_object (OstreeRepo *self,
g_assert (actual_checksum != NULL); /* Pacify static analysis */
- if (indexable)
+ if (indexable && temp_file_is_regular)
{
- gsize archived_size;
- gs_unref_object GFileInfo *compressed_info =
- g_file_query_info (temp_file, G_FILE_ATTRIBUTE_STANDARD_SIZE, 0,
- cancellable, error);
- if (!compressed_info)
- goto out;
- archived_size = g_file_info_get_size (compressed_info);
- repo_store_size_entry (self, actual_checksum, unpacked_size, archived_size);
+ struct stat stbuf;
+
+ if (fstatat (self->tmp_dir_fd, temp_filename, &stbuf, AT_SYMLINK_NOFOLLOW) == -1)
+ {
+ gs_set_error_from_errno (error, errno);
+ goto out;
+ }
+
+ repo_store_size_entry (self, actual_checksum, unpacked_size, stbuf.st_size);
}
if (!_ostree_repo_has_loose_object (self, actual_checksum, objtype,
@@ -697,11 +762,27 @@ write_object (OstreeRepo *self,
if (do_commit)
{
- if (!commit_loose_object_trusted (self, actual_checksum,
- objtype, loose_objpath,
- temp_file, temp_filename,
- object_is_symlink, file_info,
- xattrs, temp_out,
+ guint32 uid, gid, mode;
+ int fd = -1;
+
+ if (file_info)
+ {
+ uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid");
+ gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid");
+ mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+ }
+ else
+ uid = gid = mode = 0;
+
+ if (temp_out)
+ fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)temp_out);
+
+ if (!commit_loose_object_trusted (self, actual_checksum, objtype,
+ loose_objpath,
+ temp_filename,
+ object_is_symlink,
+ uid, gid, mode,
+ xattrs, fd,
cancellable, error))
goto out;
@@ -720,7 +801,6 @@ write_object (OstreeRepo *self,
}
g_clear_pointer (&temp_filename, g_free);
- g_clear_object (&temp_file);
}
g_mutex_lock (&self->txn_stats_lock);
diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h
index 82d3534..a19108f 100644
--- a/src/libostree/ostree-repo-private.h
+++ b/src/libostree/ostree-repo-private.h
@@ -194,4 +194,30 @@ _ostree_repo_gpg_verify_file_with_metadata (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
+typedef struct {
+ int fd;
+ char *temp_filename;
+} OstreeRepoTrustedContentBareCommit;
+
+gboolean
+_ostree_repo_open_trusted_content_bare (OstreeRepo *self,
+ const char *checksum,
+ guint64 content_len,
+ OstreeRepoTrustedContentBareCommit *out_state,
+ GOutputStream **out_stream,
+ gboolean *out_have_object,
+ GCancellable *cancellable,
+ GError **error);
+
+gboolean
+_ostree_repo_commit_trusted_content_bare (OstreeRepo *self,
+ const char *checksum,
+ OstreeRepoTrustedContentBareCommit *state,
+ guint32 uid,
+ guint32 gid,
+ guint32 mode,
+ GVariant *xattrs,
+ GCancellable *cancellable,
+ GError **error);
+
G_END_DECLS
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
