[ostree] repo: Add ostree_repo_import_archive_to_mtree

[Colin Walters <walters src gnome org>]

commit 3a555114bc22f599d72d95d71c2579fd2345cf6f
Author: Colin Walters <walters verbum org>
Date:   Thu Feb 18 16:49:30 2016 -0500

    repo: Add ostree_repo_import_archive_to_mtree
    
    This is a more flexible version of the previous
    ostree_repo_write_archive_to_mtree() which took a file reference.
    This has an extensible options structure, and in particular
    now supports `ignore_unsupported_content`.
    
    I plan to use this for importing Docker images which contain device
    nodes.  (There's no reason for container images to have those, so
    we'll just ignore them).
    
    Also here, just like the export variant, the caller is responsible for
    setting up libarchive.

 Makefile-tests.am                      |    8 ++
 src/libostree/ostree-repo-libarchive.c |  141 ++++++++++++++++------
 src/libostree/ostree-repo.h            |   23 ++++
 tests/test-libarchive-import.c         |  200 ++++++++++++++++++++++++++++++++
 4 files changed, 333 insertions(+), 39 deletions(-)
---
diff --git a/Makefile-tests.am b/Makefile-tests.am
index 292699e..475519d 100644
--- a/Makefile-tests.am
+++ b/Makefile-tests.am
@@ -140,6 +140,10 @@ TESTS = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test-
        tests/test-keyfile-utils tests/test-ot-opt-utils tests/test-ot-tool-util \
        tests/test-gpg-verify-result tests/test-checksum tests/test-lzma tests/test-rollsum
 
+if USE_LIBARCHIVE
+TESTS += tests/test-libarchive-import
+endif
+
 check_PROGRAMS =  $(TESTS)
 TESTS_ENVIRONMENT = \
        G_TEST_SRCDIR=$(abs_srcdir)/tests \
@@ -172,6 +176,10 @@ tests_test_checksum_SOURCES = src/libostree/ostree-core.c tests/test-checksum.c
 tests_test_checksum_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags)
 tests_test_checksum_LDADD = $(TESTS_LDADD)
 
+tests_test_libarchive_import_SOURCES = tests/test-libarchive-import.c
+tests_test_libarchive_import_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags)
+tests_test_libarchive_import_LDADD = $(TESTS_LDADD)
+
 tests_test_keyfile_utils_CFLAGS = $(TESTS_CFLAGS)
 tests_test_keyfile_utils_LDADD = $(TESTS_LDADD)
 
diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c
index 19fae75..64410cb 100644
--- a/src/libostree/ostree-repo-libarchive.c
+++ b/src/libostree/ostree-repo-libarchive.c
@@ -78,6 +78,7 @@ file_info_from_archive_entry_and_modifier (OstreeRepo *repo,
 
 static gboolean
 import_libarchive_entry_file (OstreeRepo           *self,
+                              OstreeRepoImportArchiveOptions  *opts,
                               struct archive       *a,
                               struct archive_entry *entry,
                               GFileInfo            *file_info,
@@ -93,8 +94,27 @@ import_libarchive_entry_file (OstreeRepo           *self,
   if (g_cancellable_set_error_if_cancelled (cancellable, error))
     return FALSE;
 
-  if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
-    archive_stream = _ostree_libarchive_input_stream_new (a);
+  switch (g_file_info_get_file_type (file_info))
+    {
+    case G_FILE_TYPE_REGULAR:
+      archive_stream = _ostree_libarchive_input_stream_new (a);
+      break;
+    case G_FILE_TYPE_SYMBOLIC_LINK:
+      break;
+    default:
+      if (opts->ignore_unsupported_content)
+        {
+          ret = TRUE;
+          goto out;
+        }
+      else
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Unable to import non-regular/non-symlink file '%s'",
+                       archive_entry_pathname (entry));
+          goto out;
+        }
+    }
   
   if (!ostree_raw_file_to_content_stream (archive_stream, file_info, NULL,
                                           &file_object_input, &length, cancellable, error))
@@ -111,6 +131,7 @@ import_libarchive_entry_file (OstreeRepo           *self,
 
 static gboolean
 write_libarchive_entry_to_mtree (OstreeRepo           *self,
+                                 OstreeRepoImportArchiveOptions  *opts,
                                  OstreeMutableTree    *root,
                                  struct archive       *a,
                                  struct archive_entry *entry,
@@ -249,24 +270,92 @@ write_libarchive_entry_to_mtree (OstreeRepo           *self,
               goto out;
             }
 
-          if (!import_libarchive_entry_file (self, a, entry, file_info, &tmp_csum,
+          if (!import_libarchive_entry_file (self, opts, a, entry, file_info, &tmp_csum,
                                              cancellable, error))
             goto out;
+
+          if (tmp_csum)
+            { 
+              g_free (tmp_checksum);
+              tmp_checksum = ostree_checksum_from_bytes (tmp_csum);
+              if (!ostree_mutable_tree_replace_file (parent, basename,
+                                                     tmp_checksum,
+                                                     error))
+                goto out;
+            }
+        }
+    }
+
+  ret = TRUE;
+ out:
+  return ret;
+}
+#endif
+
+/**
+ * ostree_repo_import_archive_to_mtree:
+ * @self: An #OstreeRepo
+ * @opts: Options structure, ensure this is zeroed, then set specific variables
+ * @archive: Really this is "struct archive*"
+ * @mtree: The #OstreeMutableTree to write to
+ * @modifier: (allow-none): Optional commit modifier
+ * @cancellable: Cancellable
+ * @error: Error
+ *
+ * Import an archive file @archive into the repository, and write its
+ * file structure to @mtree.
+ */
+gboolean
+ostree_repo_import_archive_to_mtree (OstreeRepo                   *self,
+                                     OstreeRepoImportArchiveOptions  *opts,
+                                     void                         *archive,
+                                     OstreeMutableTree            *mtree,
+                                     OstreeRepoCommitModifier     *modifier,
+                                     GCancellable                 *cancellable,
+                                     GError                      **error)
+{
+  gboolean ret = FALSE;
+  struct archive *a = archive;
+  struct archive_entry *entry;
+  g_autofree guchar *tmp_csum = NULL;
+  int r;
+
+  while (TRUE)
+    {
+      r = archive_read_next_header (a, &entry);
+      if (r == ARCHIVE_EOF)
+        break;
+      else if (r != ARCHIVE_OK)
+        {
+          propagate_libarchive_error (error, a);
+          goto out;
+        }
+
+      /* TODO - refactor this to only create the metadata on demand
+       * (i.e. if there is a missing parent dir)
+       */
+      if (opts->autocreate_parents && !tmp_csum)
+        {
+          g_autoptr(GFileInfo) tmp_dir_info = g_file_info_new ();
           
-          g_free (tmp_checksum);
-          tmp_checksum = ostree_checksum_from_bytes (tmp_csum);
-          if (!ostree_mutable_tree_replace_file (parent, basename,
-                                                 tmp_checksum,
-                                                 error))
+          g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", archive_entry_uid (entry));
+          g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", archive_entry_gid (entry));
+          g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR);
+          
+          if (!_ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, &tmp_csum, cancellable, error))
             goto out;
         }
+
+      if (!write_libarchive_entry_to_mtree (self, opts, mtree, a,
+                                            entry, modifier, tmp_csum,
+                                            cancellable, error))
+        goto out;
     }
 
   ret = TRUE;
  out:
   return ret;
 }
-#endif
                           
 /**
  * ostree_repo_write_archive_to_mtree:
@@ -293,10 +382,8 @@ ostree_repo_write_archive_to_mtree (OstreeRepo                *self,
 #ifdef HAVE_LIBARCHIVE
   gboolean ret = FALSE;
   struct archive *a = NULL;
-  struct archive_entry *entry;
-  int r;
   g_autoptr(GFileInfo) tmp_dir_info = NULL;
-  g_autofree guchar *tmp_csum = NULL;
+  OstreeRepoImportArchiveOptions opts = { 0, };
 
   a = archive_read_new ();
 #ifdef HAVE_ARCHIVE_READ_SUPPORT_FILTER_ALL
@@ -311,35 +398,11 @@ ostree_repo_write_archive_to_mtree (OstreeRepo                *self,
       goto out;
     }
 
-  while (TRUE)
-    {
-      r = archive_read_next_header (a, &entry);
-      if (r == ARCHIVE_EOF)
-        break;
-      else if (r != ARCHIVE_OK)
-        {
-          propagate_libarchive_error (error, a);
-          goto out;
-        }
+  opts.autocreate_parents = !!autocreate_parents;
 
-      if (autocreate_parents && !tmp_csum)
-        {
-          tmp_dir_info = g_file_info_new ();
-          
-          g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", archive_entry_uid (entry));
-          g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", archive_entry_gid (entry));
-          g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR);
-          
-          if (!_ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, &tmp_csum, cancellable, error))
-            goto out;
-        }
+  if (!ostree_repo_import_archive_to_mtree (self, &opts, a, mtree, modifier, cancellable, error))
+    goto out;
 
-      if (!write_libarchive_entry_to_mtree (self, mtree, a,
-                                            entry, modifier,
-                                            autocreate_parents ? tmp_csum : NULL,
-                                            cancellable, error))
-        goto out;
-    }
   if (archive_read_close (a) != ARCHIVE_OK)
     {
       propagate_libarchive_error (error, a);
diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h
index 3b1040a..4629ea5 100644
--- a/src/libostree/ostree-repo.h
+++ b/src/libostree/ostree-repo.h
@@ -451,6 +451,29 @@ gboolean      ostree_repo_write_archive_to_mtree (OstreeRepo                   *
                                                   GError                      **error);
 
 /**
+ * OstreeRepoImportArchiveOptions:
+ *
+ * An extensible options structure controlling archive import.  Ensure that
+ * you have entirely zeroed the structure, then set just the desired
+ * options.  This is used by ostree_repo_import_archive_to_mtree().
+ */
+typedef struct {
+  guint ignore_unsupported_content : 1;
+  guint autocreate_parents : 1;
+  guint reserved : 30;
+
+  guint unused_uint[8];
+  gpointer unused_ptrs[8];
+} OstreeRepoImportArchiveOptions;
+
+gboolean      ostree_repo_import_archive_to_mtree (OstreeRepo                   *self,
+                                                   OstreeRepoImportArchiveOptions  *opts,
+                                                   void                         *archive, /* Really struct 
archive * */
+                                                   OstreeMutableTree            *mtree,
+                                                   OstreeRepoCommitModifier     *modifier,
+                                                   GCancellable                 *cancellable,
+                                                   GError                      **error);
+/**
  * OstreeRepoExportArchiveOptions:
  *
  * An extensible options structure controlling archive creation.  Ensure that
diff --git a/tests/test-libarchive-import.c b/tests/test-libarchive-import.c
new file mode 100644
index 0000000..1dd0c68
--- /dev/null
+++ b/tests/test-libarchive-import.c
@@ -0,0 +1,200 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2015 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+#include "libglnx.h"
+#include <glib.h>
+#include <stdlib.h>
+#include <gio/gio.h>
+#include <string.h>
+
+#include <ostree.h>
+#include <archive.h>
+#include <archive_entry.h>
+
+typedef struct {
+  OstreeRepo *repo;
+  int fd;
+  char *tmpd;
+} TestData;
+
+static void
+test_data_init (TestData *td)
+{
+  GError *error = NULL;
+  struct archive *a = archive_write_new ();
+  struct archive_entry *ae;
+
+  td->tmpd = g_mkdtemp (g_strdup ("/var/tmp/test-libarchive-import-XXXXXX"));
+  g_assert_cmpint (0, ==, chdir (td->tmpd));
+
+  td->fd = openat (AT_FDCWD, "foo.tar.gz", O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0644);
+  (void) unlink ("foo.tar.gz");
+
+  g_assert_no_error (error);
+  g_assert (td->fd >= 0);
+  
+  g_assert_cmpint (0, ==, archive_write_set_format_pax (a));
+  g_assert_cmpint (0, ==, archive_write_add_filter_gzip (a));
+  g_assert_cmpint (0, ==, archive_write_open_fd (a, td->fd));
+
+  ae = archive_entry_new ();
+  archive_entry_set_pathname (ae, "/");
+  archive_entry_set_mode (ae, S_IFDIR | 0755);
+  g_assert_cmpint (0, ==, archive_write_header (a, ae));
+  archive_entry_free (ae);
+
+  ae = archive_entry_new ();
+  archive_entry_set_pathname (ae, "/file");
+  archive_entry_set_mode (ae, S_IFREG | 0777);
+  archive_entry_set_size (ae, 4);
+  g_assert_cmpint (0, ==, archive_write_header (a, ae));
+  g_assert_cmpint (4, ==, archive_write_data (a, "foo\n", 4));
+  archive_entry_free (ae);
+
+  ae = archive_entry_new ();
+  archive_entry_set_pathname (ae, "/devnull");
+  archive_entry_set_mode (ae, S_IFCHR | 0777);
+  archive_entry_set_devmajor (ae, 1);
+  archive_entry_set_devminor (ae, 3);
+  g_assert_cmpint (0, ==, archive_write_header (a, ae));
+  archive_entry_free (ae);
+
+  ae = archive_entry_new ();
+  archive_entry_set_pathname (ae, "/anotherfile");
+  archive_entry_set_mode (ae, S_IFREG | 0777);
+  archive_entry_set_size (ae, 4);
+  g_assert_cmpint (0, ==, archive_write_header (a, ae));
+  g_assert_cmpint (4, ==, archive_write_data (a, "bar\n", 4));
+  archive_entry_free (ae);
+
+  g_assert_cmpint (ARCHIVE_OK, ==, archive_write_close (a));
+  g_assert_cmpint (ARCHIVE_OK, ==, archive_write_free (a));
+
+  { g_autoptr(GFile) repopath = g_file_new_for_path ("repo");
+    td->repo = ostree_repo_new (repopath);
+
+    g_assert_cmpint (0, ==, mkdir ("repo", 0755));
+
+    ostree_repo_create (td->repo, OSTREE_REPO_MODE_BARE_USER, NULL, &error);
+    g_assert_no_error (error);
+  }
+}
+
+static gboolean
+spawn_cmdline (const char *cmd, GError **error)
+{
+  int estatus;
+  if (!g_spawn_command_line_sync (cmd, NULL, NULL, &estatus, error))
+    return FALSE;
+  if (!g_spawn_check_exit_status (estatus, error))
+    return FALSE;
+  return TRUE;
+}
+
+static void
+test_libarchive_error_device_file (gconstpointer data)
+{
+  TestData *td = (void*)data;
+  GError *error = NULL;
+  struct archive *a = archive_read_new ();
+  OstreeRepoImportArchiveOptions opts = { 0, };
+  glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new ();
+
+  g_assert_cmpint (0, ==, lseek (td->fd, 0, SEEK_SET));
+  g_assert_cmpint (0, ==, archive_read_support_format_all (a));
+  g_assert_cmpint (0, ==, archive_read_support_filter_all (a));
+  g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd, 8192));
+
+  (void)ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error);
+  g_assert (error != NULL);
+}
+
+static void
+test_libarchive_ignore_device_file (gconstpointer data)
+{
+  TestData *td = (void*)data;
+  GError *error = NULL;
+  GCancellable *cancellable = NULL;
+  struct archive *a = archive_read_new ();
+  OstreeRepoImportArchiveOptions opts = { 0, };
+  glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new ();
+  glnx_unref_object GFile *root = NULL;
+  g_autofree char *commit_checksum = NULL;
+
+  g_assert_cmpint (0, ==, lseek (td->fd, 0, SEEK_SET));
+  g_assert_cmpint (0, ==, archive_read_support_format_all (a));
+  g_assert_cmpint (0, ==, archive_read_support_filter_all (a));
+  g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd, 8192));
+
+  opts.ignore_unsupported_content = TRUE;
+
+  if (!ostree_repo_prepare_transaction (td->repo, NULL, cancellable, &error))
+    goto out;
+
+  if (!ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error))
+    goto out;
+
+  if (!ostree_repo_write_mtree (td->repo, mtree, &root, cancellable, &error))
+    goto out;
+
+  if (!ostree_repo_write_commit (td->repo, NULL, "", "", NULL,
+                                 OSTREE_REPO_FILE (root),
+                                 &commit_checksum, cancellable, &error))
+    goto out;
+
+  ostree_repo_transaction_set_ref (td->repo, NULL, "foo", commit_checksum);
+  
+  if (!ostree_repo_commit_transaction (td->repo, NULL, cancellable, &error))
+    goto out;
+
+  if (!spawn_cmdline ("ostree --repo=repo ls foo file", &error))
+    goto out;
+
+  if (!spawn_cmdline ("ostree --repo=repo ls foo anotherfile", &error))
+    goto out;
+
+  if (spawn_cmdline ("ostree --repo=repo ls foo devnull", &error))
+    g_assert_not_reached ();
+  g_assert (error != NULL);
+  g_clear_error (&error);
+
+ out:
+  g_assert_no_error (error);
+}
+
+int main (int argc, char **argv)
+{
+  TestData td = {NULL,};
+  int r;
+
+  test_data_init (&td);
+
+  g_test_init (&argc, &argv, NULL);
+
+  g_test_add_data_func ("/libarchive/error-device-file", &td, test_libarchive_error_device_file);
+  g_test_add_data_func ("/libarchive/ignore-device-file", &td, test_libarchive_ignore_device_file);
+
+  r = g_test_run();
+
+  if (td.tmpd)
+    (void) glnx_shutil_rm_rf_at (AT_FDCWD, td.tmpd, NULL, NULL);
+  return r;
+}