GNOME.org
 

[hacktree/wip/import: 2/3] wip import

commit c7969ce318f01fc4416e80238b86c2f529f4c67e
Author: Colin Walters <walters verbum org>
Date:   Thu Oct 13 17:11:01 2011 -0400

    wip import

 TODO                            |    4 +-
 src/libhacktree/hacktree-core.c |   73 +++++++++--
 src/libhacktree/hacktree-core.h |   43 ++++++-
 src/libhacktree/hacktree-repo.c |  264 ++++++++++++++++++++++++++++++++++++++-
 src/libhacktree/hacktree-repo.h |   16 +++
 src/libhtutil/ht-unix-utils.c   |   63 +++++++++
 src/libhtutil/ht-unix-utils.h   |    6 +
 7 files changed, 452 insertions(+), 17 deletions(-)
---
diff --git a/TODO b/TODO
index aeff7f7..dd4bc1d 100644
--- a/TODO
+++ b/TODO
@@ -1,5 +1,5 @@
-* hacktree import-tar
-  - Accepts tarball on stdin, stores the objects
 * tree and commit objects
 * hacktree import
+* hacktree import-tar
+  - Accepts tarball on stdin, stores the objects
 * branches
diff --git a/src/libhacktree/hacktree-core.c b/src/libhacktree/hacktree-core.c
index ddda1b5..a499985 100644
--- a/src/libhacktree/hacktree-core.c
+++ b/src/libhacktree/hacktree-core.c
@@ -27,7 +27,10 @@
 char *
 stat_to_string (struct stat *stbuf)
 {
-  return g_strdup_printf ("%d:%d:%d", stbuf->st_mode, stbuf->st_uid, stbuf->st_gid);
+  return g_strdup_printf ("%u:%u:%u",
+                          (guint32)(stbuf->st_mode & ~S_IFMT),
+                          (guint32)stbuf->st_uid, 
+                          (guint32)stbuf->st_gid);
 }
 
 static char *
@@ -56,6 +59,52 @@ canonicalize_xattrs (char *xattr_string, size_t len)
 }
 
 gboolean
+hacktree_get_xattrs_for_directory (const char *path,
+                                   char      **out_xattrs, /* out */
+                                   gsize      *out_len, /* out */
+                                   GError **error)
+{
+  gboolean ret = FALSE;
+  char *xattrs = NULL;
+  ssize_t bytes_read;
+
+  bytes_read = llistxattr (path, NULL, 0);
+
+  if (bytes_read < 0)
+    {
+      if (errno != ENOTSUP)
+        {
+          ht_util_set_error_from_errno (error, errno);
+          goto out;
+        }
+    }
+  else if (bytes_read > 0)
+    {
+      xattrs = g_malloc (bytes_read);
+      if (!llistxattr (path, xattrs, bytes_read))
+        {
+          ht_util_set_error_from_errno (error, errno);
+          g_free (xattrs);
+          xattrs = NULL;
+          goto out;
+        }
+
+      *out_xattrs = canonicalize_xattrs (xattrs, bytes_read);
+      *out_len = (gsize)bytes_read;
+    }
+  else
+    {
+      *out_xattrs = NULL;
+      *out_len = 0;
+    }
+
+  ret = TRUE;
+ out:
+  g_free (xattrs);
+  return ret;
+}
+
+gboolean
 hacktree_stat_and_checksum_file (int dir_fd, const char *path,
                                  GChecksum **out_checksum,
                                  struct stat *out_stbuf,
@@ -73,6 +122,7 @@ hacktree_stat_and_checksum_file (int dir_fd, const char *path,
   gboolean ret = FALSE;
   char *symlink_target = NULL;
   char *device_id = NULL;
+  struct stat stbuf;
 
   basename = g_path_get_basename (path);
 
@@ -89,13 +139,13 @@ hacktree_stat_and_checksum_file (int dir_fd, const char *path,
       dir_fd = dirfd (temp_dir);
     }
 
-  if (fstatat (dir_fd, basename, out_stbuf, AT_SYMLINK_NOFOLLOW) < 0)
+  if (fstatat (dir_fd, basename, &stbuf, AT_SYMLINK_NOFOLLOW) < 0)
     {
       ht_util_set_error_from_errno (error, errno);
       goto out;
     }
 
-  if (!S_ISLNK(out_stbuf->st_mode))
+  if (!S_ISLNK(&stbuf->st_mode))
     {
       fd = ht_util_open_file_read_at (dir_fd, basename, error);
       if (fd < 0)
@@ -105,10 +155,10 @@ hacktree_stat_and_checksum_file (int dir_fd, const char *path,
         }
     }
 
-  stat_string = stat_to_string (out_stbuf);
+  stat_string = stat_to_string (&stbuf);
 
   /* FIXME - Add llistxattrat */
-  if (!S_ISLNK(out_stbuf->st_mode))
+  if (!S_ISLNK(&stbuf->st_mode))
     bytes_read = flistxattr (fd, NULL, 0);
   else
     bytes_read = llistxattr (path, NULL, 0);
@@ -121,12 +171,12 @@ hacktree_stat_and_checksum_file (int dir_fd, const char *path,
           goto out;
         }
     }
-  if (errno != ENOTSUP)
+  else if (bytes_read > 0)
     {
       gboolean tmp;
       xattrs = g_malloc (bytes_read);
       /* FIXME - Add llistxattrat */
-      if (!S_ISLNK(out_stbuf->st_mode))
+      if (!S_ISLNK(&stbuf->st_mode))
         tmp = flistxattr (fd, xattrs, bytes_read);
       else
         tmp = llistxattr (path, xattrs, bytes_read);
@@ -142,7 +192,7 @@ hacktree_stat_and_checksum_file (int dir_fd, const char *path,
 
   content_sha256 = g_checksum_new (G_CHECKSUM_SHA256);
  
-  if (S_ISREG(out_stbuf->st_mode))
+  if (S_ISREG(&stbuf->st_mode))
     {
       guint8 buf[8192];
 
@@ -154,7 +204,7 @@ hacktree_stat_and_checksum_file (int dir_fd, const char *path,
           goto out;
         }
     }
-  else if (S_ISLNK(out_stbuf->st_mode))
+  else if (S_ISLNK(&stbuf->st_mode))
     {
       symlink_target = g_malloc (PATH_MAX);
 
@@ -165,9 +215,9 @@ hacktree_stat_and_checksum_file (int dir_fd, const char *path,
         }
       g_checksum_update (content_sha256, symlink_target, strlen (symlink_target));
     }
-  else if (S_ISCHR(out_stbuf->st_mode) || S_ISBLK(out_stbuf->st_mode))
+  else if (S_ISCHR(&stbuf->st_mode) || S_ISBLK(&stbuf->st_mode))
     {
-      device_id = g_strdup_printf ("%d", out_stbuf->st_rdev);
+      device_id = g_strdup_printf ("%d", &stbuf->st_rdev);
       g_checksum_update (content_sha256, device_id, strlen (device_id));
     }
   else
@@ -184,6 +234,7 @@ hacktree_stat_and_checksum_file (int dir_fd, const char *path,
   g_checksum_update (content_and_meta_sha256, stat_string, strlen (stat_string));
   g_checksum_update (content_and_meta_sha256, xattrs_canonicalized, strlen (stat_string));
 
+  *out_stbuf = stbuf;
   *out_checksum = content_and_meta_sha256;
   ret = TRUE;
  out:
diff --git a/src/libhacktree/hacktree-core.h b/src/libhacktree/hacktree-core.h
index 124a90a..cc10926 100644
--- a/src/libhacktree/hacktree-core.h
+++ b/src/libhacktree/hacktree-core.h
@@ -18,7 +18,6 @@
  *
  * Author: Colin Walters <walters verbum org>
  */
-/* hacktree-repo.h */
 
 #ifndef _HACKTREE_CORE
 #define _HACKTREE_CORE
@@ -27,6 +26,48 @@
 
 G_BEGIN_DECLS
 
+#define HACKTREE_EMPTY_STRING_SHA256 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+
+typedef enum {
+  HACKTREE_SERIALIZED_TREE_VARIANT = 1,
+  HACKTREE_SERIALIZED_COMMIT_VARIANT = 2
+  HACKTREE_SERIALIZED_XATTRS_VARIANT = 3
+} HacktreeSerializedVariantType;
+
+#define HACKTREE_SERIALIZED_VARIANT_FORMAT G_VARIANT_TYPE("(uv)")
+
+/*
+ * Tree objects:
+ * u - Version
+ * a{sv} - Metadata
+ * a(ss) - array of (checksum, filename) for files
+ * as - array of tree checksums for directories
+ * a(ssuay) - array of (dirname, uid, gid, mode, xattr_checksum) for directories
+ */
+#define HACKTREE_TREE_GVARIANT_FORMAT G_VARIANT_TYPE("(ua{sv}a(ss)asa(suuus)")
+/*
+ * Commit objects:
+ * u - Version
+ * a{sv} - Metadata
+ * s - subject 
+ * s - body
+ * t - Timestamp in seconds since the epoch (UTC)
+ * s - Tree SHA256
+ */
+#define HACKTREE_COMMIT_GVARIANT_FORMAT G_VARIANT_TYPE("(ua{sv}ssts)")
+
+/*
+ * xattr objects:
+ * u - Version
+ * ay - data
+ */
+#define HACKTREE_XATTR_GVARIANT_FORMAT G_VARIANT_TYPE("(uay)")
+
+gboolean   hacktree_get_xattrs_for_directory (const char *path,
+                                              char      **out_xattrs,
+                                              gsize      *out_len,
+                                              GError    **error);
+
 gboolean hacktree_stat_and_checksum_file (int dirfd, const char *path,
                                           GChecksum **out_checksum,
                                           struct stat *out_stbuf,
diff --git a/src/libhacktree/hacktree-repo.c b/src/libhacktree/hacktree-repo.c
index 7f6fce0..11e64b4 100644
--- a/src/libhacktree/hacktree-repo.c
+++ b/src/libhacktree/hacktree-repo.c
@@ -39,6 +39,7 @@ typedef struct _HacktreeRepoPrivate HacktreeRepoPrivate;
 
 struct _HacktreeRepoPrivate {
   char *path;
+  char *index_path;
   char *objects_path;
 
   gboolean inited;
@@ -51,6 +52,7 @@ hacktree_repo_finalize (GObject *object)
   HacktreeRepoPrivate *priv = GET_PRIVATE (self);
 
   g_free (priv->path);
+  g_free (priv->index_path);
   g_free (priv->objects_path);
 
   G_OBJECT_CLASS (hacktree_repo_parent_class)->finalize (object);
@@ -113,6 +115,7 @@ hacktree_repo_constructor (GType                  gtype,
   g_assert (priv->path != NULL);
 
   priv->objects_path = g_build_filename (priv->path, HACKTREE_REPO_DIR, "objects", NULL);
+  priv->index_path = g_build_filename (priv->path, HACKTREE_REPO_DIR, "index", NULL);
 
   return object;
 }
@@ -142,7 +145,6 @@ static void
 hacktree_repo_init (HacktreeRepo *self)
 {
   HacktreeRepoPrivate *priv = GET_PRIVATE (self);
-  
 }
 
 HacktreeRepo*
@@ -167,6 +169,121 @@ hacktree_repo_check (HacktreeRepo *self, GError **error)
   return TRUE;
 }
 
+static gboolean
+import_gvariant (HacktreeRepo  *repo,
+                 HacktreeSerializedVariantType type,
+                 GVariant       *variant,
+                 GChecksum    **out_checksum,
+                 GError       **error)
+{
+  HacktreeRepoPrivate *priv = GET_PRIVATE (self);
+  GVariantBuilder builder;
+  GVariant *serialized = NULL;
+  struct stat stbuf;
+  gboolean ret = FALSE;
+  gsize i, n_items;
+  gsize size;
+  char *tmp_name = NULL;
+  int fd = -1;
+  GUnixOutputStream *stream = NULL;
+
+  g_variant_builder_init (&builder, HACKTREE_SERIALIZED_VARIANT_FORMAT);
+  g_variant_builder_add (&builder, "uv", (guint32)type, variant);
+  serialized = g_variant_builder_end (&builder);
+
+  tmp_name = g_build_filename (priv->objects_path, "variant-XXXXXX.tmp", NULL);
+  fd = mkstemp (tmp_name);
+  if (fd < 0)
+    {
+      ht_util_set_error_from_errno (error, errno);
+      goto out;
+    }
+
+  stream = g_unix_output_stream_new (fd, FALSE);
+  if (!g_output_stream_write_all ((GOutputStream*)stream,
+                                  g_variant_get_data (serialized),
+                                  g_variant_get_size (serialized),
+                                  NULL,
+                                  error))
+    goto out;
+  if (!g_output_stream_close ((GOutputStream*)stream,
+                              NULL, error))
+    goto out;
+
+  if (!link_one_file (repo, tmp_name, FALSE, FALSE, out_checksum, error))
+    goto out;
+  
+  ret = TRUE;
+ out:
+  /* Unconditionally unlink; if we suceeded, there's a new link, if not, clean up. */
+  (void) unlink (tmp_name);
+  if (fd != -1)
+    close (fd);
+  if (serialized != NULL)
+    g_variant_unref (serialized);
+  g_free (tmp_name);
+  g_clear_object (&stream);
+  return ret;
+}
+
+static GVariant *
+import_directory (HacktreeRepo  *self,
+                  const char *path,
+                  GError    **error)
+{
+  GVariant *ret = NULL;
+  struct stat stbuf;
+  char *basename = NULL;
+  GChecksum *xattr_checksum = NULL;
+  const char *xattr_checksum_string;
+  GVariant *xattr_variant = NULL;
+  char *xattrs = NULL;
+  gsize xattr_len;
+
+  if (lstat (path, &stbuf) < 0)
+    {
+      ht_util_set_error_from_errno (error, errno);
+      goto out;
+    }
+  
+  if (!S_ISDIR(stbuf->st_mode))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Not a directory: '%s'", path);
+      goto out;
+    }
+
+  if (!hacktree_get_xattrs_for_directory (path, &xattrs, &xattr_len, error))
+    goto out;
+
+  if (xattrs != NULL)
+    {
+      xattr_variant = g_variant_new_fixed_array (G_VARIANT_TYPE ("y"),
+                                                 xattrs, xattr_len, 1);
+      g_variant_ref_sink (xattr_variant);
+      if (!import_gvariant (self, HACKTREE_SERIALIZED_XATTRS_VARIANT, xattr_variant, &xattr_checksum, error))
+        goto out;
+      xattr_checksum_string = g_checksum_get_string (xattr_checksum);
+    }
+  else
+    xattr_checksum_string = HACKTREE_EMPTY_STRING_SHA256;
+ 
+  basename = g_path_get_basename (path);
+
+  ret = g_variant_new (G_VARIANT_TYPE ("(suuus)"),
+                       basename,
+                       (guint32)stbuf.st_uid,
+                       (guint32)stbuf.st_gid,
+                       (guint32)(stbuf.st_mode & ~S_IFMT),
+                       xattr_checksum_string);
+
+ out:
+  if (xattr_checksum != NULL)
+    g_checksum_free (xattr_checksum);
+  g_free (xattrs);
+  g_free (basename);
+  return ret;
+}
 
 static char *
 prepare_dir_for_checksum_get_object_path (HacktreeRepo *self,
@@ -195,7 +312,9 @@ prepare_dir_for_checksum_get_object_path (HacktreeRepo *self,
 
 static gboolean
 link_one_file (HacktreeRepo *self, const char *path,
-               gboolean ignore_exists, gboolean force, GError **error)
+               gboolean ignore_exists, gboolean force,
+               GChecksum **out_checksum,
+               GError **error)
 {
   HacktreeRepoPrivate *priv = GET_PRIVATE (self);
   char *src_basename = NULL;
@@ -265,6 +384,8 @@ link_one_file (HacktreeRepo *self, const char *path,
       (void) unlinkat (dirfd (dest_dir), tmp_dest_basename, 0);
     }
 
+  *out_checksum = id;
+  id = NULL;
   ret = TRUE;
  out:
   if (id != NULL)
@@ -289,10 +410,147 @@ hacktree_repo_link_file (HacktreeRepo *self,
                          GError      **error)
 {
   HacktreeRepoPrivate *priv = GET_PRIVATE (self);
+  GChecksum *checksum = NULL;
+
+  g_return_val_if_fail (priv->inited, FALSE);
+
+  if (!link_one_file (self, path, ignore_exists, force, &checksum, error))
+    return FALSE;
+  g_checksum_free (checksum);
+  return TRUE;
+}
+
+typedef struct {
+  GChecksum *checksum;
+  struct stat stbuf;
+} CommitData;
+
+static void
+commit_data_free (CommitData *data)
+{
+  if (!data)
+    return;
+  g_checksum_free (data->checksum);
+}
+
+gboolean
+hacktree_repo_commit_files (HacktreeRepo *repo,
+                            const char   *subject,
+                            const char   *body,
+                            GVariant     *metadata,
+                            const char   *base,
+                            GPtrArray    *files,
+                            GError      **error)
+{
+  char *abspath = NULL;
+  gboolean ret = FALSE;
+  int i;
+  int current_tree_depth = -1;
+  GPtrArray *sorted_files = NULL;
+  GHashTable *filename_to_commit_data = NULL;
+
+  filename_to_commit_data = g_hash_table_new_full (g_str_hash, g_str_equal,
+                                                   NULL, (GDestroyNotify)commit_data_free);
+  sorted_files = ht_util_sort_filenames_by_component_length (files);
+
+  for (i = 0; i < sorted_files->len; i++)
+    {
+      const char *filename = files->pdata[i];
+      CommitData *data = NULL;
+      int n_components;
+
+      if (ht_util_filename_has_dotdot (filename))
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Path uplink '..' in filename '%s' not allowed (yet)", filename);
+          goto out;
+        }
+      
+      if (g_path_is_absolute (filename))
+        {
+          g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                       "Abolute filename '%s' not allowed (yet)", filename);
+          goto out;
+        }
+
+      n_components = ht_util_count_filename_components (filename);
+      if (current_tree_depth == -1)
+        current_tree_depth = n_components;
+      else if (n_components < current_tree_depth)
+        {
+          
+        }
+      
+      g_free (abspath);
+      abspath = g_build_filename (base, filename, NULL);
+      commitdata = g_new0 (CommitData, 1);
+      
+      /* Takes ownership of commitdata */
+      g_hash_table_insert (filename_to_checksum, filename, commitdata);
+
+      if (lstat (abspath, &commitdata->stbuf) < 0)
+        {
+          ht_util_set_error_from_errno (error, errno);
+          goto out;
+        }
+      
+      if (S_ISDIR (stbuf.st_mode))
+        {
+          GVariant *dirdata;
+
+          dirdata = create_directory_data (abspath, error);
+          if (!dirdata)
+            goto out;
+          g_variant_ref_sink (dirdata);
+          
+          if (!import_gvariant (self, dirdata, error))
+            {
+              g_variant_unref (dirdata);
+              goto out;
+            }
+          g_variant_unref (dirdata);
+        }
+      else
+        {
+          if (!link_one_file (self, abspath, TRUE, FALSE, error))
+            goto out;
+        }
+    }
+
+ out:
+  if (sorted_files != NULL)
+    g_ptr_array_free (sorted_files);
+  if (filename_to_checksum != NULL)
+    g_hash_table_destroy (filename_to_checksum);
+  g_free (abspath);
+  return ret;
+}
+
+
+gboolean
+hacktree_repo_import_tree (HacktreeRepo *self,
+                           GVariant     *variant,
+                           GError      **error)
+{
+  HacktreeRepoPrivate *priv = GET_PRIVATE (self);
+
+  g_return_val_if_fail (priv->inited, FALSE);
+  g_return_val_if_fail (g_variant_is_of_type (variant, HACKTREE_TREE_GVARIANT_FORMAT), FALSE);
+
+  return import_gvariant (self, tree_variant, error);
+}
+
+gboolean      
+hacktree_repo_import_commit (HacktreeRepo *self,
+                             GVariant     *variant,
+                             GError      **error)
+{
+  HacktreeRepoPrivate *priv = GET_PRIVATE (self);
 
   g_return_val_if_fail (priv->inited, FALSE);
+  g_return_val_if_fail (g_variant_is_of_type (variant, HACKTREE_COMMIT_GVARIANT_FORMAT), FALSE);
 
-  return link_one_file (self, path, ignore_exists, force, error);
+  return import_gvariant (self, variant, error);
 }
 
 static gboolean
diff --git a/src/libhacktree/hacktree-repo.h b/src/libhacktree/hacktree-repo.h
index 633771c..0c97422 100644
--- a/src/libhacktree/hacktree-repo.h
+++ b/src/libhacktree/hacktree-repo.h
@@ -59,6 +59,22 @@ gboolean      hacktree_repo_link_file (HacktreeRepo *repo,
                                        gboolean      force,
                                        GError      **error);
 
+gboolean      hacktree_repo_commit_files (HacktreeRepo *repo,
+                                          const char   *subject,
+                                          const char   *body,
+                                          GVariant     *metadata,
+                                          const char   *base,
+                                          GPtrArray    *files,
+                                          GError      **error);
+
+gboolean      hacktree_repo_import_tree (HacktreeRepo *repo,
+                                         GVariant     *tree_variant,
+                                         GError      **error);
+
+gboolean      hacktree_repo_import_commit (HacktreeRepo *repo,
+                                           GVariant     *commit_variant,
+                                           GError      **error);
+
 typedef void (*HacktreeRepoObjectIter) (HacktreeRepo *repo, const char *path,
                                         GFileInfo *fileinfo, gpointer user_data);
 
diff --git a/src/libhtutil/ht-unix-utils.c b/src/libhtutil/ht-unix-utils.c
index 4637c7d..10c38df 100644
--- a/src/libhtutil/ht-unix-utils.c
+++ b/src/libhtutil/ht-unix-utils.c
@@ -30,6 +30,69 @@
 #include <sys/types.h>
 #include <dirent.h>
 
+static int
+compare_filenames_by_component_length (const char *a,
+                                       const char *b)
+{
+  char *a_slash, *b_slash;
+
+  a_slash = strchr (a, '/');
+  b_slash = strchr (b, '/');
+  while (a_slash && b_slash)
+    {
+      a = a_slash + 1;
+      b = b_slash + 1;
+      a_slash = strchr (a, '/');
+      b_slash = strchr (b, '/');
+    }
+  if (a_slash)
+    return -1;
+  else if (b_slash)
+    return 1;
+  else
+    return 0;
+}
+
+GPtrArray *
+ht_util_sort_filenames_by_component_length (GPtrArray *files)
+{
+  GPtrArray *array = g_ptr_array_sized_new (files->len);
+  memcpy (array->pdata, files->pdata, sizeof (gpointer) * files->len);
+  g_ptr_array_sort (array, (GCompareFunc) compare_filenames_by_component_length);
+  return array;
+}
+
+int
+ht_util_count_filename_components (const char *path)
+{
+  int i = 0;
+  char *p;
+
+  while (path)
+    {
+      i++;
+      path = strchr (path, '/');
+      if (path)
+        path++;
+    }
+  return i;
+}
+
+gboolean
+ht_util_filename_has_dotdot (const char *path)
+{
+  char *p;
+  char last;
+
+  if (strcmp (path, "..") == 0)
+    return TRUE;
+  if (g_str_has_prefix (path, "../"))
+    return TRUE;
+  p = strstr (path, "/..");
+  last = *(p + 1);
+  return last == '\0' || last == '/';
+}
+
 void
 ht_util_set_error_from_errno (GError **error,
                               gint     saved_errno)
diff --git a/src/libhtutil/ht-unix-utils.h b/src/libhtutil/ht-unix-utils.h
index ae3731b..1c69b09 100644
--- a/src/libhtutil/ht-unix-utils.h
+++ b/src/libhtutil/ht-unix-utils.h
@@ -34,6 +34,12 @@
 
 G_BEGIN_DECLS
 
+gboolean ht_util_filename_has_dotdot (const char *path);
+
+GPtrArray *ht_util_sort_filenames_by_component_length (GPtrArray *files);
+
+int ht_util_count_filename_components (const char *path);
+
 int ht_util_open_file_read (const char *path, GError **error);
 
 int ht_util_open_file_read_at (int dirfd, const char *name, GError **error);
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]