[glib/gvdb] GHashFile

[Ryan Lortie <ryanl src gnome org>]

commit 2d0ee89c69c918f2d0153d68cb16d2ee079d6088
Author: Ryan Lortie <desrt desrt ca>
Date:   Mon Apr 12 17:12:44 2010 -0400

    GHashFile

 glib/Makefile.am |    2 +
 glib/ghashfile.c |  373 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 glib/ghashfile.h |   42 ++++++
 glib/glib.h      |    1 +
 glib/gvdb.h      |   61 +++++++++
 5 files changed, 479 insertions(+), 0 deletions(-)
---
diff --git a/glib/Makefile.am b/glib/Makefile.am
index 8882ae2..03de96b 100644
--- a/glib/Makefile.am
+++ b/glib/Makefile.am
@@ -125,6 +125,7 @@ libglib_2_0_la_SOURCES = 	\
 	gerror.c		\
 	gfileutils.c		\
 	ghash.c			\
+	ghashfile.c		\
 	ghook.c			\
 	ghostutils.c		\
 	giochannel.c    	\
@@ -222,6 +223,7 @@ glibsubinclude_HEADERS =   \
 	gerror.h	\
 	gfileutils.h	\
 	ghash.h		\
+	ghashfile.h	\
 	ghook.h		\
 	ghostutils.h	\
 	gi18n.h		\
diff --git a/glib/ghashfile.c b/glib/ghashfile.c
new file mode 100644
index 0000000..6922844
--- /dev/null
+++ b/glib/ghashfile.c
@@ -0,0 +1,373 @@
+/*
+ * Copyright © 2010 Codethink Limited
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the licence, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include "ghashfile.h"
+
+#include <string.h>
+#include "gvdb.h"
+
+struct _GHashFile {
+  gint ref_count;
+
+  const gchar *data;
+  gsize size;
+
+  GMappedFile *mapped;
+  gboolean trusted;
+
+  const guint32 *bloom_words;
+  guint32 n_bloom_words;
+  guint bloom_shift;
+
+  const guint32 *hash_buckets;
+  guint32 n_buckets;
+
+  struct gvdb_hash_item *hash_items;
+  guint32 n_hash_items;
+};
+
+static const gchar *
+g_hash_file_item_get_key (GHashFile                   *file,
+                          const struct gvdb_hash_item *item,
+                          gsize                       *size)
+{
+  guint32 start, end;
+
+  start = guint32_from_le (item->key_start);
+  *size = guint16_from_le (item->key_size);
+  end = start + *size;
+
+  if G_UNLIKELY (start > end || end > file->size)
+    return NULL;
+
+  return file->data + start;
+}
+
+static gconstpointer
+g_hash_file_dereference (GHashFile                 *file,
+                         const struct gvdb_pointer *pointer,
+                         gint                       alignment,
+                         gsize                     *size)
+{
+  guint32 start, end;
+
+  start = guint32_from_le (pointer->start);
+  end = guint32_from_le (pointer->end);
+
+  if G_UNLIKELY (start > end || end > file->size || start & (alignment - 1))
+    return NULL;
+
+  *size = end - start;
+
+  return file->data + start;
+}
+
+static void
+g_hash_file_setup_root (GHashFile                 *file,
+                        const struct gvdb_pointer *pointer)
+{
+  const struct gvdb_hash_header *header;
+  guint32 n_bloom_words;
+  guint32 bloom_shift;
+  guint32 n_buckets;
+  gsize size;
+
+  header = g_hash_file_dereference (file, pointer, 4, &size);
+
+  if G_UNLIKELY (header == NULL || size < sizeof *header)
+    return;
+
+  size -= sizeof *header;
+
+  n_bloom_words = guint32_from_le (header->n_bloom_words);
+  n_buckets = guint32_from_le (header->n_buckets);
+  bloom_shift = n_bloom_words >> 27;
+  n_bloom_words &= (1u << 27) - 1;
+
+  if G_UNLIKELY (n_bloom_words * sizeof (guint32_le) > size)
+    return;
+
+  file->bloom_words = (gpointer) (header + 1);
+  size -= n_bloom_words * sizeof (guint32_le);
+  file->n_bloom_words = n_bloom_words;
+
+  if G_UNLIKELY (n_buckets > G_MAXUINT / sizeof (guint32_le) ||
+                 n_buckets * sizeof (guint32_le) > size)
+    return;
+
+  file->hash_buckets = file->bloom_words + file->n_bloom_words;
+  size -= n_buckets * sizeof (guint32_le);
+  file->n_buckets = n_buckets;
+
+  if G_UNLIKELY (size % sizeof (struct gvdb_hash_item))
+    return;
+
+  file->hash_items = (gpointer) (file->hash_buckets + n_buckets);
+  file->n_hash_items = size / sizeof (struct gvdb_hash_item);
+}
+
+GHashFile *
+g_hash_file_new (const gchar  *filename,
+                 gboolean      trusted,
+                 GError      **error)
+{
+  GMappedFile *mapped;
+  GHashFile *file;
+
+  if ((mapped = g_mapped_file_new (filename, FALSE, error)) == NULL)
+    return NULL;
+
+  file = g_slice_new0 (GHashFile);
+  file->data = g_mapped_file_get_contents (mapped);
+  file->size = g_mapped_file_get_length (mapped);
+  file->mapped = mapped;
+  file->ref_count = 1;
+
+  if (sizeof (struct gvdb_header) <= file->size)
+    {
+      const struct gvdb_header *header = (gpointer) file->data;
+      g_hash_file_setup_root (file, &header->root);
+    }
+
+  return file;
+}
+
+static gboolean
+g_hash_file_bloom_filter (GHashFile *file,
+                          guint32    hash_value)
+{
+  guint32 word, mask;
+
+  if (file->n_bloom_words == 0)
+    return TRUE;
+
+  word = (hash_value / 32) % file->n_bloom_words;
+  mask = 1 << (hash_value & 31);
+  mask |= 1 << ((hash_value >> file->bloom_shift) & 31);
+
+  return (file->bloom_words[word] & mask) == mask;
+}
+
+static gboolean
+g_hash_file_check_name (GHashFile             *file,
+                        struct gvdb_hash_item *item,
+                        const gchar           *key,
+                        guint                  key_length)
+{
+  const gchar *this_key;
+  gsize this_size;
+  guint32 parent;
+
+  this_key = g_hash_file_item_get_key (file, item, &this_size);
+
+  if G_UNLIKELY (this_key == NULL || this_size > key_length)
+    return FALSE;
+
+  key_length -= this_size;
+
+  if G_UNLIKELY (memcmp (this_key, key + key_length, this_size) != 0)
+    return FALSE;
+
+  parent = guint32_from_le (item->parent);
+  if (key_length == 0 && parent == -1)
+    return TRUE;
+
+  if G_LIKELY (parent < file->n_hash_items && this_size > 0)
+    return g_hash_file_check_name (file,
+                                   &file->hash_items[parent],
+                                   key, key_length);
+
+  return FALSE;
+}
+
+const struct gvdb_hash_item *
+g_hash_file_lookup (GHashFile   *file,
+                    const gchar *key,
+                    gchar        type)
+{
+  guint32 hash_value = 5381;
+  guint key_length;
+  guint32 itemno;
+
+  if G_UNLIKELY (file->n_buckets == 0 || file->n_hash_items == 0)
+    return NULL;
+
+  for (key_length = 0; key[key_length]; key_length++)
+    hash_value = (hash_value * 33) + key[key_length];
+
+  if (!g_hash_file_bloom_filter (file, hash_value))
+    return NULL;
+
+  itemno = file->hash_buckets[hash_value % file->n_buckets];
+  hash_value &= ~(1u << 31);
+
+  while G_LIKELY (itemno < file->n_hash_items)
+    {
+      struct gvdb_hash_item *item = &file->hash_items[itemno];
+
+      if (hash_value == (guint32_from_le (item->hash_value) & ~(1u << 31)))
+        if G_LIKELY (g_hash_file_check_name (file, item, key, key_length))
+          if G_LIKELY (item->type == type)
+            return item;
+
+      if (guint32_from_le (item->hash_value) & (1u << 31))
+        return NULL;
+
+      itemno++;
+    }
+
+  return NULL;
+}
+
+gchar **
+g_hash_file_list (GHashFile   *file,
+                  const gchar *key)
+{
+  const struct gvdb_hash_item *item;
+  const guint32_le *list;
+  gchar **strv;
+  gsize size;
+  gint i;
+
+  if ((item = g_hash_file_lookup (file, key, 'L')) == NULL)
+    return NULL;
+
+  list = g_hash_file_dereference (file, &item->value.pointer, 4, &size);
+
+  if G_UNLIKELY (list == NULL || size % 4)
+    return NULL;
+
+  size /= 4;
+
+  strv = g_new (gchar *, size + 1);
+  for (i = 0; i < size; i++)
+    {
+      guint32 itemno = guint32_from_le (list[i]);
+
+      if (itemno < file->n_hash_items)
+        {
+          const struct gvdb_hash_item *item;
+          const gchar *string;
+          gsize strsize;
+
+          item = file->hash_items + itemno;
+
+          string = g_hash_file_item_get_key (file, item, &strsize);
+
+          if (string != NULL)
+            strv[i] = g_strndup (string, strsize);
+          else
+            strv[i] = g_malloc0 (1);
+        }
+      else
+        strv[i] = g_malloc0 (1);
+    }
+
+  strv[i] = NULL;
+
+  return strv;
+}
+
+GVariant *
+g_hash_file_get_value (GHashFile    *file,
+                       const gchar  *key,
+                       GVariant    **options)
+{
+  const struct gvdb_hash_item *item;
+  GVariant *variant, *value;
+  gconstpointer data;
+  gsize size;
+
+  if ((item = g_hash_file_lookup (file, key, 'v')) == NULL)
+    return NULL;
+
+  data = g_hash_file_dereference (file, &item->value.pointer, 8, &size);
+
+  if G_UNLIKELY (data == NULL)
+    return NULL;
+
+  variant = g_variant_new_from_data (G_VARIANT_TYPE_VARIANT,
+                                     data, size, file->trusted,
+                                     (GDestroyNotify) g_mapped_file_unref,
+                                     g_mapped_file_ref (file->mapped));
+  value = g_variant_get_variant (variant);
+  g_variant_unref (variant);
+
+  if (options != NULL)
+    {
+      data = g_hash_file_dereference (file, &item->options, 8, &size);
+
+      if (data != NULL || size > 0)
+        {
+          *options = g_variant_new_from_data (G_VARIANT_TYPE ("a{sv}"),
+                                              data, size, file->trusted,
+                                              (GDestroyNotify) g_mapped_file_unref,
+                                              g_mapped_file_ref (file->mapped));
+          g_variant_ref_sink (*options);
+        }
+      else
+        *options = NULL;
+    }
+
+  return value;
+}
+
+GHashFile *
+g_hash_file_get_hash (GHashFile   *file,
+                      const gchar *key)
+{
+  const struct gvdb_hash_item *item;
+  GHashFile *new;
+
+  item = g_hash_file_lookup (file, key, 'H');
+
+  if (item == NULL)
+    return NULL;
+
+  new = g_slice_new0 (GHashFile);
+  new->mapped = g_mapped_file_ref (file->mapped);
+  new->trusted = file->trusted;
+  new->data = file->data;
+  new->size = file->size;
+  new->ref_count = 1;
+
+  g_hash_file_setup_root (new, &item->value.pointer);
+
+  return new;
+}
+
+GHashFile *
+g_hash_file_ref (GHashFile *file)
+{
+  g_atomic_int_inc (&file->ref_count);
+
+  return file;
+}
+
+void
+g_hash_file_unref (GHashFile *file)
+{
+  if (g_atomic_int_dec_and_test (&file->ref_count))
+    {
+      g_mapped_file_unref (file->mapped);
+      g_slice_free (GHashFile, file);
+    }
+}
diff --git a/glib/ghashfile.h b/glib/ghashfile.h
new file mode 100644
index 0000000..5bd9d54
--- /dev/null
+++ b/glib/ghashfile.h
@@ -0,0 +1,42 @@
+/*
+ * Copyright © 2010 Codethink Limited
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the licence, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307, USA.
+ */
+
+#ifndef __G_HASH_FILE_H__
+#define __G_HASH_FILE_H__
+
+#include <glib/gvariant.h>
+#include <glib/gerror.h>
+
+typedef struct _GHashFile GHashFile;
+
+GHashFile *             g_hash_file_new                                 (const gchar  *filename,
+                                                                         gboolean      trusted,
+                                                                         GError      **error);
+GHashFile *		g_hash_file_ref					(GHashFile    *file);
+void			g_hash_file_unref				(GHashFile    *file);
+
+gchar **                g_hash_file_list                                (GHashFile    *file,
+                                                                         const gchar  *key);
+GHashFile *		g_hash_file_get_hash				(GHashFile    *file,
+									 const gchar  *key);
+GVariant *              g_hash_file_get_value                           (GHashFile    *file,
+                                                                         const gchar  *key,
+                                                                         GVariant    **options);
+
+#endif /* __G_HASH_FILE_H__ */
diff --git a/glib/glib.h b/glib/glib.h
index e07ec82..76f5b42 100644
--- a/glib/glib.h
+++ b/glib/glib.h
@@ -88,6 +88,7 @@
 #include <glib/gutils.h>
 #include <glib/gvarianttype.h>
 #include <glib/gvariant.h>
+#include <glib/ghashfile.h>
 #ifdef G_PLATFORM_WIN32
 #include <glib/gwin32.h>
 #endif
diff --git a/glib/gvdb.h b/glib/gvdb.h
new file mode 100644
index 0000000..73d15b7
--- /dev/null
+++ b/glib/gvdb.h
@@ -0,0 +1,61 @@
+#include <glib.h>
+
+typedef struct { guint16 __value; } guint16_le;
+typedef struct { guint32 __value; } guint32_le;
+
+struct gvdb_pointer {
+  guint32_le start;
+  guint32_le end;
+};
+
+struct gvdb_hash_header {
+  guint32_le n_bloom_words;
+  guint32_le n_buckets;
+};
+
+struct gvdb_hash_item {
+  guint32_le hash_value;
+  guint32_le parent;
+
+  guint32_le key_start;
+  guint16_le key_size;
+  gchar type;
+  gchar unused;
+
+  union
+  {
+    struct gvdb_pointer pointer;
+    gchar direct[8];
+  } value;
+
+  struct gvdb_pointer options;
+};
+
+struct gvdb_header {
+  guint32 signature[2];
+  guint32_le version;
+  guint32_le options;
+
+  struct gvdb_pointer root;
+};
+
+static inline guint32_le guint32_to_le (guint32 value) {
+  guint32_le result = { GUINT32_TO_LE (value) };
+  return result;
+}
+
+static inline guint32 guint32_from_le (guint32_le value) {
+  return GUINT32_FROM_LE (value.__value);
+}
+
+static inline guint16_le guint16_to_le (guint16 value) {
+  guint16_le result = { GUINT16_TO_LE (value) };
+  return result;
+}
+
+static inline guint16 guint16_from_le (guint16_le value) {
+  return GUINT16_FROM_LE (value.__value);
+}
+
+#define GVDB_SIGNATURE0 1918981703
+#define GVDB_SIGNATURE1 1953390953