[tracker/extractor-rules-list: 1/5] libtracker-extract: Add extractor modules manager.

[Carlos Garnacho <carlosg src gnome org>]

commit ffd82572f95e2750f200458624428d58037494d3
Author: Carlos Garnacho <carlosg gnome org>
Date:   Thu Mar 10 13:06:24 2011 +0100

    libtracker-extract: Add extractor modules manager.
    
    This manager parses a directory of keyfiles, describing rules
    for matches, the modules aren't preloaded on initialization,
    instead they are loaded lazily as needed.
    
    This supersedes the module loading within
    src/tracker-extract/tracker-extract.c, and will be used
    as a replacement in following commits. The specific/generic
    matching there can be replaced by rules at different priorities
    calling the same module.

 src/libtracker-extract/Makefile.am                 |    4 +
 .../tracker-extract-module-manager.c               |  287 ++++++++++++++++++++
 .../tracker-extract-module-manager.h               |   46 +++
 3 files changed, 337 insertions(+), 0 deletions(-)
---
diff --git a/src/libtracker-extract/Makefile.am b/src/libtracker-extract/Makefile.am
index e4b691e..3bdf9ad 100644
--- a/src/libtracker-extract/Makefile.am
+++ b/src/libtracker-extract/Makefile.am
@@ -5,6 +5,7 @@ AM_CPPFLAGS =                                          \
 	-I$(top_srcdir)/src                            \
 	-I$(top_builddir)/src                          \
 	-DSHAREDIR=\""$(datadir)"\"                    \
+	-DTRACKER_EXTRACTOR_RULES_DIR=\""$(datadir)/tracker/extract-rules"\" \
 	$(LIBTRACKER_EXTRACT_CFLAGS)
 
 lib_LTLIBRARIES = libtracker-extract- TRACKER_API_VERSION@.la
@@ -15,6 +16,8 @@ libtracker_extract_ TRACKER_API_VERSION@_la_SOURCES =  \
 	tracker-data.h                                 \
 	tracker-exif.c                                 \
 	tracker-exif.h                                 \
+	tracker-extract-module-manager.c               \
+	tracker-extract-module-manager.h               \
 	tracker-guarantee.c                            \
 	tracker-guarantee.h                            \
 	tracker-iptc.c                                 \
@@ -29,6 +32,7 @@ libtracker_extractinc_HEADERS =                        \
 	tracker-data.h                                 \
 	tracker-exif.h                                 \
 	tracker-extract.h                              \
+	tracker-extract-module-manager.h               \
 	tracker-guarantee.h                            \
 	tracker-iptc.h                                 \
 	tracker-utils.h                                \
diff --git a/src/libtracker-extract/tracker-extract-module-manager.c b/src/libtracker-extract/tracker-extract-module-manager.c
new file mode 100644
index 0000000..a45f681
--- /dev/null
+++ b/src/libtracker-extract/tracker-extract-module-manager.c
@@ -0,0 +1,287 @@
+/*
+ * Copyright (C) 2010, Nokia <ivan frade nokia com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+#include <string.h>
+#include "tracker-extract-module-manager.h"
+
+#define EXTRACTOR_FUNCTION "tracker_extract_get_metadata"
+
+typedef struct {
+	const gchar *module_path; /* intern string */
+	GList *patterns;
+} RuleInfo;
+
+typedef struct {
+	GModule *module;
+	TrackerExtractMetadataFunc func;
+} ModuleInfo;
+
+static GHashTable *modules = NULL;
+static GHashTable *mimetype_map = NULL;
+static gboolean initialized = FALSE;
+static GArray *rules = NULL;
+
+static gboolean
+load_extractor_rule (GKeyFile  *key_file,
+                     GError   **error)
+{
+	gchar *module_path, **mimetypes;
+	gsize n_mimetypes, i;
+	RuleInfo rule = { 0 };
+
+	module_path = g_key_file_get_string (key_file, "ExtractorRule", "ModulePath", error);
+
+	if (!module_path) {
+		return FALSE;
+	}
+
+	mimetypes = g_key_file_get_string_list (key_file, "ExtractorRule", "MimeTypes", &n_mimetypes, error);
+
+	if (!mimetypes) {
+		g_free (module_path);
+		return FALSE;
+	}
+
+	/* Construct the rule */
+	rule.module_path = g_intern_string (module_path);
+
+	for (i = 0; i < n_mimetypes; i++) {
+		GPatternSpec *pattern;
+
+		pattern = g_pattern_spec_new (mimetypes[i]);
+		rule.patterns = g_list_prepend (rule.patterns, pattern);
+	}
+
+	if (G_UNLIKELY (!rules)) {
+		rules = g_array_new (FALSE, TRUE, sizeof (RuleInfo));
+	}
+
+	g_array_append_val (rules, rule);
+
+	return TRUE;
+}
+
+gboolean
+tracker_extract_module_manager_init (void)
+{
+	const gchar *extractors_dir, *name;
+	GList *files = NULL, *l;
+	GError *error = NULL;
+	GDir *dir;
+
+	if (initialized) {
+		return TRUE;
+	}
+
+	if (!g_module_supported ()) {
+		g_error ("Modules are not supported for this platform");
+		return FALSE;
+	}
+
+	extractors_dir = g_getenv ("TRACKER_EXTRACTOR_RULES_DIR");
+	if (G_LIKELY (extractors_dir == NULL)) {
+		extractors_dir = TRACKER_EXTRACTOR_RULES_DIR;
+	} else {
+		g_message ("Extractor rules directory is '%s' (set in env)", extractors_dir);
+	}
+
+	dir = g_dir_open (extractors_dir, 0, &error);
+
+	if (!dir) {
+		g_error ("Error opening extractor rules directory: %s", error->message);
+		g_error_free (error);
+		return FALSE;
+	}
+
+	while ((name = g_dir_read_name (dir)) != NULL) {
+		files = g_list_insert_sorted (files, (gpointer) name, (GCompareFunc) g_strcmp0);
+	}
+
+	g_message ("Loading extractor rules...");
+
+	for (l = files; l; l = l->next) {
+		GKeyFile *key_file;
+		const gchar *name;
+		gchar *path;
+
+		name = l->data;
+
+		if (!g_str_has_suffix (l->data, ".rule")) {
+			g_message ("  Skipping file '%s', no '.rule' suffix", name);
+			continue;
+		}
+
+		path = g_build_filename (extractors_dir, name, NULL);
+		key_file = g_key_file_new ();
+
+		if (!g_key_file_load_from_file (key_file, path, G_KEY_FILE_NONE, &error) ||
+		    !load_extractor_rule (key_file, &error)) {
+			g_warning ("  Could not load extractor rule file '%s': %s", name, error->message);
+			g_clear_error (&error);
+			continue;
+		}
+
+		g_key_file_free (key_file);
+		g_free (path);
+	}
+
+	g_message ("Extractor rules loaded");
+	g_dir_close (dir);
+
+	/* Initialize miscellaneous data */
+	mimetype_map = g_hash_table_new_full (g_str_hash, g_str_equal,
+	                                      (GDestroyNotify) g_free,
+	                                      NULL);
+	initialized = TRUE;
+
+	return TRUE;
+}
+
+static RuleInfo *
+lookup_rule (const gchar *mimetype)
+{
+	RuleInfo *info;
+	gchar *reversed;
+	gint len, i;
+
+	if (!rules) {
+		return NULL;
+	}
+
+	if (!mimetype_map) {
+		info = g_hash_table_lookup (mimetype_map, mimetype);
+
+		if (info) {
+			return info;
+		}
+	}
+
+	reversed = g_strdup (mimetype);
+	g_strreverse (reversed);
+	len = strlen (mimetype);
+
+	/* Apply the rules! */
+	for (i = 0; i < rules->len; i++) {
+		GList *l;
+
+		info = &g_array_index (rules, RuleInfo, i);
+
+		for (l = info->patterns; l; l = l->next) {
+			if (g_pattern_match (l->data, len, mimetype, reversed)) {
+				/* Match, store for future queries and return */
+				g_hash_table_insert (mimetype_map, g_strdup (mimetype), info);
+				g_free (reversed);
+				return info;
+			}
+		}
+	}
+
+	g_free (reversed);
+
+	return NULL;
+}
+
+GModule *
+tracker_extract_module_manager_get_for_mimetype (const gchar                *mimetype,
+                                                 TrackerExtractMetadataFunc *func)
+{
+	TrackerExtractMetadataFunc extract_func;
+	ModuleInfo *module_info;
+	GModule *module;
+	RuleInfo *info;
+
+	if (*func) {
+		*func = NULL;
+	}
+
+	if (!initialized &&
+	    !tracker_extract_module_manager_init ()) {
+		return NULL;
+	}
+
+	info = lookup_rule (mimetype);
+
+	if (!info) {
+		return NULL;
+	}
+
+	if (modules) {
+		module_info = g_hash_table_lookup (modules, info->module_path);
+
+		if (module_info) {
+			if (func) {
+				*func = module_info->func;
+			}
+
+			return module_info->module;
+		}
+	}
+
+	/* Load the module */
+	module = g_module_open (info->module_path, G_MODULE_BIND_LOCAL);
+
+	if (!module) {
+		g_warning ("Could not load module '%s': %s",
+		           info->module_path,
+		           g_module_error ());
+		return NULL;
+	}
+
+	g_module_make_resident (module);
+
+	if (!g_module_symbol (module, EXTRACTOR_FUNCTION, (gpointer *) &extract_func)) {
+		g_warning ("Could not load module '%s': Function %s() was not found, is it exported?",
+		           g_module_name (module), EXTRACTOR_FUNCTION);
+		return NULL;
+	}
+
+	/* Add it to the cache */
+	if (G_UNLIKELY (!modules)) {
+		/* Key is an intern string, so
+		 * pointer comparison suffices
+		 */
+		modules = g_hash_table_new (NULL, NULL);
+	}
+
+	module_info = g_slice_new0 (ModuleInfo);
+	module_info->module = module;
+	module_info->func = extract_func;
+
+	g_hash_table_insert (modules, (gpointer) info->module_path, module_info);
+
+	if (func) {
+		*func = extract_func;
+	}
+
+	return module;
+}
+
+gboolean
+tracker_extract_module_manager_mimetype_is_handled (const gchar *mimetype)
+{
+	RuleInfo *info;
+
+	if (!initialized &&
+	    !tracker_extract_module_manager_init ()) {
+		return FALSE;
+	}
+
+	info = lookup_rule (mimetype);
+	return (info != NULL);
+}
diff --git a/src/libtracker-extract/tracker-extract-module-manager.h b/src/libtracker-extract/tracker-extract-module-manager.h
new file mode 100644
index 0000000..9a7eca0
--- /dev/null
+++ b/src/libtracker-extract/tracker-extract-module-manager.h
@@ -0,0 +1,46 @@
+/*
+ * Copyright (C) 2011, Nokia <ivan frade nokia com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __TRACKER_EXTRACT_MODULE_MANAGER_H__
+#define __TRACKER_EXTRACT_MODULE_MANAGER_H__
+
+#include <glib.h>
+#include <gmodule.h>
+#include <libtracker-sparql/tracker-sparql.h>
+
+G_BEGIN_DECLS
+
+#define __LIBTRACKER_EXTRACT_INSIDE__
+
+typedef gboolean (* TrackerExtractMetadataFunc) (const gchar          *uri,
+                                                 const gchar          *mime_type,
+                                                 TrackerSparqlBuilder *preupdate,
+                                                 TrackerSparqlBuilder *metadata);
+
+
+gboolean  tracker_extract_module_manager_init                (void) G_GNUC_CONST;
+GModule * tracker_extract_module_manager_get_for_mimetype    (const gchar                *mimetype,
+                                                              TrackerExtractMetadataFunc *func);
+gboolean  tracker_extract_module_manager_mimetype_is_handled (const gchar *mimetype);
+
+#undef __LIBTRACKER_EXTRACT_INSIDE__
+
+G_END_DECLS
+
+#endif /* __TRACKER_EXTRACT_MODULE_MANAGER_H__ */