tracker r1752 - in branches/indexer-split: . src/tracker-indexer src/tracker-indexer/modules

[carlosg svn gnome org]

Author: carlosg
Date: Tue Jun 24 15:07:10 2008
New Revision: 1752
URL: http://svn.gnome.org/viewvc/tracker?rev=1752&view=rev

Log:
2008-06-24  Carlos Garnacho  <carlos imendio com>

        Extend/Change indexer modules API. From now on, modules will deal with
        TrackerFile structs instead of paths. This struct, besides the path,
        contains a pointer to private data that modules can attach/free
        through tracker_module_file_get_data() and
        tracker_module_file_free_data().

        In order to deal with files that are potential containers for sets of
        metadata, tracker_module_file_iter_contents() has been added, for
        modules that implement this call, tracker-indexer will be able to
        collect data through tracker_module_file_get_metadata(). If
        iter_contents() returns FALSE, the indexer assumes no more data is
        left in the file, moving on to the next one.

        * src/tracker-indexer/tracker-module.h: New header, modules must
        conform to the API described in it.

        * src/tracker-indexer/tracker-indexer-module.[ch]: Use definitions
        from tracker-module.h
        (tracker_indexer_module_file_new) (tracker_indexer_module_file_free):
        New functions to create/free TrackerFile structs.
        (tracker_indexer_module_file_iter_contents): New function to iterate
        through a file contents, the meaning of this will depend on the module
        implementation.

        * src/tracker-indexer/tracker-indexer.c: Deal with TrackerFile
        structs.
        (process_file): Iterate through file contents, once the file doesn't
        have more elements to process, the file will be removed from the
        queue.

        * src/tracker-indexer/modules/applications.c:
        * src/tracker-indexer/modules/files.c: Adapt to the new API.


Added:
   branches/indexer-split/src/tracker-indexer/tracker-module.h
Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/src/tracker-indexer/modules/applications.c
   branches/indexer-split/src/tracker-indexer/modules/files.c
   branches/indexer-split/src/tracker-indexer/tracker-indexer-module.c
   branches/indexer-split/src/tracker-indexer/tracker-indexer-module.h
   branches/indexer-split/src/tracker-indexer/tracker-indexer.c

Modified: branches/indexer-split/src/tracker-indexer/modules/applications.c
==============================================================================
--- branches/indexer-split/src/tracker-indexer/modules/applications.c	(original)
+++ branches/indexer-split/src/tracker-indexer/modules/applications.c	Tue Jun 24 15:07:10 2008
@@ -19,6 +19,7 @@
 
 #include <stdlib.h>
 #include <glib.h>
+#include <tracker-indexer/tracker-module.h>
 
 #define GROUP_DESKTOP_ENTRY "Desktop Entry"
 #define KEY_TYPE            "Type"
@@ -112,20 +113,20 @@
 }
 
 GHashTable *
-tracker_module_get_file_metadata (const gchar *file)
+tracker_module_file_get_metadata (TrackerFile *file)
 {
 	GHashTable *metadata;
 	GKeyFile *key_file;
 	gchar *type, *filename;
 
 	/* Check we're dealing with a desktop file */
-	if (!g_str_has_suffix (file, ".desktop")) {
+	if (!g_str_has_suffix (file->path, ".desktop")) {
 		return NULL;
 	}
 
 	key_file = g_key_file_new ();
 
-	if (!g_key_file_load_from_file (key_file, file, G_KEY_FILE_NONE, NULL)) {
+	if (!g_key_file_load_from_file (key_file, file->path, G_KEY_FILE_NONE, NULL)) {
 		g_key_file_free (key_file);
 		return NULL;
 	}
@@ -157,7 +158,7 @@
 
 	/* FIXME: mimetypes list and categories? */
 
-	filename = g_filename_display_basename (file);
+	filename = g_filename_display_basename (file->path);
 	g_hash_table_insert (metadata, METADATA_FILE_NAME, filename);
 
 	g_key_file_free (key_file);

Modified: branches/indexer-split/src/tracker-indexer/modules/files.c
==============================================================================
--- branches/indexer-split/src/tracker-indexer/modules/files.c	(original)
+++ branches/indexer-split/src/tracker-indexer/modules/files.c	Tue Jun 24 15:07:10 2008
@@ -29,6 +29,7 @@
 #include <libtracker-common/tracker-type-utils.h>
 #include <libtracker-common/tracker-os-dependant.h>
 #include <libtracker-common/tracker-ontology.h>
+#include <tracker-indexer/tracker-module.h>
 
 #define METADATA_FILE_NAME_DELIMITED "File:NameDelimited"
 #define METADATA_FILE_EXT            "File:Ext"
@@ -243,39 +244,42 @@
 }
 
 GHashTable *
-tracker_module_get_file_metadata (const gchar *file)
+tracker_module_file_get_metadata (TrackerFile *file)
 {
+	const gchar *path;
 	GHashTable *metadata;
 	struct stat st;
 	const gchar *ext;
 	gchar *mimetype;
 
-	if (check_exclude_file (file)) {
+	path = file->path;
+
+	if (check_exclude_file (path)) {
 		return NULL;
 	}
 
-	g_lstat (file, &st);
+	g_lstat (path, &st);
 	metadata = g_hash_table_new_full (g_str_hash, g_str_equal,
 					  NULL,
 					  (GDestroyNotify) g_free);
-	ext = strrchr (file, '.');
+	ext = strrchr (path, '.');
 
 	if (ext) {
 		g_hash_table_insert (metadata, METADATA_FILE_EXT, g_strdup (ext + 1));
 	}
 
-	mimetype = tracker_file_get_mime_type (file);
+	mimetype = tracker_file_get_mime_type (path);
 
-	g_hash_table_insert (metadata, METADATA_FILE_NAME, g_filename_display_basename (file));
-	g_hash_table_insert (metadata, METADATA_FILE_PATH, g_path_get_dirname (file));
+	g_hash_table_insert (metadata, METADATA_FILE_NAME, g_filename_display_basename (path));
+	g_hash_table_insert (metadata, METADATA_FILE_PATH, g_path_get_dirname (path));
 	g_hash_table_insert (metadata, METADATA_FILE_NAME_DELIMITED,
-			     g_filename_to_utf8 (file, -1, NULL, NULL, NULL));
+			     g_filename_to_utf8 (path, -1, NULL, NULL, NULL));
 	g_hash_table_insert (metadata, METADATA_FILE_MIMETYPE, mimetype);
 
 	if (S_ISLNK (st.st_mode)) {
 		gchar *link_path;
 
-		link_path = g_file_read_link (file, NULL);
+		link_path = g_file_read_link (path, NULL);
 		g_hash_table_insert (metadata, METADATA_FILE_LINK,
 				     g_filename_to_utf8 (link_path, -1, NULL, NULL, NULL));
 		g_free (link_path);
@@ -289,12 +293,12 @@
 	g_hash_table_insert (metadata, METADATA_FILE_ACCESSED,
 			     tracker_uint_to_string (st.st_atime));
 
-	tracker_metadata_get_embedded (file, mimetype, metadata);
+	tracker_metadata_get_embedded (path, mimetype, metadata);
 
 	return metadata;
 }
 
-gchar *
+static gchar *
 tracker_metadata_call_text_filter (const gchar *path,
 				   const gchar *mime)
 {
@@ -335,12 +339,12 @@
 }
 
 gchar *
-tracker_module_get_file_text (const gchar *file)
+tracker_module_file_get_text (TrackerFile *file)
 {
 	gchar *mimetype, *service_type;
 	gchar *text = NULL;
 
-	mimetype = tracker_file_get_mime_type (file);
+	mimetype = tracker_file_get_mime_type (file->path);
 	service_type = tracker_ontology_get_service_type_for_mime (mimetype);
 
 	/* No need to filter text based files - index them directly */
@@ -349,14 +353,14 @@
              strcmp (service_type, "Development") == 0)) {
 		GMappedFile *mapped_file;
 
-		mapped_file = g_mapped_file_new (file, FALSE, NULL);
+		mapped_file = g_mapped_file_new (file->path, FALSE, NULL);
 
 		if (mapped_file) {
 			text = g_strdup (g_mapped_file_get_contents (mapped_file));
 			g_mapped_file_free (mapped_file);
 		}
 	} else {
-		text = tracker_metadata_call_text_filter (file, mimetype);
+		text = tracker_metadata_call_text_filter (file->path, mimetype);
 	}
 
 	g_free (mimetype);

Modified: branches/indexer-split/src/tracker-indexer/tracker-indexer-module.c
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-indexer-module.c	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer-module.c	Tue Jun 24 15:07:10 2008
@@ -23,11 +23,6 @@
 
 #include "tracker-indexer-module.h"
 
-typedef const gchar * (* TrackerIndexerModuleGetName) (void);
-typedef gchar **      (* TrackerIndexerModuleGetDirectories) (void);
-typedef GHashTable *  (* TrackerIndexerModuleGetData) (const gchar *path);
-typedef gchar *       (* TrackerIndexerModuleGetText) (const gchar *path);
-
 GModule *
 tracker_indexer_module_load (const gchar *module_name)
 {
@@ -56,7 +51,7 @@
 G_CONST_RETURN gchar *
 tracker_indexer_module_get_name (GModule *module)
 {
-	TrackerIndexerModuleGetName func;
+	TrackerModuleGetNameFunc func;
 
 	if (g_module_symbol (module, "tracker_module_get_name", (gpointer *) &func)) {
 		return (func) ();
@@ -68,7 +63,7 @@
 gchar **
 tracker_indexer_module_get_directories (GModule *module)
 {
-	TrackerIndexerModuleGetDirectories func;
+	TrackerModuleGetDirectoriesFunc func;
 
 	if (g_module_symbol (module, "tracker_module_get_directories", (gpointer *) &func)) {
 		return (func) ();
@@ -80,7 +75,7 @@
 gchar **
 tracker_indexer_module_get_ignore_directories (GModule *module)
 {
-	TrackerIndexerModuleGetDirectories func;
+	TrackerModuleGetDirectoriesFunc func;
 
 	if (g_module_symbol (module, "tracker_module_get_ignore_directories", (gpointer *) &func)) {
 		return (func) ();
@@ -89,13 +84,45 @@
 	return NULL;
 }
 
+TrackerFile *
+tracker_indexer_module_file_new (GModule     *module,
+				 const gchar *path)
+{
+	TrackerModuleFileGetDataFunc func;
+	TrackerFile *file = NULL;
+
+	file = g_slice_new0 (TrackerFile);
+	file->path = g_strdup (path);
+
+	if (g_module_symbol (module, "tracker_module_file_get_data", (gpointer *) &func)) {
+		file->data = (func) (path);
+	}
+
+	return file;
+}
+
+void
+tracker_indexer_module_file_free (GModule     *module,
+				  TrackerFile *file)
+{
+	TrackerModuleFileFreeDataFunc func;
+
+	if (file->data &&
+	    g_module_symbol (module, "tracker_module_file_free_data", (gpointer *) &func)) {
+		(func) (file->data);
+	}
+
+	g_free (file->path);
+	g_slice_free (TrackerFile, file);
+}
+
 GHashTable *
-tracker_indexer_module_get_file_metadata (GModule     *module,
-					  const gchar *file)
+tracker_indexer_module_file_get_metadata (GModule     *module,
+					  TrackerFile *file)
 {
-	TrackerIndexerModuleGetData func;
+	TrackerModuleFileGetMetadataFunc func;
 
-	if (g_module_symbol (module, "tracker_module_get_file_metadata", (gpointer *) &func)) {
+	if (g_module_symbol (module, "tracker_module_file_get_metadata", (gpointer *) &func)) {
 		return (func) (file);
         }
 
@@ -103,14 +130,27 @@
 }
 
 gchar *
-tracker_indexer_module_get_text (GModule     *module,
-				 const gchar *file)
+tracker_indexer_module_file_get_text (GModule     *module,
+				      TrackerFile *file)
 {
-	TrackerIndexerModuleGetText func;
+	TrackerModuleFileGetText func;
 
-	if (g_module_symbol (module, "tracker_module_get_file_text", (gpointer *) &func)) {
+	if (g_module_symbol (module, "tracker_module_file_get_text", (gpointer *) &func)) {
 		return (func) (file);
         }
 
 	return NULL;
 }
+
+gboolean
+tracker_indexer_module_file_iter_contents (GModule     *module,
+					   TrackerFile *file)
+{
+	TrackerModuleFileIterContents func;
+
+	if (g_module_symbol (module, "tracker_module_file_iter_contents", (gpointer *) &func)) {
+		return (func) (file);
+	}
+
+	return FALSE;
+}

Modified: branches/indexer-split/src/tracker-indexer/tracker-indexer-module.h
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-indexer-module.h	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer-module.h	Tue Jun 24 15:07:10 2008
@@ -23,6 +23,7 @@
 #define __TRACKER_INDEXER_MODULE_H__
 
 #include <glib.h>
+#include "tracker-module.h"
 
 G_BEGIN_DECLS
 
@@ -32,10 +33,18 @@
 gchar **                tracker_indexer_module_get_directories        (GModule     *module);
 gchar **                tracker_indexer_module_get_ignore_directories (GModule     *module);
 
-GHashTable *            tracker_indexer_module_get_file_metadata      (GModule     *module,
-								       const gchar *file);
-gchar *                 tracker_indexer_module_get_text               (GModule     *module,
-								       const gchar *file);
+TrackerFile *           tracker_indexer_module_file_new               (GModule     *module,
+								       const gchar *path);
+void                    tracker_indexer_module_file_free              (GModule     *module,
+								       TrackerFile *file);
+
+GHashTable *            tracker_indexer_module_file_get_metadata      (GModule     *module,
+								       TrackerFile *file);
+gchar *                 tracker_indexer_module_file_get_text          (GModule     *module,
+								       TrackerFile *file);
+
+gboolean                tracker_indexer_module_file_iter_contents     (GModule     *module,
+								       TrackerFile *file);
 
 G_END_DECLS
 

Modified: branches/indexer-split/src/tracker-indexer/tracker-indexer.c
==============================================================================
--- branches/indexer-split/src/tracker-indexer/tracker-indexer.c	(original)
+++ branches/indexer-split/src/tracker-indexer/tracker-indexer.c	Tue Jun 24 15:07:10 2008
@@ -61,6 +61,7 @@
 #include "tracker-indexer-module.h"
 #include "tracker-indexer-db.h"
 #include "tracker-index.h"
+#include "tracker-module.h"
 
 #define TRACKER_INDEXER_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_INDEXER, TrackerIndexerPrivate))
 
@@ -99,7 +100,7 @@
 
 struct PathInfo {
 	GModule *module;
-	gchar *path;
+	TrackerFile *file;
 };
 
 struct MetadataForeachData {
@@ -135,7 +136,7 @@
 
 	info = g_slice_new (PathInfo);
 	info->module = module;
-	info->path = g_strdup (path);
+	info->file = tracker_indexer_module_file_new (module, path);
 
 	return info;
 }
@@ -143,7 +144,7 @@
 static void
 path_info_free (PathInfo *info)
 {
-	g_free (info->path);
+	tracker_indexer_module_file_free (info->module, info->file);
 	g_slice_free (PathInfo, info);
 }
 
@@ -382,7 +383,7 @@
 
 	if (ignore_dirs) {
 		for (i = 0; ignore_dirs[i]; i++) {
-			if (strcmp (info->path, ignore_dirs[i]) == 0) {
+			if (strcmp (info->file->path, ignore_dirs[i]) == 0) {
 				ignore = TRUE;
 				break;
 			}
@@ -392,7 +393,7 @@
 	if (!ignore) {
 		g_queue_push_tail (priv->dir_queue, info);
 	} else {
-		g_message ("Ignoring directory:'%s'", info->path);
+		g_message ("Ignoring directory:'%s'", info->file->path);
 		path_info_free (info);
 	}
 
@@ -465,15 +466,15 @@
 	}
 }
 
-static void
+static gboolean
 process_file (TrackerIndexer *indexer,
 	      PathInfo       *info)
 {
 	GHashTable *metadata;
 
-	g_message ("Processing file:'%s'", info->path);
+	g_message ("Processing file:'%s'", info->file->path);
 
-	metadata = tracker_indexer_module_get_file_metadata (info->module, info->path);
+	metadata = tracker_indexer_module_file_get_metadata (info->module, info->file);
 
 	if (metadata) {
 		TrackerService *service;
@@ -496,7 +497,7 @@
 
 		tracker_db_interface_start_transaction (priv->metadata);
 
-		if (tracker_db_create_service (priv->metadata, id, service, info->path, metadata)) {
+		if (tracker_db_create_service (priv->metadata, id, service, info->file->path, metadata)) {
 			gchar *text;
 			guint32 eid;
 
@@ -508,7 +509,7 @@
 
 			index_metadata (indexer, id, service, metadata);
 
-			text = tracker_indexer_module_get_text (info->module, info->path);
+			text = tracker_indexer_module_file_get_text (info->module, info->file);
 
 			if (text) {
 				tracker_db_set_text (priv->contents, id, text);
@@ -522,6 +523,8 @@
 
 		g_hash_table_destroy (metadata);
 	}
+
+	return !tracker_indexer_module_file_iter_contents (info->module, info->file);
 }
 
 static void
@@ -532,9 +535,9 @@
 	const gchar *name;
 	GDir *dir;
 
-	g_message ("Processing directory:'%s'", info->path);
+	g_message ("Processing directory:'%s'", info->file->path);
 
-	dir = g_dir_open (info->path, 0, NULL);
+	dir = g_dir_open (info->file->path, 0, NULL);
 
 	if (!dir) {
 		return;
@@ -544,7 +547,7 @@
 		PathInfo *new_info;
 		gchar *path;
 
-		path = g_build_filename (info->path, name, NULL);
+		path = g_build_filename (info->file->path, name, NULL);
 
 		new_info = path_info_new (info->module, path);
 		tracker_indexer_add_file (indexer, new_info);
@@ -604,10 +607,12 @@
 	indexer = (TrackerIndexer *) data;
 	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
 
-	if ((path = g_queue_pop_head (priv->file_process_queue)) != NULL) {
+	if ((path = g_queue_peek_head (priv->file_process_queue)) != NULL) {
 		/* Process file */
-		process_file (indexer, path);
-		path_info_free (path);
+		if (process_file (indexer, path)) {
+			path = g_queue_pop_head (priv->file_process_queue);
+			path_info_free (path);
+		}
 	} else if ((path = g_queue_pop_head (priv->dir_queue)) != NULL) {
 		/* Process directory contents */
 		process_directory (indexer, path, TRUE);

Added: branches/indexer-split/src/tracker-indexer/tracker-module.h
==============================================================================
--- (empty file)
+++ branches/indexer-split/src/tracker-indexer/tracker-module.h	Tue Jun 24 15:07:10 2008
@@ -0,0 +1,60 @@
+/* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
+ * Copyright (C) 2008, Nokia
+
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __TRACKER_MODULE_H__
+#define __TRACKER_MODULE_H__
+
+G_BEGIN_DECLS
+
+#include <glib.h>
+
+typedef struct TrackerFile TrackerFile;
+
+struct TrackerFile {
+	gchar    *path;
+	gpointer  data;
+};
+
+
+typedef const gchar * (* TrackerModuleGetNameFunc)        (void);
+typedef gchar **      (* TrackerModuleGetDirectoriesFunc) (void);
+
+typedef gpointer      (* TrackerModuleFileGetDataFunc)  (const gchar *path);
+typedef void          (* TrackerModuleFileFreeDataFunc) (gpointer     data);
+
+typedef GHashTable *  (* TrackerModuleFileGetMetadataFunc) (TrackerFile *file);
+typedef gchar *       (* TrackerModuleFileGetText)         (TrackerFile *path);
+typedef gboolean      (* TrackerModuleFileIterContents)    (TrackerFile *path);
+
+
+G_CONST_RETURN gchar * tracker_module_get_name               (void);
+gchar **               tracker_module_get_directories        (void);
+gchar **               tracker_module_get_ignore_directories (void);
+
+gpointer               tracker_module_file_get_data  (const gchar *path);
+void                   tracker_module_file_free_data (gpointer     file_data);
+
+GHashTable *           tracker_module_file_get_metadata  (TrackerFile *file);
+gchar *                tracker_module_file_get_text      (TrackerFile *file);
+gboolean               tracker_module_file_iter_contents (TrackerFile *file);
+
+
+G_END_DECLS
+
+#endif /* __TRACKER_MODULE_H__ */