tracker r1411 - in branches/xesam-support: . src/libtracker-common src/tracker-indexer src/tracker-indexer/modules src/trackerd
- From: pvanhoof svn gnome org
- To: svn-commits-list gnome org
- Subject: tracker r1411 - in branches/xesam-support: . src/libtracker-common src/tracker-indexer src/tracker-indexer/modules src/trackerd
- Date: Tue, 13 May 2008 14:47:17 +0100 (BST)
Author: pvanhoof
Date: Tue May 13 13:47:16 2008
New Revision: 1411
URL: http://svn.gnome.org/viewvc/tracker?rev=1411&view=rev
Log:
* Merged with indexer-split (Rev 1410)
-> svn merge -r 1398:1410 ../indexer-split
Added:
branches/xesam-support/src/tracker-indexer/main.c
- copied unchanged from r1410, /branches/indexer-split/src/tracker-indexer/main.c
branches/xesam-support/src/tracker-indexer/modules/
- copied from r1410, /branches/indexer-split/src/tracker-indexer/modules/
branches/xesam-support/src/tracker-indexer/tracker-indexer-module.c
- copied unchanged from r1410, /branches/indexer-split/src/tracker-indexer/tracker-indexer-module.c
branches/xesam-support/src/tracker-indexer/tracker-indexer-module.h
- copied unchanged from r1410, /branches/indexer-split/src/tracker-indexer/tracker-indexer-module.h
branches/xesam-support/src/tracker-indexer/tracker-indexer.h
- copied unchanged from r1410, /branches/indexer-split/src/tracker-indexer/tracker-indexer.h
branches/xesam-support/src/trackerd/tracker-index-stage.c
- copied unchanged from r1410, /branches/indexer-split/src/trackerd/tracker-index-stage.c
branches/xesam-support/src/trackerd/tracker-index-stage.h
- copied unchanged from r1410, /branches/indexer-split/src/trackerd/tracker-index-stage.h
Removed:
branches/xesam-support/src/trackerd/tracker-convert-file.c
Modified:
branches/xesam-support/ChangeLog
branches/xesam-support/configure.ac
branches/xesam-support/src/libtracker-common/tracker-config.c
branches/xesam-support/src/libtracker-common/tracker-config.h
branches/xesam-support/src/tracker-indexer/Makefile.am
branches/xesam-support/src/tracker-indexer/tracker-indexer.c
branches/xesam-support/src/trackerd/Makefile.am
branches/xesam-support/src/trackerd/tracker-cache.c
branches/xesam-support/src/trackerd/tracker-main.c
branches/xesam-support/src/trackerd/tracker-main.h
branches/xesam-support/src/trackerd/tracker-metadata.c
branches/xesam-support/src/trackerd/tracker-process-files.c
Modified: branches/xesam-support/configure.ac
==============================================================================
--- branches/xesam-support/configure.ac (original)
+++ branches/xesam-support/configure.ac Tue May 13 13:47:16 2008
@@ -827,6 +827,7 @@
src/libtracker-db/Makefile
src/libtracker-gtk/Makefile
src/tracker-indexer/Makefile
+ src/tracker-indexer/modules/Makefile
src/tracker-applet/Makefile
src/tracker-applet/tracker-applet.desktop.in
src/tracker-search-tool/Makefile
Modified: branches/xesam-support/src/libtracker-common/tracker-config.c
==============================================================================
--- branches/xesam-support/src/libtracker-common/tracker-config.c (original)
+++ branches/xesam-support/src/libtracker-common/tracker-config.c Tue May 13 13:47:16 2008
@@ -60,6 +60,7 @@
#define KEY_LOW_DISK_SPACE_LIMIT "LowDiskSpaceLimit"
#define KEY_INDEX_MOUNTED_DIRECTORIES "IndexMountedDirectories"
#define KEY_INDEX_REMOVABLE_DEVICES "IndexRemovableMedia"
+#define KEY_INDEX_MODULES "IndexModules"
#define GROUP_EMAILS "Emails"
#define KEY_EMAIL_CLIENT "IndexEMailClient"
@@ -96,7 +97,8 @@
#define DEFAULT_DISABLE_INDEXING_ON_BATTERY TRUE
#define DEFAULT_DISABLE_INDEXING_ON_BATTERY_INIT FALSE
#define DEFAULT_INDEX_MOUNTED_DIRECTORIES TRUE
-#define DEFAULT_INDEX_REMOVABLE_DEVICES TRUE
+#define DEFAULT_INDEX_REMOVABLE_DEVICES TRUE
+#define DEFAULT_INDEX_MODULES "applications;files;gaim-conversations;firefox-history"
#define DEFAULT_INDEX_EMAIL_CLIENT "evolution"
#define DEFAULT_LOW_DISK_SPACE_LIMIT 1 /* 0->100 / -1 */
#define DEFAULT_MAX_TEXT_TO_INDEX 1048576 /* Bytes */
@@ -141,6 +143,7 @@
gint low_disk_space_limit;
gboolean index_mounted_directories;
gboolean index_removable_devices;
+ GSList *index_modules;
/* Emails */
gchar *email_client;
@@ -202,6 +205,7 @@
PROP_LOW_DISK_SPACE_LIMIT,
PROP_INDEX_MOUNTED_DIRECTORIES,
PROP_INDEX_REMOVABLE_DEVICES,
+ PROP_INDEX_MODULES,
/* Emails */
PROP_EMAIL_CLIENT,
@@ -424,7 +428,12 @@
"which are for removable devices",
DEFAULT_INDEX_REMOVABLE_DEVICES,
G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
-
+ g_object_class_install_property (object_class,
+ PROP_INDEX_MODULES,
+ g_param_spec_pointer ("index-modules",
+ "Used index modules",
+ "Modules used to index data",
+ G_PARAM_READWRITE | G_PARAM_CONSTRUCT));
/* Emails */
g_object_class_install_property (object_class,
PROP_EMAIL_CLIENT,
@@ -638,6 +647,9 @@
case PROP_INDEX_REMOVABLE_DEVICES:
g_value_set_boolean (value, priv->index_removable_devices);
break;
+ case PROP_INDEX_MODULES:
+ g_value_set_pointer (value, priv->index_modules);
+ break;
/* Emails */
case PROP_EMAIL_CLIENT:
@@ -782,6 +794,9 @@
tracker_config_set_index_removable_devices (TRACKER_CONFIG (object),
g_value_get_boolean (value));
break;
+ case PROP_INDEX_MODULES:
+ /* Not writable */
+ break;
/* Emails */
case PROP_EMAIL_CLIENT:
@@ -973,22 +988,23 @@
}
static gboolean
-config_create_with_defaults (const gchar *filename)
+config_create_with_defaults (const gchar *filename,
+ GKeyFile *key_file)
{
- GKeyFile *key_file;
GError *error = NULL;
gchar *content = NULL;
gchar *language;
const gchar *watch_directory_roots[2] = { NULL, NULL };
const gchar *empty_string_list[] = { NULL };
-
- key_file = g_key_file_new ();
+ gchar **index_modules;
/* Get default values */
language = tracker_language_get_default_code ();
watch_directory_roots[0] = g_get_home_dir ();
+ index_modules = g_strsplit (DEFAULT_INDEX_MODULES, ";", -1);
+
/* General */
g_key_file_set_integer (key_file, GROUP_GENERAL, KEY_VERBOSITY, DEFAULT_VERBOSITY);
g_key_file_set_comment (key_file, GROUP_GENERAL, KEY_VERBOSITY,
@@ -1109,6 +1125,11 @@
g_key_file_set_comment (key_file, GROUP_INDEXING, KEY_INDEX_REMOVABLE_DEVICES,
" Set to true to enable traversing mounted directories for removable devices",
NULL);
+ g_key_file_set_string_list (key_file, GROUP_INDEXING, KEY_INDEX_MODULES,
+ (const gchar **) index_modules, g_strv_length (index_modules));
+ g_key_file_set_comment (key_file, GROUP_INDEXING, KEY_INDEX_MODULES,
+ " Modules used to extract data, they will be queried in the same order than they're written here",
+ NULL);
/* Emails */
g_key_file_set_string (key_file, GROUP_EMAILS, KEY_EMAIL_CLIENT, DEFAULT_INDEX_EMAIL_CLIENT);
@@ -1159,7 +1180,6 @@
content = g_key_file_to_data (key_file, NULL, &error);
g_free (language);
- g_key_file_free (key_file);
if (error) {
g_warning ("Couldn't produce default configuration, %s", error->message);
@@ -1175,6 +1195,7 @@
}
g_print ("Writting default configuration to file:'%s'\n", filename);
+ g_strfreev (index_modules);
g_free (content);
return TRUE;
@@ -1315,6 +1336,12 @@
config_string_list_to_gslist ((const gchar **) value, FALSE);
}
}
+ else if (strcmp (property, "index-modules") == 0) {
+ if (value) {
+ priv->index_modules =
+ config_string_list_to_gslist ((const gchar **) value, FALSE);
+ }
+ }
else {
g_warning ("Property '%s' not recognized to set string list from key '%s'",
property, key);
@@ -1347,7 +1374,7 @@
/* Load options */
g_key_file_load_from_file (key_file, filename, G_KEY_FILE_NONE, &error);
if (error) {
- config_create_with_defaults (filename);
+ config_create_with_defaults (filename, key_file);
g_clear_error (&error);
}
@@ -1381,6 +1408,7 @@
config_load_int (config, "low-disk-space-limit", key_file, GROUP_INDEXING, KEY_LOW_DISK_SPACE_LIMIT);
config_load_boolean (config, "index-mounted-directories", key_file, GROUP_INDEXING, KEY_INDEX_MOUNTED_DIRECTORIES);
config_load_boolean (config, "index-removable-devices", key_file, GROUP_INDEXING, KEY_INDEX_REMOVABLE_DEVICES);
+ config_load_string_list (config, "index-modules", key_file, GROUP_INDEXING, KEY_INDEX_MODULES);
/* Emails */
config_load_string (config, "email-client", key_file, GROUP_EMAILS, KEY_EMAIL_CLIENT);
@@ -1726,6 +1754,18 @@
return priv->index_removable_devices;
}
+GSList *
+tracker_config_get_index_modules (TrackerConfig *config)
+{
+ TrackerConfigPriv *priv;
+
+ g_return_val_if_fail (TRACKER_IS_CONFIG (config), NULL);
+
+ priv = GET_PRIV (config);
+
+ return priv->index_modules;
+}
+
const gchar *
tracker_config_get_email_client (TrackerConfig *config)
{
Modified: branches/xesam-support/src/libtracker-common/tracker-config.h
==============================================================================
--- branches/xesam-support/src/libtracker-common/tracker-config.h (original)
+++ branches/xesam-support/src/libtracker-common/tracker-config.h Tue May 13 13:47:16 2008
@@ -73,6 +73,7 @@
gboolean tracker_config_get_disable_indexing_on_battery_init (TrackerConfig *config);
gint tracker_config_get_low_disk_space_limit (TrackerConfig *config);
gboolean tracker_config_get_index_removable_devices (TrackerConfig *config);
+GSList * tracker_config_get_index_modules (TrackerConfig *config);
gboolean tracker_config_get_index_mounted_directories (TrackerConfig *config);
const gchar * tracker_config_get_email_client (TrackerConfig *config);
gint tracker_config_get_max_text_to_index (TrackerConfig *config);
Modified: branches/xesam-support/src/tracker-indexer/Makefile.am
==============================================================================
--- branches/xesam-support/src/tracker-indexer/Makefile.am (original)
+++ branches/xesam-support/src/tracker-indexer/Makefile.am Tue May 13 13:47:16 2008
@@ -1,8 +1,10 @@
+SUBDIRS = modules
+
INCLUDES = \
-DSHAREDIR=\""$(datadir)"\" \
-DLIBDIR=\""$(libdir)"\" \
-DLOCALEDIR=\""$(localedir)"\" \
- -DMAIL_MODULES_DIR=\"$(mail_modulesdir)\" \
+ -DINDEXER_MODULES_DIR=\""$(libdir)"/tracker/indexer-modules\" \
$(GLIB2_CFLAGS) \
$(GMODULE_CFLAGS) \
$(GMIME_CFLAGS) \
@@ -43,7 +45,11 @@
bin_PROGRAMS = tracker-indexer
tracker_indexer_SOURCES = \
- tracker-indexer.c
+ main.c \
+ tracker-indexer.c \
+ tracker-indexer.h \
+ tracker-indexer-module.c \
+ tracker-indexer-module.h
tracker_indexer_LDADD = \
$(GLIB2_LIBS) \
@@ -57,6 +63,7 @@
$(GTHREAD_LIBS) \
-lz \
-lm \
+ $(top_builddir)/src/libtracker-db/libtracker-db.la \
$(top_builddir)/src/libtracker-common/libtracker-common.la \
$(top_builddir)/src/libstemmer/libstemmer-private.la \
$(top_builddir)/src/xdgmime/libxdgmime.la \
Modified: branches/xesam-support/src/tracker-indexer/tracker-indexer.c
==============================================================================
--- branches/xesam-support/src/tracker-indexer/tracker-indexer.c (original)
+++ branches/xesam-support/src/tracker-indexer/tracker-indexer.c Tue May 13 13:47:16 2008
@@ -1,9 +1,8 @@
/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
-/*
+/*
* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
- * Copyright (C) 2007, Michal Pryc (Michal Pryc Sun Com)
- * Copyright (C) 2008, Nokia (urho konttori nokia com)
- *
+ * Copyright (C) 2008, Nokia
+
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
* License as published by the Free Software Foundation; either
@@ -11,7 +10,7 @@
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*
* You should have received a copy of the GNU General Public
@@ -20,154 +19,446 @@
* Boston, MA 02110-1301, USA.
*/
-#include "config.h"
+/* The indexer works as a state machine, there are 3 different queues:
+ *
+ * * The files queue: the highest priority one, individual files are
+ * stored here, waiting for metadata extraction, etc... files are
+ * taken one by one in order to be processed, when this queue is
+ * empty, a single token from the next queue is processed.
+ *
+ * * The directories queue: directories are stored here, waiting for
+ * being inspected. When a directory is inspected, contained files
+ * and directories will be prepended in their respective queues.
+ * When this queue is empty, a single token from the next queue
+ * is processed.
+ *
+ * * The modules list: indexing modules are stored here, these modules
+ * can either prepend the files or directories to be inspected in
+ * their respective queues.
+ *
+ * Once all queues are empty, all elements have been inspected, and the
+ * indexer will emit the ::finished signal, this behavior can be observed
+ * in the indexing_func() function.
+ */
-#include <string.h>
#include <stdlib.h>
-#include <locale.h>
-#include <glib/gi18n.h>
-#include <glib-object.h>
+#include <gmodule.h>
+
+#include <libtracker-common/tracker-config.h>
+#include <libtracker-db/tracker-db-interface-sqlite.h>
+
+#include "tracker-indexer.h"
+#include "tracker-indexer-module.h"
+
+#define TRACKER_INDEXER_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_INDEXER, TrackerIndexerPrivate))
+
+typedef struct TrackerIndexerPrivate TrackerIndexerPrivate;
+typedef struct PathInfo PathInfo;
+
+struct TrackerIndexerPrivate {
+ GQueue *dir_queue;
+ GQueue *file_process_queue;
+
+ GSList *module_names;
+ GSList *current_module;
+ GHashTable *indexer_modules;
+
+ TrackerDBInterface *index;
+
+ TrackerConfig *config;
+
+ guint idle_id;
+};
-#define COPYRIGHT \
- "Tracker version " PACKAGE_VERSION "\n" \
- "Copyright (c) 2005-2008 by Jamie McCracken (jamiemcc gnome org)"
-
-#define WARRANTY \
- "This program is free software and comes without any warranty.\n" \
- "It is licensed under version 2 or later of the General Public\n" \
- "License which can be viewed at:\n" \
- "\n" \
- "\thttp://www.gnu.org/licenses/gpl.txt";
-
-static gchar **no_watch_dirs;
-static gchar **watch_dirs;
-static gchar **crawl_dirs;
-static gchar *language;
-static gboolean disable_indexing;
-static gboolean reindex;
-static gboolean fatal_errors;
-static gboolean low_memory;
-static gint throttle = -1;
-static gint verbosity;
-static gint initial_sleep = -1;
-
-static GOptionEntry entries[] = {
- { "exclude-dir", 'e', 0, G_OPTION_ARG_STRING_ARRAY, &no_watch_dirs,
- N_("Directory to exclude from indexing"),
- N_("/PATH/DIR")
- },
- { "include-dir", 'i', 0, G_OPTION_ARG_STRING_ARRAY, &watch_dirs,
- N_("Directory to include in indexing"),
- N_("/PATH/DIR")
- },
- { "crawl-dir", 'c', 0, G_OPTION_ARG_STRING_ARRAY, &crawl_dirs,
- N_("Directory to crawl for indexing at start up only"),
- N_("/PATH/DIR")
- },
- { "no-indexing", 'n', 0, G_OPTION_ARG_NONE, &disable_indexing,
- N_("Disable any indexing or watching taking place"),
- NULL
- },
- { "verbosity", 'v', 0, G_OPTION_ARG_INT, &verbosity,
- N_("Value that controls the level of logging. Valid values "
- "are 0=errors, 1=minimal, 2=detailed, 3=debug"),
- N_("VALUE")
- },
- { "throttle", 't', 0, G_OPTION_ARG_INT, &throttle,
- N_("Value to use for throttling indexing. Value must be in "
- "range 0-99 (default=0) with lower values increasing "
- "indexing speed"),
- N_("VALUE")
- },
- { "low-memory", 'm', 0, G_OPTION_ARG_NONE, &low_memory,
- N_("Minimizes the use of memory but may slow indexing down"),
- NULL
- },
- { "initial-sleep", 's', 0, G_OPTION_ARG_INT, &initial_sleep,
- N_("Initial sleep time, just before indexing, in seconds"),
- NULL
- },
- { "language", 'l', 0, G_OPTION_ARG_STRING, &language,
- N_("Language to use for stemmer and stop words list "
- "(ISO 639-1 2 characters code)"),
- N_("LANG")
- },
- { "reindex", 'R', 0, G_OPTION_ARG_NONE, &reindex,
- N_("Force a re-index of all content"),
- NULL
- },
- { "fatal-errors", 'f', 0, G_OPTION_ARG_NONE, &fatal_errors,
- N_("Make tracker errors fatal"),
- NULL
- },
- { NULL }
+struct PathInfo {
+ GModule *module;
+ gchar *path;
};
-gint
-main (gint argc, gchar *argv[])
+enum {
+ PROP_0,
+ PROP_RUNNING
+};
+
+enum {
+ FINISHED,
+ LAST_SIGNAL
+};
+
+static guint signals[LAST_SIGNAL] = { 0, };
+
+G_DEFINE_TYPE (TrackerIndexer, tracker_indexer, G_TYPE_OBJECT)
+
+static PathInfo *
+path_info_new (GModule *module,
+ const gchar *path)
+{
+ PathInfo *info;
+
+ info = g_slice_new (PathInfo);
+ info->module = module;
+ info->path = g_strdup (path);
+
+ return info;
+}
+
+static void
+path_info_free (PathInfo *info)
+{
+ g_free (info->path);
+ g_slice_free (PathInfo, info);
+}
+
+static void
+tracker_indexer_finalize (GObject *object)
{
- GMainLoop *main_loop;
- GOptionContext *context;
- GError *error = NULL;
- gchar *summary = NULL;
- gchar *example;
-
- g_type_init ();
-
- if (!g_thread_supported ())
- g_thread_init (NULL);
-
- setlocale (LC_ALL, "");
-
- bindtextdomain (GETTEXT_PACKAGE, LOCALEDIR);
- bind_textdomain_codeset (GETTEXT_PACKAGE, "UTF-8");
- textdomain (GETTEXT_PACKAGE);
-
- /* Set timezone info */
- tzset ();
-
- /* Translators: this messagge will apper immediately after the
- * usage string - Usage: COMMAND <THIS_MESSAGE>
- */
- context = g_option_context_new (_("- start the tracker daemon"));
- example = g_strconcat ("-i ", _("DIRECTORY"),
- "-i ", _("DIRECTORY"),
- "-e ", _("DIRECTORY"),
- "-e ", _("DIRECTORY"),
- NULL);
-
-#ifdef HAVE_RECENT_GLIB
- /* Translators: this message will appear after the usage
- * string and before the list of options, showing an usage
- * example.
- */
- summary = g_strdup_printf (_("To include or exclude multiple directories "
- "at the same time, join multiple options like:\n"
- "\n"
- "\t%s"),
- example);
- g_option_context_set_summary (context, summary);
-#endif /* HAVE_RECENT_GLIB */
-
- g_option_context_add_main_entries (context, entries, NULL);
- g_option_context_parse (context, &argc, &argv, &error);
- g_option_context_free (context);
- g_free (summary);
- g_free (example);
-
- g_print ("\n"
- COPYRIGHT "\n"
- "\n"
- WARRANTY "\n"
- "\n");
+ TrackerIndexerPrivate *priv;
+
+ priv = TRACKER_INDEXER_GET_PRIVATE (object);
+
+ g_queue_foreach (priv->dir_queue, (GFunc) path_info_free, NULL);
+ g_queue_free (priv->dir_queue);
+
+ g_queue_foreach (priv->file_process_queue, (GFunc) path_info_free, NULL);
+ g_queue_free (priv->file_process_queue);
+
+ g_slist_foreach (priv->module_names, (GFunc) g_free, NULL);
+ g_slist_free (priv->module_names);
+
+ g_hash_table_destroy (priv->indexer_modules);
+
+ g_object_unref (priv->config);
+
+ G_OBJECT_CLASS (tracker_indexer_parent_class)->finalize (object);
+}
+
+static void
+tracker_indexer_set_property (GObject *object,
+ guint prop_id,
+ const GValue *value,
+ GParamSpec *pspec)
+{
+ TrackerIndexer *indexer;
+ TrackerIndexerPrivate *priv;
+
+ indexer = TRACKER_INDEXER (object);
+ priv = TRACKER_INDEXER_GET_PRIVATE (object);
+
+ switch (prop_id) {
+ case PROP_RUNNING:
+ tracker_indexer_set_running (indexer, g_value_get_boolean (value));
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+ }
+}
+
+static void
+tracker_indexer_get_property (GObject *object,
+ guint prop_id,
+ GValue *value,
+ GParamSpec *pspec)
+{
+ TrackerIndexerPrivate *priv;
+
+ priv = TRACKER_INDEXER_GET_PRIVATE (object);
+
+ switch (prop_id) {
+ case PROP_RUNNING:
+ g_value_set_boolean (value, (priv->idle_id != 0));
+ break;
+ default:
+ G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+ }
+}
+
+static void
+tracker_indexer_class_init (TrackerIndexerClass *class)
+{
+ GObjectClass *object_class = G_OBJECT_CLASS (class);
+
+ object_class->finalize = tracker_indexer_finalize;
+ object_class->set_property = tracker_indexer_set_property;
+ object_class->get_property = tracker_indexer_get_property;
+
+ signals [FINISHED] = g_signal_new ("finished",
+ G_OBJECT_CLASS_TYPE (object_class),
+ G_SIGNAL_RUN_LAST,
+ G_STRUCT_OFFSET (TrackerIndexerClass, finished),
+ NULL, NULL,
+ g_cclosure_marshal_VOID__VOID,
+ G_TYPE_NONE, 0);
+
+ g_object_class_install_property (object_class,
+ PROP_RUNNING,
+ g_param_spec_boolean ("running",
+ "Running",
+ "Whether the indexer is running",
+ TRUE,
+ G_PARAM_READWRITE));
+
+ g_type_class_add_private (object_class,
+ sizeof (TrackerIndexerPrivate));
+}
+
+static gboolean
+init_indexer (TrackerIndexer *indexer)
+{
+ tracker_indexer_set_running (indexer, TRUE);
+ return FALSE;
+}
+
+TrackerDBInterface *
+create_db_interface (const gchar *filename)
+{
+#if 0
+ TrackerDBInterface *interface;
+ gchar *path;
+
+ path = g_build_filename (g_get_user_cache_dir (),
+ "tracker",
+ filename,
+ NULL);
+
+ interface = tracker_db_interface_sqlite_new (path);
+ g_free (path);
+
+ return interface;
+#endif
+ return NULL;
+}
+
+static void
+tracker_indexer_init (TrackerIndexer *indexer)
+{
+ TrackerIndexerPrivate *priv;
+ gint initial_sleep;
+ GSList *m;
+
+ priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+ priv->dir_queue = g_queue_new ();
+ priv->file_process_queue = g_queue_new ();
+ priv->config = tracker_config_new ();
+
+ priv->index = create_db_interface ("file-meta.db");
- g_print ("Initialising...\n");
+ priv->module_names = tracker_config_get_index_modules (priv->config);
+
+ priv->indexer_modules = g_hash_table_new_full (g_direct_hash,
+ g_direct_equal,
+ NULL,
+ (GDestroyNotify) g_module_close);
+
+ for (m = priv->module_names; m; m = m->next) {
+ GModule *module;
+
+ module = tracker_indexer_module_load (m->data);
+
+ if (module) {
+ g_hash_table_insert (priv->indexer_modules,
+ m->data, module);
+ }
+ }
+
+ initial_sleep = tracker_config_get_initial_sleep (priv->config);
+ g_timeout_add (initial_sleep * 1000, (GSourceFunc) init_indexer, indexer);
+}
+
+static void
+tracker_indexer_add_file (TrackerIndexer *indexer,
+ PathInfo *info)
+{
+ TrackerIndexerPrivate *priv;
+
+ g_return_if_fail (info != NULL);
+
+ priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+ g_queue_push_tail (priv->file_process_queue, info);
+}
+
+static void
+tracker_indexer_add_directory (TrackerIndexer *indexer,
+ PathInfo *info)
+{
+ TrackerIndexerPrivate *priv;
+
+ g_return_if_fail (info != NULL);
+
+ priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+ g_queue_push_tail (priv->dir_queue, info);
+}
+
+static void
+process_file (TrackerIndexer *indexer,
+ PathInfo *info)
+{
+ GHashTable *metadata;
+
+ g_message ("Processing file: %s\n", info->path);
+
+ metadata = tracker_indexer_module_get_file_metadata (info->module, info->path);
+
+ if (metadata) {
+ /* FIXME: store metadata in DB */
+ GList *keys, *k;
+
+ keys = g_hash_table_get_keys (metadata);
+
+ for (k = keys; k; k = k->next) {
+ g_print (" %s = %s\n",
+ (gchar *) k->data,
+ (gchar *) g_hash_table_lookup (metadata, k->data));
+ }
+
+ g_hash_table_destroy (metadata);
+ g_list_free (keys);
+ }
+}
+
+static void
+process_directory (TrackerIndexer *indexer,
+ PathInfo *info,
+ gboolean recurse)
+{
+ const gchar *name;
+ GDir *dir;
+
+ g_message ("Processing directory: %s\n", info->path);
+
+ dir = g_dir_open (info->path, 0, NULL);
+
+ if (!dir) {
+ return;
+ }
+
+ while ((name = g_dir_read_name (dir)) != NULL) {
+ PathInfo *new_info;
+ gchar *path;
+
+ path = g_build_filename (info->path, name, NULL);
+
+ new_info = path_info_new (info->module, path);
+ tracker_indexer_add_file (indexer, new_info);
+
+ if (recurse && g_file_test (path, G_FILE_TEST_IS_DIR)) {
+ new_info = path_info_new (info->module, path);
+ tracker_indexer_add_directory (indexer, new_info);
+ }
+
+ g_free (path);
+ }
+
+ g_dir_close (dir);
+}
+
+static void
+process_module (TrackerIndexer *indexer,
+ const gchar *module_name)
+{
+ TrackerIndexerPrivate *priv;
+ GModule *module;
+ gchar **dirs;
+ gint i;
+
+ g_message ("Starting module: %s\n", module_name);
+
+ priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+ module = g_hash_table_lookup (priv->indexer_modules, module_name);
+ dirs = tracker_indexer_module_get_directories (module);
+
+ g_return_if_fail (dirs != NULL);
+
+ for (i = 0; dirs[i]; i++) {
+ PathInfo *info;
+
+ info = path_info_new (module, dirs[i]);
+ tracker_indexer_add_directory (indexer, info);
+ }
+
+ g_strfreev (dirs);
+}
+
+static gboolean
+indexing_func (gpointer data)
+{
+ TrackerIndexer *indexer;
+ TrackerIndexerPrivate *priv;
+ PathInfo *path;
+
+ indexer = (TrackerIndexer *) data;
+ priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+ if ((path = g_queue_pop_head (priv->file_process_queue)) != NULL) {
+ /* process file */
+ process_file (indexer, path);
+ path_info_free (path);
+ } else if ((path = g_queue_pop_head (priv->dir_queue)) != NULL) {
+ /* process directory contents */
+ process_directory (indexer, path, TRUE);
+ path_info_free (path);
+ } else {
+ /* dirs/files queues are empty, process the next module */
+ if (!priv->current_module) {
+ priv->current_module = priv->module_names;
+ } else {
+ priv->current_module = priv->current_module->next;
+ }
+
+ if (!priv->current_module) {
+ /* no more modules to query, we're done */
+ g_signal_emit (indexer, signals[FINISHED], 0);
+ return FALSE;
+ }
+
+ process_module (indexer, priv->current_module->data);
+ }
+
+ return TRUE;
+}
+
+TrackerIndexer *
+tracker_indexer_new (void)
+{
+ return g_object_new (TRACKER_TYPE_INDEXER, NULL);
+}
+
+void
+tracker_indexer_set_running (TrackerIndexer *indexer,
+ gboolean running)
+{
+ TrackerIndexerPrivate *priv;
+ gboolean changed = FALSE;
+
+ g_return_if_fail (TRACKER_IS_INDEXER (indexer));
+
+ priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+ if (running && priv->idle_id == 0) {
+ priv->idle_id = g_idle_add ((GSourceFunc) indexing_func, indexer);
+ changed = TRUE;
+ } else if (!running && priv->idle_id != 0) {
+ g_source_remove (priv->idle_id);
+ priv->idle_id = 0;
+ changed = TRUE;
+ }
+
+ if (changed) {
+ g_object_notify (G_OBJECT (indexer), "running");
+ }
+}
+
+gboolean
+tracker_indexer_get_running (TrackerIndexer *indexer)
+{
+ TrackerIndexerPrivate *priv;
- main_loop = g_main_loop_new (NULL, FALSE);
- g_main_loop_run (main_loop);
+ g_return_val_if_fail (TRACKER_IS_INDEXER (indexer), FALSE);
- g_print ("Shutting down...\n");
+ priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
- return EXIT_SUCCESS;
+ return (priv->idle_id == 0);
}
Modified: branches/xesam-support/src/trackerd/Makefile.am
==============================================================================
--- branches/xesam-support/src/trackerd/Makefile.am (original)
+++ branches/xesam-support/src/trackerd/Makefile.am Tue May 13 13:47:16 2008
@@ -38,20 +38,6 @@
tracker-hal.h
endif
-db_sources = \
- tracker-indexer.c \
- tracker-indexer.h \
- tracker-cache.c \
- tracker-cache.h \
- tracker-parser.c \
- tracker-parser.h \
- tracker-db-sqlite.c \
- tracker-db-sqlite.h \
- tracker-db-manager.c \
- tracker-db-manager.h \
- tracker-db-email.c \
- tracker-db-email.h
-
mail_modulesdir = $(libdir)/tracker/mail-modules
mail_modules_LTLIBRARIES = \
libemail-evolution.la \
@@ -85,11 +71,18 @@
$(win_sources) \
$(unix_sources) \
$(hal_sources) \
- $(db_sources) \
tracker-apps.c \
tracker-apps.h \
+ tracker-cache.c \
+ tracker-cache.h \
tracker-db.c \
tracker-db.h \
+ tracker-db-email.c \
+ tracker-db-email.h \
+ tracker-db-sqlite.c \
+ tracker-db-sqlite.h \
+ tracker-db-manager.c \
+ tracker-db-manager.h \
tracker-dbus.c \
tracker-dbus.h \
tracker-dbus-daemon.c \
@@ -108,11 +101,17 @@
tracker-email.h \
tracker-email-utils.c \
tracker-email-utils.h \
+ tracker-indexer.c \
+ tracker-indexer.h \
+ tracker-index-stage.c \
+ tracker-index-stage.h \
tracker-main.c \
tracker-main.h \
tracker-marshal-main.c \
tracker-metadata.c \
tracker-metadata.h \
+ tracker-parser.c \
+ tracker-parser.h \
tracker-process-files.c \
tracker-process-files.h \
tracker-query-tree.c \
Modified: branches/xesam-support/src/trackerd/tracker-cache.c
==============================================================================
--- branches/xesam-support/src/trackerd/tracker-cache.c (original)
+++ branches/xesam-support/src/trackerd/tracker-cache.c Tue May 13 13:47:16 2008
@@ -379,7 +379,7 @@
sleep = TRUE;
}
- if (tracker->index_status > INDEX_APPLICATIONS &&
+ if (tracker_index_stage_get () > TRACKER_INDEX_STAGE_APPLICATIONS &&
tracker_should_pause ()) {
if (db_con) {
stopped_trans = TRUE;
Modified: branches/xesam-support/src/trackerd/tracker-main.c
==============================================================================
--- branches/xesam-support/src/trackerd/tracker-main.c (original)
+++ branches/xesam-support/src/trackerd/tracker-main.c Tue May 13 13:47:16 2008
@@ -127,8 +127,11 @@
DBConnection *main_thread_db_con;
DBConnection *main_thread_cache_con;
+/* Private */
static GMainLoop *main_loop;
+static gchar *log_filename;
+/* Private command line parameters */
static gchar **no_watch_dirs;
static gchar **watch_dirs;
static gchar **crawl_dirs;
@@ -456,6 +459,7 @@
{
gchar *str;
+ /* Public locations */
str = g_strdup_printf ("Tracker-%s.%d", g_get_user_name (), getpid ());
tracker->sys_tmp_root_dir = g_build_filename (g_get_tmp_dir (), str, NULL);
g_free (str);
@@ -467,7 +471,9 @@
tracker->xesam_dir = g_build_filename (g_get_home_dir (), ".xesam", NULL);
tracker->email_attachments_dir = g_build_filename (tracker->sys_tmp_root_dir, "Attachments", NULL);
- tracker->log_filename = g_build_filename (tracker->root_dir, "tracker.log", NULL);
+
+ /* Private locations */
+ log_filename = g_build_filename (tracker->root_dir, "tracker.log", NULL);
}
static void
@@ -509,7 +515,7 @@
g_mkdir_with_parents (tracker->email_attachments_dir, 00700);
/* Remove existing log files */
- tracker_file_unlink (tracker->log_filename);
+ tracker_file_unlink (log_filename);
}
static void
@@ -527,31 +533,10 @@
static void
initialise_defaults (void)
{
- tracker->grace_period = 0;
-
- tracker->reindex = FALSE;
- tracker->in_merge = FALSE;
-
- tracker->index_status = INDEX_CONFIG;
-
- tracker->black_list_timer_active = FALSE;
-
- tracker->pause_manual = FALSE;
- tracker->pause_battery = FALSE;
- tracker->pause_io = FALSE;
-
- tracker->watch_limit = 0;
- tracker->index_count = 0;
-
tracker->max_process_queue_size = MAX_PROCESS_QUEUE_SIZE;
tracker->max_extract_queue_size = MAX_EXTRACT_QUEUE_SIZE;
tracker->index_number_min_length = 6;
-
- tracker->folders_count = 0;
- tracker->folders_processed = 0;
- tracker->mbox_count = 0;
- tracker->folders_processed = 0;
}
static void
@@ -787,6 +772,22 @@
}
static void
+shutdown_locations (void)
+{
+ /* Public locations */
+ g_free (tracker->data_dir);
+ g_free (tracker->config_dir);
+ g_free (tracker->root_dir);
+ g_free (tracker->user_data_dir);
+ g_free (tracker->sys_tmp_root_dir);
+ g_free (tracker->email_attachments_dir);
+ g_free (tracker->xesam_dir);
+
+ /* Private locations */
+ g_free (log_filename);
+}
+
+static void
shutdown_directories (void)
{
/* If we are reindexing, just remove the databases */
@@ -799,15 +800,6 @@
if (tracker->sys_tmp_root_dir) {
tracker_dir_remove (tracker->sys_tmp_root_dir);
}
-
- g_free (tracker->data_dir);
- g_free (tracker->config_dir);
- g_free (tracker->root_dir);
- g_free (tracker->user_data_dir);
- g_free (tracker->sys_tmp_root_dir);
- g_free (tracker->email_attachments_dir);
- g_free (tracker->xesam_dir);
- g_free (tracker->log_filename);
}
gint
@@ -895,7 +887,7 @@
tracker->config = tracker_config_new ();
tracker->language = tracker_language_new (tracker->config);
- tracker_log_init (tracker->log_filename,
+ tracker_log_init (log_filename,
tracker_config_get_verbosity (tracker->config),
fatal_errors);
tracker_nfs_lock_init (tracker->root_dir,
@@ -1053,6 +1045,7 @@
shutdown_indexer ();
shutdown_databases ();
shutdown_directories ();
+ shutdown_locations ();
/* Clean up other struct members */
if (tracker->file_process_queue) {
Modified: branches/xesam-support/src/trackerd/tracker-main.h
==============================================================================
--- branches/xesam-support/src/trackerd/tracker-main.h (original)
+++ branches/xesam-support/src/trackerd/tracker-main.h Tue May 13 13:47:16 2008
@@ -29,10 +29,14 @@
#include <glib.h>
+#include <libtracker-common/tracker-config.h>
+#include <libtracker-common/tracker-language.h>
#include <libtracker-db/tracker-db-action.h>
#include "tracker-parser.h"
#include "tracker-indexer.h"
+#include "tracker-index-stage.h"
+#include "tracker-hal.h"
/* default performance options */
#define MAX_PROCESS_QUEUE_SIZE 100
@@ -40,123 +44,101 @@
G_BEGIN_DECLS
-typedef enum {
- INDEX_CONFIG,
- INDEX_APPLICATIONS,
- INDEX_FILES,
- INDEX_WEBHISTORY,
- INDEX_CRAWL_FILES,
- INDEX_CONVERSATIONS,
- INDEX_EXTERNAL,
- INDEX_EMAILS,
- INDEX_FINISHED
-} IndexStatus;
-
typedef struct {
- gboolean is_running;
- gboolean readonly;
+ gboolean is_running;
+ gboolean readonly;
+
+ gint pid;
- gint pid;
+ gboolean reindex;
- gpointer hal;
- gboolean reindex;
- gpointer config;
- gpointer language;
+ TrackerHal *hal;
+ TrackerConfig *config;
+ TrackerLanguage *language;
/* Config options */
- guint32 watch_limit;
- gpointer index_db;
+ guint32 watch_limit;
+ gpointer index_db;
/* Data directories */
- gchar *data_dir;
- gchar *config_dir;
- gchar *root_dir;
- gchar *user_data_dir;
- gchar *sys_tmp_root_dir;
- gchar *email_attachments_dir;
- gchar *xesam_dir;
-
- gchar *log_filename;
+ gchar *data_dir;
+ gchar *config_dir;
+ gchar *root_dir;
+ gchar *user_data_dir;
+ gchar *sys_tmp_root_dir;
+ gchar *email_attachments_dir;
+ gchar *xesam_dir;
/* Performance and memory usage options */
- gint max_process_queue_size;
- gint max_extract_queue_size;
- gint memory_limit;
-
+ gint max_process_queue_size;
+ gint max_extract_queue_size;
+ gint memory_limit;
+
/* Pause/shutdown */
- gboolean shutdown;
- gboolean pause_manual;
- gboolean pause_battery;
- gboolean pause_io;
+ gboolean shutdown;
+ gboolean pause_manual;
+ gboolean pause_battery;
+ gboolean pause_io;
/* Indexing options */
- Indexer *file_index;
- Indexer *file_update_index;
- Indexer *email_index;
+ Indexer *file_index;
+ Indexer *file_update_index;
+ Indexer *email_index;
/* Table of stop words that are to be ignored by the parser */
- GHashTable *stop_words;
- gint index_number_min_length;
+ GHashTable *stop_words;
+ gint index_number_min_length;
- gboolean first_time_index;
+ gboolean first_time_index;
- time_t index_time_start;
- gint folders_count;
- gint folders_processed;
- gint mbox_count;
- gint mbox_processed;
-
- IndexStatus index_status;
+ time_t index_time_start;
+ gint folders_count;
+ gint folders_processed;
+ gint mbox_count;
+ gint mbox_processed;
- gint grace_period;
- gboolean request_waiting;
+ gint grace_period;
+ gboolean request_waiting;
/* Lookup tables for service and metadata IDs */
- GHashTable *metadata_table;
+ GHashTable *metadata_table;
/* Email config options */
- gint email_service_min;
- gint email_service_max;
+ gint email_service_min;
+ gint email_service_max;
/* Queue for recorad file changes */
- GQueue *file_change_queue;
- gboolean black_list_timer_active;
+ GQueue *file_change_queue;
+ gboolean black_list_timer_active;
/* Progress info for merges */
- gboolean in_merge;
- gint merge_count;
- gint merge_processed;
+ gboolean in_merge;
+ gint merge_count;
+ gint merge_processed;
/* Application run time values */
- gint index_count;
+ gint index_count;
- gint word_detail_count;
- gint word_count;
- gint word_update_count;
+ gint word_detail_count;
+ gint word_count;
+ gint word_update_count;
- GAsyncQueue *file_process_queue;
- GAsyncQueue *file_metadata_queue;
- GAsyncQueue *dir_queue;
+ GAsyncQueue *file_process_queue;
+ GAsyncQueue *file_metadata_queue;
+ GAsyncQueue *dir_queue;
- GMutex *files_check_mutex;
- GMutex *files_signal_mutex;
- GCond *files_signal_cond;
+ GMutex *files_check_mutex;
+ GMutex *files_signal_mutex;
+ GCond *files_signal_cond;
- GMutex *metadata_check_mutex;
- GMutex *metadata_signal_mutex;
- GCond *metadata_signal_cond;
+ GMutex *metadata_check_mutex;
+ GMutex *metadata_signal_mutex;
+ GCond *metadata_signal_cond;
- GHashTable *xesam_sessions;
+ GHashTable *xesam_sessions;
} Tracker;
-void tracker_shutdown (void);
-GSList * tracker_get_watch_root_dirs (void);
-gboolean tracker_spawn (gchar **argv,
- gint timeout,
- gchar **tmp_stdout,
- gint *exit_status);
-gboolean tracker_watch_dir (const gchar *uri);
-void tracker_scan_directory (const gchar *uri);
+void tracker_shutdown (void);
G_END_DECLS
Modified: branches/xesam-support/src/trackerd/tracker-metadata.c
==============================================================================
--- branches/xesam-support/src/trackerd/tracker-metadata.c (original)
+++ branches/xesam-support/src/trackerd/tracker-metadata.c Tue May 13 13:47:16 2008
@@ -33,6 +33,7 @@
#include <libtracker-common/tracker-log.h>
#include <libtracker-common/tracker-file-utils.h>
+#include <libtracker-common/tracker-os-dependant.h>
#include "tracker-metadata.h"
#include "tracker-utils.h"
Modified: branches/xesam-support/src/trackerd/tracker-process-files.c
==============================================================================
--- branches/xesam-support/src/trackerd/tracker-process-files.c (original)
+++ branches/xesam-support/src/trackerd/tracker-process-files.c Tue May 13 13:47:16 2008
@@ -636,7 +636,7 @@
process_queue_files_foreach (uri, tracker);
- if (tracker->index_status != INDEX_EMAILS) {
+ if (tracker_index_stage_get () != TRACKER_INDEX_STAGE_EMAILS) {
tracker->folders_processed++;
}
}
@@ -1035,7 +1035,7 @@
return;
}
- tracker->index_status = INDEX_EMAILS;
+ tracker_index_stage_set (TRACKER_INDEX_STAGE_EMAILS);
/* Signal progress */
daemon = tracker_dbus_get_object (TRACKER_TYPE_DBUS_DAEMON);
@@ -1077,8 +1077,9 @@
static gboolean
process_files (Tracker *tracker)
{
- DBConnection *db_con;
- GObject *object;
+ DBConnection *db_con;
+ GObject *object;
+ TrackerIndexStage stage;
db_con = tracker->index_db;
object = tracker_dbus_get_object (TRACKER_TYPE_DBUS_DAEMON);
@@ -1098,46 +1099,49 @@
}
}
- if (tracker->index_status != INDEX_FINISHED) {
+ stage = tracker_index_stage_get ();
+
+ if (stage != TRACKER_INDEX_STAGE_FINISHED) {
g_mutex_unlock (tracker->files_check_mutex);
- switch (tracker->index_status) {
- case INDEX_CONFIG:
+ switch (stage) {
+ case TRACKER_INDEX_STAGE_CONFIG:
process_index_config (tracker);
break;
- case INDEX_APPLICATIONS:
+ case TRACKER_INDEX_STAGE_APPLICATIONS:
process_index_applications (tracker);
break;
- case INDEX_FILES:
+ case TRACKER_INDEX_STAGE_FILES:
process_index_files (tracker);
break;
- case INDEX_CRAWL_FILES:
+ case TRACKER_INDEX_STAGE_CRAWL_FILES:
process_index_crawl_files (tracker);
break;
- case INDEX_CONVERSATIONS:
+ case TRACKER_INDEX_STAGE_CONVERSATIONS:
process_index_conversations (tracker);
break;
- case INDEX_WEBHISTORY:
+ case TRACKER_INDEX_STAGE_WEBHISTORY:
process_index_webhistory (tracker);
break;
- case INDEX_EXTERNAL:
+ case TRACKER_INDEX_STAGE_EXTERNAL:
break;
- case INDEX_EMAILS:
+ case TRACKER_INDEX_STAGE_EMAILS:
process_index_emails (tracker);
break;
- case INDEX_FINISHED:
+ case TRACKER_INDEX_STAGE_FINISHED:
break;
}
- tracker->index_status++;
+ tracker_index_stage_set (++stage);
+
return TRUE;
}
@@ -1162,7 +1166,7 @@
return FALSE;
}
- tracker->index_status = INDEX_FILES;
+ tracker_index_stage_set (TRACKER_INDEX_STAGE_FILES);
/* Signal progress */
g_signal_emit_by_name (object,
@@ -1173,7 +1177,7 @@
tracker->folders_processed,
tracker->folders_count);
- tracker->index_status = INDEX_FINISHED;
+ tracker_index_stage_set (TRACKER_INDEX_STAGE_FINISHED);
if (tracker->is_running && tracker->first_time_index) {
gint time_taken;
@@ -1271,7 +1275,7 @@
tracker_process_files_should_be_watched (tracker->config, info->uri)) {
g_async_queue_push (tracker->dir_queue, g_strdup (info->uri));
- if (tracker->index_status != INDEX_EMAILS) {
+ if (tracker_index_stage_get () != TRACKER_INDEX_STAGE_EMAILS) {
tracker->folders_count++;
}
}
@@ -1527,9 +1531,7 @@
tracker);
/* Start processing */
- g_print ("oooye!\n");
g_mutex_unlock (tracker->files_signal_mutex);
- g_print ("oooye2!\n");
/* Signal state change */
g_signal_emit_by_name (object,
@@ -1559,7 +1561,7 @@
tracker_log ("Proceeding with indexing...");
- tracker->index_status = INDEX_CONFIG;
+ tracker_index_stage_set (TRACKER_INDEX_STAGE_CONFIG);
object = tracker_dbus_get_object (TRACKER_TYPE_DBUS_DAEMON);
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
