tracker r1421 - in branches/indexer-split: . src/trackerd
- From: mr svn gnome org
- To: svn-commits-list gnome org
- Subject: tracker r1421 - in branches/indexer-split: . src/trackerd
- Date: Wed, 14 May 2008 11:10:31 +0100 (BST)
Author: mr
Date: Wed May 14 10:10:31 2008
New Revision: 1421
URL: http://svn.gnome.org/viewvc/tracker?rev=1421&view=rev
Log:
* src/trackerd/tracker-db-email.c:
* src/trackerd/tracker-db-sqlite.c:
* src/trackerd/tracker-db.c:
* src/trackerd/tracker-dbus-daemon.c: Changed the "config"
property to be an object spec instead of a pointer spec. Also
fixed the way we set the object.
* src/trackerd/tracker-dbus-xesam.c:
* src/trackerd/tracker-xesam-live-search.c:
* src/trackerd/tracker-xesam-session.h: Fix includes.
* src/trackerd/tracker-main.[ch]: Removed the "stop_words" member
of the Tracker struct, it is not used. Also removed the
"index_number_min_length" member and replaced the one time it is
used for a #define in tracker-parser.c.
Modified:
branches/indexer-split/ChangeLog
branches/indexer-split/src/trackerd/tracker-db-email.c
branches/indexer-split/src/trackerd/tracker-db-sqlite.c
branches/indexer-split/src/trackerd/tracker-db.c
branches/indexer-split/src/trackerd/tracker-dbus-daemon.c
branches/indexer-split/src/trackerd/tracker-dbus-search.c
branches/indexer-split/src/trackerd/tracker-dbus-search.h
branches/indexer-split/src/trackerd/tracker-dbus-xesam.c
branches/indexer-split/src/trackerd/tracker-dbus.c
branches/indexer-split/src/trackerd/tracker-dbus.h
branches/indexer-split/src/trackerd/tracker-main.c
branches/indexer-split/src/trackerd/tracker-main.h
branches/indexer-split/src/trackerd/tracker-parser.c
branches/indexer-split/src/trackerd/tracker-parser.h
branches/indexer-split/src/trackerd/tracker-query-tree.c
branches/indexer-split/src/trackerd/tracker-query-tree.h
branches/indexer-split/src/trackerd/tracker-xesam-live-search.c
branches/indexer-split/src/trackerd/tracker-xesam-session.c
branches/indexer-split/src/trackerd/tracker-xesam-session.h
branches/indexer-split/src/trackerd/tracker-xesam.c
branches/indexer-split/src/trackerd/tracker-xesam.h
Modified: branches/indexer-split/src/trackerd/tracker-db-email.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-db-email.c (original)
+++ branches/indexer-split/src/trackerd/tracker-db-email.c Wed May 14 10:10:31 2008
@@ -666,9 +666,7 @@
}
tracker_db_update_indexes_for_new_service (id, type_id, index_table);
-
- tracker_word_table_free (index_table);
-
+ tracker_parser_text_free (index_table);
g_free (str_id);
Modified: branches/indexer-split/src/trackerd/tracker-db-sqlite.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-db-sqlite.c (original)
+++ branches/indexer-split/src/trackerd/tracker-db-sqlite.c Wed May 14 10:10:31 2008
@@ -18,19 +18,12 @@
* Boston, MA 02110-1301, USA.
*/
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE
-#endif
-
#include "config.h"
#include <fcntl.h>
#include <regex.h>
#include <zlib.h>
-
-#ifdef OS_WIN32
-#include "mingw-compat.h"
-#endif
+#include <string.h>
#include <glib.h>
#include <glib/gstdio.h>
@@ -1585,7 +1578,12 @@
gchar *st;
tracker_db_result_set_get (result_set, 0, &st, -1);
- old_table = tracker_parse_text (old_table, st, 1, TRUE, FALSE);
+ old_table = tracker_parser_text (old_table, st, 1,
+ tracker->language,
+ tracker_config_get_max_words_to_index (tracker->config),
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ TRUE, FALSE);
valid = tracker_db_result_set_iter_next (result_set);
g_free (st);
@@ -1622,7 +1620,7 @@
1, &weight,
-1);
- table = tracker_parse_text_fast (table, value, weight);
+ table = tracker_parser_text_fast (table, value, weight);
g_free (value);
valid = tracker_db_result_set_iter_next (result_set);
@@ -1651,7 +1649,12 @@
3, &delimited
-1);
- table = tracker_parse_text (table, value, weight, filtered, delimited);
+ table = tracker_parser_text (table, value, weight,
+ tracker->language,
+ tracker_config_get_max_words_to_index (tracker->config),
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ filtered, delimited);
valid = tracker_db_result_set_iter_next (result_set);
g_free (value);
}
@@ -1825,9 +1828,19 @@
}
if (use_buffer) {
- index_table = tracker_parse_text (index_table, buffer, 1, TRUE, FALSE);
+ index_table = tracker_parser_text (index_table, buffer, 1,
+ tracker->language,
+ tracker_config_get_max_words_to_index (tracker->config),
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ TRUE, FALSE);
} else {
- index_table = tracker_parse_text (index_table, value, 1, TRUE, FALSE);
+ index_table = tracker_parser_text (index_table, value, 1,
+ tracker->language,
+ tracker_config_get_max_words_to_index (tracker->config),
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ TRUE, FALSE);
}
strm.avail_in = buffer_length;
@@ -1918,7 +1931,10 @@
GSList *duds = NULL;
guint i = 0;
- array = tracker_parse_text_into_array (search_string);
+ array = tracker_parser_text_into_array (search_string,
+ tracker->language,
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config));
result_set = tracker_exec_proc (db_con->common, "GetRelatedServiceIDs", service, service, NULL);
@@ -1942,7 +1958,11 @@
g_object_unref (result_set);
}
- tree = tracker_query_tree_new (search_string, db_con->word_index, services);
+ tree = tracker_query_tree_new (search_string,
+ db_con->word_index,
+ tracker->config,
+ tracker->language,
+ services);
hits = tracker_query_tree_get_hits (tree, offset, limit);
result = NULL;
@@ -2214,24 +2234,36 @@
new_table = NULL;
if (old_value) {
- old_table = tracker_parse_text (old_table,
- old_value,
- def->weight,
- def->filtered,
- def->delimited);
+ old_table = tracker_parser_text (old_table,
+ old_value,
+ def->weight,
+ tracker->language,
+ tracker_config_get_max_words_to_index (tracker->config),
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ def->filtered,
+ def->delimited);
}
/* parse new metadata value */
if (new_value) {
- new_table = tracker_parse_text (new_table, new_value, def->weight, def->filtered, def->delimited);
+ new_table = tracker_parser_text (new_table,
+ new_value,
+ def->weight,
+ tracker->language,
+ tracker_config_get_max_words_to_index (tracker->config),
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ def->filtered,
+ def->delimited);
}
/* we only do differential updates so only changed words scores are updated */
sid = tracker_service_manager_get_id_for_service (service);
tracker_db_update_differential_index (old_table, new_table, id, sid);
- tracker_word_table_free (old_table);
- tracker_word_table_free (new_table);
+ tracker_parser_text_free (old_table);
+ tracker_parser_text_free (new_table);
}
@@ -2367,9 +2399,16 @@
}
if (table) {
- gchar *mvalue = tracker_parse_text_to_string (values[i], FALSE, FALSE, FALSE);
+ gchar *mvalue;
- table = tracker_parse_text_fast (table, mvalue, def->weight);
+ mvalue = tracker_parser_text_to_string (values[i],
+ tracker->language,
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ FALSE,
+ FALSE,
+ FALSE);
+ table = tracker_parser_text_fast (table, mvalue, def->weight);
g_free (mvalue);
}
@@ -2388,10 +2427,16 @@
continue;
}
- mvalue = tracker_parse_text_to_string (values[i], def->filtered, def->filtered, def->delimited);
+ mvalue = tracker_parser_text_to_string (values[i],
+ tracker->language,
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ def->filtered,
+ def->filtered,
+ def->delimited);
if (table) {
- table = tracker_parse_text_fast (table, mvalue, def->weight);
+ table = tracker_parser_text_fast (table, mvalue, def->weight);
}
tracker_exec_proc (db_con, "SetMetadata", id, def->id, mvalue, values[i], NULL);
@@ -2409,7 +2454,15 @@
}
if (table) {
- table = tracker_parse_text (table, values[i], def->weight, def->filtered, def->delimited);
+ table = tracker_parser_text (table,
+ values[i],
+ def->weight,
+ tracker->language,
+ tracker_config_get_max_words_to_index (tracker->config),
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ def->filtered,
+ def->delimited);
}
save_full_text (db_con->blob, id, values[i], strlen (values[i]));
@@ -2432,12 +2485,19 @@
case DATA_STRING: {
gint i;
for (i = 0; i < length; i++) {
+ gchar *mvalue;
+
if (!values[i]) {
continue;
}
- gchar *mvalue = tracker_parse_text_to_string (values[i], def->filtered, def->filtered, def->delimited);
-
+ mvalue = tracker_parser_text_to_string (values[i],
+ tracker->language,
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ def->filtered,
+ def->filtered,
+ def->delimited);
tracker_exec_proc (db_con, "SetMetadata", id, def->id, mvalue, values[i], NULL);
g_free (mvalue);
@@ -2677,9 +2737,12 @@
case DATA_INDEX:
- for (i=0; i<length; i++) {
+ for (i = 0; i < length; i++) {
+ gchar *mvalue;
- if (!values[i] || !values[i][0]) continue;
+ if (!values[i] || !values[i][0]) {
+ continue;
+ }
if (str) {
g_string_append_printf (str, " %s ", values[i]);
@@ -2694,9 +2757,13 @@
backup_non_embedded_metadata (db_con, id, def->id, values[i]);
}
-
- char *mvalue = tracker_parse_text_to_string (values[i], def->filtered, def->filtered, def->delimited);
-
+ mvalue = tracker_parser_text_to_string (values[i],
+ tracker->language,
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ def->filtered,
+ def->filtered,
+ def->delimited);
tracker_exec_proc (db_con, "SetMetadata", id, def->id, mvalue, values[i], NULL);
g_free (mvalue);
@@ -2719,9 +2786,12 @@
case DATA_STRING:
- for (i=0; i<length; i++) {
+ for (i = 0; i < length; i++) {
+ gchar *mvalue;
- if (!values[i] || !values[i][0]) continue;
+ if (!values[i] || !values[i][0]) {
+ continue;
+ }
/* backup non-embedded data for embedded services */
if (do_backup &&
@@ -2730,8 +2800,13 @@
backup_non_embedded_metadata (db_con, id, def->id, values[i]);
}
- char *mvalue = tracker_parse_text_to_string (values[i], def->filtered, def->filtered, def->delimited);
-
+ mvalue = tracker_parser_text_to_string (values[i],
+ tracker->language,
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ def->filtered,
+ def->filtered,
+ def->delimited);
tracker_exec_proc (db_con, "SetMetadata", id, def->id, mvalue, values[i], NULL);
g_free (mvalue);
@@ -2957,7 +3032,13 @@
case DATA_INDEX:
case DATA_STRING:
- mvalue = tracker_parse_text_to_string (value, def->filtered, def->filtered, def->delimited);
+ mvalue = tracker_parser_text_to_string (value,
+ tracker->language,
+ tracker_config_get_max_word_length (tracker->config),
+ tracker_config_get_min_word_length (tracker->config),
+ def->filtered,
+ def->filtered,
+ def->delimited);
tracker_exec_proc (db_con, "DeleteMetadataValue", id, def->id, mvalue, NULL);
g_free (mvalue);
break;
@@ -3905,7 +3986,11 @@
services = g_array_new (TRUE, TRUE, sizeof (gint));
g_array_append_vals (services, service_array, 8);
- tree = tracker_query_tree_new (text, db_con->word_index, services);
+ tree = tracker_query_tree_new (text,
+ db_con->word_index,
+ tracker->config,
+ tracker->language,
+ services);
hits = tracker_query_tree_get_hits (tree, 0, 0);
count = 0;
@@ -3999,7 +4084,11 @@
services = g_array_new (TRUE, TRUE, sizeof (gint));
g_array_append_vals (services, service_array, 8);
- tree = tracker_query_tree_new (text, db_con->word_index, services);
+ tree = tracker_query_tree_new (text,
+ db_con->word_index,
+ tracker->config,
+ tracker->language,
+ services);
hits = tracker_query_tree_get_hits (tree, 0, 0);
count = 0;
@@ -4094,7 +4183,11 @@
services = g_array_new (TRUE, TRUE, sizeof (gint));
g_array_append_vals (services, service_array, 8);
- tree = tracker_query_tree_new (text, db_con->word_index, services);
+ tree = tracker_query_tree_new (text,
+ db_con->word_index,
+ tracker->config,
+ tracker->language,
+ services);
hits = tracker_query_tree_get_hits (tree, 0, 0);
count = 0;
Modified: branches/indexer-split/src/trackerd/tracker-db.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-db.c (original)
+++ branches/indexer-split/src/trackerd/tracker-db.c Wed May 14 10:10:31 2008
@@ -1014,9 +1014,8 @@
tracker_db_update_differential_index (old_table, index_table, str_file_id, info->service_type_id);
}
- tracker_word_table_free (index_table);
- tracker_word_table_free (old_table);
-
+ tracker_parser_text_free (index_table);
+ tracker_parser_text_free (old_table);
/* check for backup user defined metadata */
if (info->is_new) {
Modified: branches/indexer-split/src/trackerd/tracker-dbus-daemon.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-dbus-daemon.c (original)
+++ branches/indexer-split/src/trackerd/tracker-dbus-daemon.c Wed May 14 10:10:31 2008
@@ -85,10 +85,11 @@
G_PARAM_WRITABLE));
g_object_class_install_property (object_class,
PROP_CONFIG,
- g_param_spec_pointer ("config",
- "Config",
- "TrackerConfig object",
- G_PARAM_WRITABLE));
+ g_param_spec_object ("config",
+ "Config",
+ "TrackerConfig object",
+ tracker_config_get_type (),
+ G_PARAM_WRITABLE));
g_object_class_install_property (object_class,
PROP_TRACKER,
g_param_spec_pointer ("tracker",
@@ -180,7 +181,7 @@
break;
case PROP_CONFIG:
tracker_dbus_daemon_set_config (TRACKER_DBUS_DAEMON (object),
- g_value_get_pointer (value));
+ g_value_get_object (value));
break;
case PROP_TRACKER:
tracker_dbus_daemon_set_tracker (TRACKER_DBUS_DAEMON (object),
@@ -239,12 +240,21 @@
priv = GET_PRIV (object);
+ /* Ref now in case it is the same object we are about to
+ * unref.
+ */
+ g_object_ref (config);
+
if (priv->config) {
g_object_unref (priv->config);
}
-
- priv->config = g_object_ref (config);
-
+
+ if (config) {
+ priv->config = config;
+ } else {
+ priv->config = NULL;
+ }
+
g_object_notify (G_OBJECT (object), "config");
}
Modified: branches/indexer-split/src/trackerd/tracker-dbus-search.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-dbus-search.c (original)
+++ branches/indexer-split/src/trackerd/tracker-dbus-search.c Wed May 14 10:10:31 2008
@@ -23,8 +23,9 @@
#include <libtracker-common/tracker-log.h>
-#include <libtracker-common/tracker-log.h>
#include <libtracker-common/tracker-config.h>
+#include <libtracker-common/tracker-language.h>
+#include <libtracker-common/tracker-log.h>
#include <libtracker-common/tracker-utils.h>
#include "tracker-dbus.h"
@@ -41,15 +42,19 @@
#define GET_PRIV(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), TRACKER_TYPE_DBUS_SEARCH, TrackerDBusSearchPriv))
typedef struct {
- DBusGProxy *fd_proxy;
- DBConnection *db_con;
- Indexer *file_index;
- Indexer *email_index;
+ DBusGProxy *fd_proxy;
+ DBConnection *db_con;
+ TrackerConfig *config;
+ TrackerLanguage *language;
+ Indexer *file_index;
+ Indexer *email_index;
} TrackerDBusSearchPriv;
enum {
PROP_0,
PROP_DB_CONNECTION,
+ PROP_CONFIG,
+ PROP_LANGUAGE,
PROP_FILE_INDEX,
PROP_EMAIL_INDEX
};
@@ -79,6 +84,19 @@
"Database connection to use in transactions",
G_PARAM_WRITABLE));
g_object_class_install_property (object_class,
+ PROP_CONFIG,
+ g_param_spec_object ("config",
+ "Config",
+ "TrackerConfig object",
+ tracker_config_get_type (),
+ G_PARAM_WRITABLE));
+ g_object_class_install_property (object_class,
+ PROP_LANGUAGE,
+ g_param_spec_pointer ("language",
+ "Language",
+ "Language",
+ G_PARAM_WRITABLE));
+ g_object_class_install_property (object_class,
PROP_FILE_INDEX,
g_param_spec_pointer ("file-index",
"File index",
@@ -128,6 +146,14 @@
tracker_dbus_search_set_db_connection (TRACKER_DBUS_SEARCH (object),
g_value_get_pointer (value));
break;
+ case PROP_CONFIG:
+ tracker_dbus_search_set_config (TRACKER_DBUS_SEARCH (object),
+ g_value_get_object (value));
+ break;
+ case PROP_LANGUAGE:
+ tracker_dbus_search_set_language (TRACKER_DBUS_SEARCH (object),
+ g_value_get_pointer (value));
+ break;
case PROP_FILE_INDEX:
tracker_dbus_search_set_file_index (TRACKER_DBUS_SEARCH (object),
g_value_get_pointer (value));
@@ -171,6 +197,46 @@
}
void
+tracker_dbus_search_set_config (TrackerDBusSearch *object,
+ TrackerConfig *config)
+{
+ TrackerDBusSearchPriv *priv;
+
+ g_return_if_fail (TRACKER_IS_DBUS_SEARCH (object));
+ g_return_if_fail (TRACKER_IS_CONFIG (config));
+
+ priv = GET_PRIV (object);
+
+ if (config) {
+ g_object_ref (config);
+ }
+
+ if (priv->config) {
+ g_object_unref (priv->config);
+ }
+
+ priv->config = config;
+
+ g_object_notify (G_OBJECT (object), "config");
+}
+
+void
+tracker_dbus_search_set_language (TrackerDBusSearch *object,
+ TrackerLanguage *language)
+{
+ TrackerDBusSearchPriv *priv;
+
+ g_return_if_fail (TRACKER_IS_DBUS_SEARCH (object));
+ g_return_if_fail (language != NULL);
+
+ priv = GET_PRIV (object);
+
+ priv->language = language;
+
+ g_object_notify (G_OBJECT (object), "language");
+}
+
+void
tracker_dbus_search_set_file_index (TrackerDBusSearch *object,
Indexer *file_index)
{
@@ -585,7 +651,11 @@
array = g_array_new (TRUE, TRUE, sizeof (gint));
g_array_append_vals (array, services, G_N_ELEMENTS (services));
- tree = tracker_query_tree_new (search_text, db_con->word_index, array);
+ tree = tracker_query_tree_new (search_text,
+ db_con->word_index,
+ priv->config,
+ priv->language,
+ array);
*value = tracker_query_tree_get_hit_count (tree);
g_object_unref (tree);
g_array_free (array, TRUE);
@@ -631,7 +701,11 @@
return FALSE;
}
- tree = tracker_query_tree_new (search_text, db_con->word_index, NULL);
+ tree = tracker_query_tree_new (search_text,
+ db_con->word_index,
+ priv->config,
+ priv->language,
+ NULL);
hit_counts = tracker_query_tree_get_hit_counts (tree);
tracker_query_tree_set_indexer (tree, priv->email_index);
@@ -925,7 +999,10 @@
gchar *text;
tracker_db_result_set_get (result_set, 0, &text, -1);
- strv = tracker_parse_text_into_array (text);
+ strv = tracker_parser_text_into_array (text,
+ priv->language,
+ tracker_config_get_max_word_length (priv->config),
+ tracker_config_get_min_word_length (priv->config));
if (strv && strv[0]) {
snippet = dbus_search_get_snippet (text, strv, 120);
Modified: branches/indexer-split/src/trackerd/tracker-dbus-search.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-dbus-search.h (original)
+++ branches/indexer-split/src/trackerd/tracker-dbus-search.h Wed May 14 10:10:31 2008
@@ -26,6 +26,8 @@
#include <dbus/dbus-glib-bindings.h>
#include <dbus/dbus-glib-lowlevel.h>
+#include <libtracker-common/tracker-language.h>
+
#include "tracker-db-sqlite.h"
#include "tracker-indexer.h"
@@ -56,9 +58,13 @@
GType tracker_dbus_search_get_type (void);
TrackerDBusSearch *
- tracker_dbus_search_new (DBConnection *db_con);
+ tracker_dbus_search_new (DBConnection *db_con);
void tracker_dbus_search_set_db_connection (TrackerDBusSearch *object,
DBConnection *db_con);
+void tracker_dbus_search_set_config (TrackerDBusSearch *object,
+ TrackerConfig *config);
+void tracker_dbus_search_set_language (TrackerDBusSearch *object,
+ TrackerLanguage *language);
void tracker_dbus_search_set_file_index (TrackerDBusSearch *object,
Indexer *file_index);
void tracker_dbus_search_set_email_index (TrackerDBusSearch *object,
Modified: branches/indexer-split/src/trackerd/tracker-dbus-xesam.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-dbus-xesam.c (original)
+++ branches/indexer-split/src/trackerd/tracker-dbus-xesam.c Wed May 14 10:10:31 2008
@@ -19,6 +19,10 @@
* Boston, MA 02110-1301, USA.
*/
+#include "config.h"
+
+#include <string.h>
+
#include <libtracker-common/tracker-log.h>
#include <libtracker-common/tracker-config.h>
#include <libtracker-common/tracker-utils.h>
Modified: branches/indexer-split/src/trackerd/tracker-dbus.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-dbus.c (original)
+++ branches/indexer-split/src/trackerd/tracker-dbus.c Wed May 14 10:10:31 2008
@@ -125,17 +125,14 @@
}
gboolean
-tracker_dbus_init (gpointer tracker_pointer)
+tracker_dbus_init (Tracker *tracker)
{
- Tracker *tracker;
DBusGConnection *connection;
DBusGProxy *proxy;
GObject *object;
GError *error = NULL;
- g_return_val_if_fail (tracker_pointer != NULL, FALSE);
-
- tracker = (Tracker*) tracker_pointer;
+ g_return_val_if_fail (tracker != NULL, FALSE);
connection = dbus_g_bus_get (DBUS_BUS_SESSION, &error);
@@ -223,6 +220,8 @@
}
g_object_set (object, "db-connection", tracker->index_db, NULL);
+ g_object_set (object, "config", tracker->config, NULL);
+ g_object_set (object, "language", tracker->language, NULL);
g_object_set (object, "file-index", tracker->file_index, NULL);
g_object_set (object, "email-index", tracker->email_index, NULL);
objects = g_slist_prepend (objects, object);
Modified: branches/indexer-split/src/trackerd/tracker-dbus.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-dbus.h (original)
+++ branches/indexer-split/src/trackerd/tracker-dbus.h Wed May 14 10:10:31 2008
@@ -34,6 +34,8 @@
#include <libtracker-db/tracker-db-interface.h>
+#include "tracker-main.h"
+
G_BEGIN_DECLS
#define TRACKER_DBUS_ERROR_DOMAIN "TrackerDBus"
@@ -76,7 +78,7 @@
gpointer data2;
} TrackerDBusData;
-gboolean tracker_dbus_init (gpointer tracker);
+gboolean tracker_dbus_init (Tracker *tracker);
void tracker_dbus_shutdown (void);
guint tracker_dbus_get_next_request_id (void);
GObject * tracker_dbus_get_object (GType type);
Modified: branches/indexer-split/src/trackerd/tracker-main.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-main.c (original)
+++ branches/indexer-split/src/trackerd/tracker-main.c Wed May 14 10:10:31 2008
@@ -531,15 +531,6 @@
}
static void
-initialise_defaults (void)
-{
- tracker->max_process_queue_size = MAX_PROCESS_QUEUE_SIZE;
- tracker->max_extract_queue_size = MAX_EXTRACT_QUEUE_SIZE;
-
- tracker->index_number_min_length = 6;
-}
-
-static void
initialise_databases (gboolean need_index)
{
Indexer *index;
@@ -869,11 +860,15 @@
initialise_signal_handler ();
+ /* Create struct */
tracker = g_new0 (Tracker, 1);
tracker->pid = getpid ();
tracker->dir_queue = g_async_queue_new ();
+ tracker->max_process_queue_size = MAX_PROCESS_QUEUE_SIZE;
+ tracker->max_extract_queue_size = MAX_EXTRACT_QUEUE_SIZE;
+
/* This makes sure we have all the locations like the data
* dir, user data dir, etc all configured.
*
@@ -925,8 +920,6 @@
/* Deal with config options with defaults, config file and
* option params.
*/
- initialise_defaults ();
-
if (watch_dirs) {
tracker_config_add_watch_directory_roots (tracker->config, watch_dirs);
}
Modified: branches/indexer-split/src/trackerd/tracker-main.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-main.h (original)
+++ branches/indexer-split/src/trackerd/tracker-main.h Wed May 14 10:10:31 2008
@@ -86,9 +86,6 @@
Indexer *email_index;
/* Table of stop words that are to be ignored by the parser */
- GHashTable *stop_words;
- gint index_number_min_length;
-
gboolean first_time_index;
time_t index_time_start;
Modified: branches/indexer-split/src/trackerd/tracker-parser.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-parser.c (original)
+++ branches/indexer-split/src/trackerd/tracker-parser.c Wed May 14 10:10:31 2008
@@ -27,18 +27,18 @@
#include <pango/pango.h>
-#include <libtracker-common/tracker-config.h>
-#include <libtracker-common/tracker-log.h>
#include <libtracker-common/tracker-language.h>
+#include <libtracker-common/tracker-log.h>
#include "tracker-parser.h"
#include "tracker-utils.h"
#include "tracker-main.h"
-extern Tracker *tracker;
-
-/* need pango for CJK ranges which are : 0x3400 - 0x4DB5, 0x4E00 - 0x9FA5, 0x20000 - <= 0x2A6D6 */
+#define INDEX_NUMBER_MIN_LENGTH 6
+/* Need pango for CJK ranges which are : 0x3400 - 0x4DB5, 0x4E00 -
+ * 0x9FA5, 0x20000 - <= 0x2A6D6
+ */
#define NEED_PANGO(c) (((c) >= 0x3400 && (c) <= 0x4DB5) || ((c) >= 0x4E00 && (c) <= 0x9FA5) || ((c) >= 0x20000 && (c) <= 0x2A6D6))
#define IS_LATIN(c) (((c) <= 0x02AF) || ((c) >= 0x1E00 && (c) <= 0x1EFF))
#define IS_ASCII(c) ((c) <= 0x007F)
@@ -50,107 +50,92 @@
#define IS_UNDERSCORE(c) ((c) == 0x005F)
typedef enum {
- WORD_ASCII_HIGHER,
- WORD_ASCII_LOWER,
- WORD_HYPHEN,
- WORD_UNDERSCORE,
- WORD_NUM,
- WORD_ALPHA_HIGHER,
- WORD_ALPHA_LOWER,
- WORD_ALPHA,
- WORD_ALPHA_NUM,
- WORD_IGNORE
+ TRACKER_PARSER_WORD_ASCII_HIGHER,
+ TRACKER_PARSER_WORD_ASCII_LOWER,
+ TRACKER_PARSER_WORD_HYPHEN,
+ TRACKER_PARSER_WORD_UNDERSCORE,
+ TRACKER_PARSER_WORD_NUM,
+ TRACKER_PARSER_WORD_ALPHA_HIGHER,
+ TRACKER_PARSER_WORD_ALPHA_LOWER,
+ TRACKER_PARSER_WORD_ALPHA,
+ TRACKER_PARSER_WORD_ALPHA_NUM,
+ TRACKER_PARSER_WORD_IGNORE
} TrackerParserWordType;
-
-
static inline TrackerParserWordType
get_word_type (gunichar c)
{
- /* fast ascii handling */
+ /* Fast ascii handling */
if (IS_ASCII (c)) {
-
-
if (IS_ASCII_ALPHA_LOWER (c)) {
- return WORD_ASCII_LOWER;
+ return TRACKER_PARSER_WORD_ASCII_LOWER;
}
if (IS_ASCII_ALPHA_HIGHER (c)) {
- return WORD_ASCII_HIGHER;
+ return TRACKER_PARSER_WORD_ASCII_HIGHER;
}
if (IS_ASCII_IGNORE (c)) {
- return WORD_IGNORE;
+ return TRACKER_PARSER_WORD_IGNORE;
}
if (IS_ASCII_NUMERIC (c)) {
- return WORD_NUM;
+ return TRACKER_PARSER_WORD_NUM;
}
if (IS_HYPHEN (c)) {
- return WORD_HYPHEN;
+ return TRACKER_PARSER_WORD_HYPHEN;
}
if (IS_UNDERSCORE (c)) {
- return WORD_UNDERSCORE;
+ return TRACKER_PARSER_WORD_UNDERSCORE;
}
-
-
} else {
-
if (g_unichar_isalpha (c)) {
-
if (!g_unichar_isupper (c)) {
- return WORD_ALPHA_LOWER;
+ return TRACKER_PARSER_WORD_ALPHA_LOWER;
} else {
- return WORD_ALPHA_HIGHER;
+ return TRACKER_PARSER_WORD_ALPHA_HIGHER;
}
-
} else if (g_unichar_isdigit (c)) {
- return WORD_NUM;
+ return TRACKER_PARSER_WORD_NUM;
}
}
- return WORD_IGNORE;
-
+ return TRACKER_PARSER_WORD_IGNORE;
}
-
-static inline char *
-strip_word (const char *str, int length, guint32 *len)
+static inline gchar *
+strip_word (const gchar *str,
+ gint length,
+ guint32 *len)
{
-
*len = length;
#ifdef HAVE_UNAC
-
if (tracker->strip_accents) {
-
- char *s = NULL;
+ gchar *s = NULL;
if (unac_string ("UTF-8", str, length, &s, &*len) != 0) {
- tracker_log ("WARNING: unac failed to strip accents");
+ tracker_warning ("UNAC failed to strip accents");
}
return s;
-
}
-
#endif
+
return NULL;
}
-
static gboolean
-text_needs_pango (const char *text)
+text_needs_pango (const gchar *text)
{
- /* grab first 1024 non-whitespace chars and test */
- const char *p;
- gunichar c;
- int i = 0;
-
- for (p = text; (*p && i < 1024); p = g_utf8_next_char (p)) {
+ const gchar *p;
+ gunichar c;
+ gint i = 0;
+ /* Grab first 1024 non-whitespace chars and test */
+ for (p = text; *p && i < 1024; p = g_utf8_next_char (p)) {
c = g_utf8_get_char (p);
if (!g_unichar_isspace (c)) {
@@ -160,349 +145,352 @@
if (NEED_PANGO(c)) {
return TRUE;
}
-
}
return FALSE;
-
}
-
-
-static const char *
-analyze_text (const char *text, char **index_word, gboolean filter_words, gboolean filter_numbers, gboolean delimit_hyphen)
+static const gchar *
+analyze_text (const gchar *text,
+ TrackerLanguage *language,
+ gint max_word_length,
+ gint min_word_length,
+ gboolean filter_words,
+ gboolean filter_numbers,
+ gboolean delimit_hyphen,
+ gchar **index_word)
{
- const char *p;
- const char *start = NULL;
-
- *index_word = NULL;
-
- if (text) {
- gunichar word[64];
- gboolean do_strip = FALSE, is_valid = TRUE;
- int length = 0;
- glong bytes = 0;
- TrackerParserWordType word_type = WORD_IGNORE;
-
- for (p = text; *p; p = g_utf8_next_char (p)) {
-
- gunichar c = g_utf8_get_char (p);
-
- TrackerParserWordType type = get_word_type (c);
-
- if (type == WORD_IGNORE || (delimit_hyphen && (type == WORD_HYPHEN || type == WORD_UNDERSCORE))) {
-
- if (!start) {
- continue;
- } else {
- break;
- }
- }
-
- if (!is_valid) continue;
-
- if (!start) {
- start = p;
-
- /* valid words must start with an alpha or underscore if we are filtering */
- if (filter_numbers) {
-
- if (type == WORD_NUM) {
- //if (!tracker->index_numbers) {
- is_valid = FALSE;
- continue;
- //}
-
- } else {
-
- if (type == WORD_HYPHEN) {
- is_valid = FALSE;
- continue;
- }
- }
- }
-
- }
-
- if (length >= tracker_config_get_max_word_length (tracker->config)) {
- continue;
- }
-
-
- length++;
-
-
- switch (type) {
-
- case WORD_ASCII_HIGHER:
- c += 32;
-
- case WORD_ASCII_LOWER:
- case WORD_HYPHEN:
- case WORD_UNDERSCORE:
-
- if (word_type == WORD_NUM || word_type == WORD_ALPHA_NUM) {
- word_type = WORD_ALPHA_NUM;
- } else {
- word_type = WORD_ALPHA;
- }
-
- break;
-
- case WORD_NUM:
-
- if (word_type == WORD_ALPHA || word_type == WORD_ALPHA_NUM) {
- word_type = WORD_ALPHA_NUM;
- } else {
- word_type = WORD_NUM;
- }
- break;
-
- case WORD_ALPHA_HIGHER:
- c = g_unichar_tolower (c);
-
- case WORD_ALPHA_LOWER:
-
- if (!do_strip) {
- do_strip = TRUE;
- }
-
- if (word_type == WORD_NUM || word_type == WORD_ALPHA_NUM) {
- word_type = WORD_ALPHA_NUM;
- } else {
- word_type = WORD_ALPHA;
- }
-
- break;
-
- default:
- break;
-
- }
-
- word[length -1] = c;
-
- }
-
-
- if (is_valid) {
-
- if (word_type == WORD_NUM) {
- if (!filter_numbers || length >= tracker->index_number_min_length) {
- *index_word = g_ucs4_to_utf8 (word, length, NULL, NULL, NULL);
- }
-
- } else {
-
- if (length >= tracker_config_get_min_word_length (tracker->config)) {
-
- char *str = NULL, *tmp;
- guint32 len;
-
- char *utf8 = g_ucs4_to_utf8 (word, length, NULL, &bytes, NULL);
-
- if (!utf8) {
- return p;
- }
-
- if (do_strip) {
- str = strip_word (utf8, bytes, &len);
- }
-
- if (!str) {
- tmp = g_utf8_normalize (utf8, bytes, G_NORMALIZE_NFC);
- } else {
- tmp = g_utf8_normalize (str, len, G_NORMALIZE_NFC);
- g_free (str);
- }
-
- g_free (utf8);
-
- /* ignore all stop words */
- if (filter_words && tracker->stop_words) {
- if (g_hash_table_lookup (tracker->stop_words, tmp)) {
- g_free (tmp);
-
- return p;
- }
- }
-
- *index_word = tracker_language_stem_word (tracker->language,
- tmp,
- strlen (tmp));
- if (*index_word) {
- g_free (tmp);
- } else {
- *index_word = tmp;
- }
+ TrackerParserWordType word_type;
+ gunichar word[64];
+ gboolean do_strip;
+ gboolean is_valid;
+ gint length;
+ glong bytes;
+ const char *p;
+ const char *start;
-
- }
- }
-
- }
+ *index_word = NULL;
- return p;
+ if (!text) {
+ return NULL;
+ }
+
+ word_type = TRACKER_PARSER_WORD_IGNORE;
+ do_strip = FALSE;
+ is_valid = TRUE;
+ length = 0;
+ bytes = 0;
+ start = NULL;
+
+ for (p = text; *p; p = g_utf8_next_char (p)) {
+ TrackerParserWordType type;
+ gunichar c;
+
+ c = g_utf8_get_char (p);
+ type = get_word_type (c);
+
+ if (type == TRACKER_PARSER_WORD_IGNORE ||
+ (delimit_hyphen &&
+ (type == TRACKER_PARSER_WORD_HYPHEN ||
+ type == TRACKER_PARSER_WORD_UNDERSCORE))) {
+ if (!start) {
+ continue;
+ } else {
+ break;
+ }
+ }
+
+ if (!is_valid) {
+ continue;
+ }
+
+ if (!start) {
+ start = p;
+
+ /* Valid words must start with an alpha or
+ * underscore if we are filtering.
+ */
+ if (filter_numbers) {
+ if (type == TRACKER_PARSER_WORD_NUM) {
+ is_valid = FALSE;
+ continue;
+ } else {
+ if (type == TRACKER_PARSER_WORD_HYPHEN) {
+ is_valid = FALSE;
+ continue;
+ }
+ }
+ }
+ }
+
+ if (length >= max_word_length) {
+ continue;
+ }
+
+ length++;
+
+ switch (type) {
+ case TRACKER_PARSER_WORD_ASCII_HIGHER:
+ c += 32;
+
+ case TRACKER_PARSER_WORD_ASCII_LOWER:
+ case TRACKER_PARSER_WORD_HYPHEN:
+ case TRACKER_PARSER_WORD_UNDERSCORE:
+ if (word_type == TRACKER_PARSER_WORD_NUM ||
+ word_type == TRACKER_PARSER_WORD_ALPHA_NUM) {
+ word_type = TRACKER_PARSER_WORD_ALPHA_NUM;
+ } else {
+ word_type = TRACKER_PARSER_WORD_ALPHA;
+ }
- }
- return NULL;
-
-}
-
-
-char *
-tracker_parse_text_to_string (const char *txt, gboolean filter_words, gboolean filter_numbers, gboolean delimit)
-{
- const char *p = txt;
-
- char *word = NULL;
- guint32 i = 0;
-
- if (txt) {
-
- if (text_needs_pango (txt)) {
- /* CJK text does not need stemming or other treatment */
-
- PangoLogAttr *attrs;
- guint nb_bytes, str_len, word_start;
- GString *strs;
-
- nb_bytes = strlen (txt);
- str_len = g_utf8_strlen (txt, -1);
-
- strs = g_string_new (" ");
-
- attrs = g_new0 (PangoLogAttr, str_len + 1);
-
- pango_get_log_attrs (txt, nb_bytes, 0, pango_language_from_string ("C"), attrs, str_len + 1);
-
- word_start = 0;
-
- for (i = 0; i < str_len + 1; i++) {
-
- if (attrs[i].is_word_end) {
- char *start_word, *end_word;
-
- start_word = g_utf8_offset_to_pointer (txt, word_start);
- end_word = g_utf8_offset_to_pointer (txt, i);
-
- if (start_word != end_word) {
-
- /* normalize word */
- char *s = g_utf8_casefold (start_word, end_word - start_word);
-
- char *index_word = g_utf8_normalize (s, -1, G_NORMALIZE_NFC);
- g_free (s);
-
- strs = g_string_append (strs, index_word);
- strs = g_string_append_c (strs, ' ');
-
- g_free (index_word);
- }
-
- word_start = i;
- }
-
- if (attrs[i].is_word_start) {
- word_start = i;
- }
- }
-
- g_free (attrs);
-
- return g_string_free (strs, FALSE);
-
+ break;
+
+ case TRACKER_PARSER_WORD_NUM:
+ if (word_type == TRACKER_PARSER_WORD_ALPHA ||
+ word_type == TRACKER_PARSER_WORD_ALPHA_NUM) {
+ word_type = TRACKER_PARSER_WORD_ALPHA_NUM;
+ } else {
+ word_type = TRACKER_PARSER_WORD_NUM;
+ }
+ break;
+
+ case TRACKER_PARSER_WORD_ALPHA_HIGHER:
+ c = g_unichar_tolower (c);
+
+ case TRACKER_PARSER_WORD_ALPHA_LOWER:
+ if (!do_strip) {
+ do_strip = TRUE;
+ }
+
+ if (word_type == TRACKER_PARSER_WORD_NUM ||
+ word_type == TRACKER_PARSER_WORD_ALPHA_NUM) {
+ word_type = TRACKER_PARSER_WORD_ALPHA_NUM;
+ } else {
+ word_type = TRACKER_PARSER_WORD_ALPHA;
+ }
- } else {
-
- GString *str = g_string_new (" ");
-
- while (TRUE) {
- i++;
- p = analyze_text (p, &word, filter_words, filter_numbers, delimit);
-
- if (word) {
-
- g_string_append (str, word);
-
- g_string_append_c (str, ' ');
-
- g_free (word);
- }
-
- if (!p || !*p) {
- return g_string_free (str, FALSE);
- }
- }
- }
- }
-
- return NULL;
+ break;
+
+ default:
+ break;
+ }
+
+ word[length -1] = c;
+ }
+
+ if (is_valid) {
+ if (word_type == TRACKER_PARSER_WORD_NUM) {
+ if (!filter_numbers || length >= INDEX_NUMBER_MIN_LENGTH) {
+ *index_word = g_ucs4_to_utf8 (word, length, NULL, NULL, NULL);
+ }
+ } else {
+ if (length >= min_word_length) {
+ gchar *str = NULL;
+ gchar *tmp;
+ guint32 len;
+ gchar *utf8;
+
+ utf8 = g_ucs4_to_utf8 (word, length, NULL, &bytes, NULL);
+
+ if (!utf8) {
+ return p;
+ }
+
+ if (do_strip) {
+ str = strip_word (utf8, bytes, &len);
+ }
+
+ if (!str) {
+ tmp = g_utf8_normalize (utf8, bytes, G_NORMALIZE_NFC);
+ } else {
+ tmp = g_utf8_normalize (str, len, G_NORMALIZE_NFC);
+ g_free (str);
+ }
+
+ g_free (utf8);
+
+ *index_word = tracker_language_stem_word (language,
+ tmp,
+ strlen (tmp));
+ if (*index_word) {
+ g_free (tmp);
+ } else {
+ *index_word = tmp;
+ }
+ }
+ }
+ }
+
+ return p;
}
-
static void
-delete_words (gpointer key,
- gpointer value,
- gpointer user_data)
+delete_words (gpointer key,
+ gpointer value,
+ gpointer user_data)
{
- char *word;
-
- word = (char *) key;
-
- g_free (word);
-
+ g_free (key);
}
-
-void
-tracker_word_table_free (GHashTable *table)
+gchar *
+tracker_parser_text_to_string (const gchar *txt,
+ TrackerLanguage *language,
+ gint max_word_length,
+ gint min_word_length,
+ gboolean filter_words,
+ gboolean filter_numbers,
+ gboolean delimit)
{
- if (table) {
- g_hash_table_foreach (table, delete_words, NULL);
- g_hash_table_destroy (table);
- }
-}
+ const gchar *p = txt;
+ gchar *word = NULL;
+ guint32 i = 0;
+
+ g_return_val_if_fail (language != NULL, NULL);
+
+ if (!txt) {
+ return NULL;
+ }
+
+ if (text_needs_pango (txt)) {
+ /* CJK text does not need stemming or other
+ * treatment.
+ */
+ PangoLogAttr *attrs;
+ guint nb_bytes, str_len, word_start;
+ GString *strs;
+
+ nb_bytes = strlen (txt);
+ str_len = g_utf8_strlen (txt, -1);
+
+ strs = g_string_new (" ");
+
+ attrs = g_new0 (PangoLogAttr, str_len + 1);
+
+ pango_get_log_attrs (txt,
+ nb_bytes,
+ 0,
+ pango_language_from_string ("C"),
+ attrs,
+ str_len + 1);
+
+ word_start = 0;
+
+ for (i = 0; i < str_len + 1; i++) {
+ if (attrs[i].is_word_end) {
+ gchar *start_word, *end_word;
+
+ start_word = g_utf8_offset_to_pointer (txt, word_start);
+ end_word = g_utf8_offset_to_pointer (txt, i);
+
+ if (start_word != end_word) {
+ /* Normalize word */
+ gchar *s;
+ gchar *index_word;
+
+ s = g_utf8_casefold (start_word, end_word - start_word);
+ index_word = g_utf8_normalize (s, -1, G_NORMALIZE_NFC);
+ g_free (s);
+
+ strs = g_string_append (strs, index_word);
+ strs = g_string_append_c (strs, ' ');
+ g_free (index_word);
+ }
+
+ word_start = i;
+ }
+
+ if (attrs[i].is_word_start) {
+ word_start = i;
+ }
+ }
+
+ g_free (attrs);
+
+ return g_string_free (strs, FALSE);
+ } else {
+ GString *str = g_string_new (" ");
+
+ while (TRUE) {
+ i++;
+ p = analyze_text (p,
+ language,
+ max_word_length,
+ min_word_length,
+ filter_words,
+ filter_numbers,
+ delimit,
+ &word);
+
+ if (word) {
+ g_string_append (str, word);
+ g_string_append_c (str, ' ');
+ g_free (word);
+ }
+
+ if (!p || !*p) {
+ return g_string_free (str, FALSE);
+ }
+ }
+ }
+ return NULL;
+}
-char **
-tracker_parse_text_into_array (const char *text)
+gchar **
+tracker_parser_text_into_array (const gchar *text,
+ TrackerLanguage *language,
+ gint max_word_length,
+ gint min_word_length)
{
- char *s = tracker_parse_text_to_string (text, TRUE, FALSE, FALSE);
+ gchar *s;
+ gchar **strv;
- char **array = g_strsplit (s, " ", -1);
+ g_return_val_if_fail (language != NULL, NULL);
+ s = tracker_parser_text_to_string (text,
+ language,
+ min_word_length,
+ max_word_length,
+ TRUE,
+ FALSE,
+ FALSE);
+ strv = g_strsplit (s, " ", -1);
g_free (s);
- return array;
+ return strv;
}
-
-/* use this for already processed text only */
GHashTable *
-tracker_parse_text_fast (GHashTable *word_table, const char *txt, int weight)
+tracker_parser_text_fast (GHashTable *word_table,
+ const gchar *txt,
+ gint weight)
{
+ gchar **tmp;
+ gchar **array;
+ gpointer k = 0;
+ gpointer v = 0;
+
+ /* Use this for already processed text only */
if (!word_table) {
- word_table = g_hash_table_new_full (g_str_hash, g_str_equal, NULL, NULL);
+ word_table = g_hash_table_new (g_str_hash, g_str_equal);
}
if (!txt || weight == 0) {
return word_table;
}
- char **tmp;
- char **array;
- gpointer k=0,v=0;
-
array = g_strsplit (txt, " ", -1);
for (tmp = array; *tmp; tmp++) {
if (**tmp) {
-
if (!g_hash_table_lookup_extended (word_table, *tmp, &k, &v)) {
- g_hash_table_insert (word_table, g_strdup (*tmp), GINT_TO_POINTER (GPOINTER_TO_INT (v) + weight));
+ g_hash_table_insert (word_table,
+ g_strdup (*tmp),
+ GINT_TO_POINTER (GPOINTER_TO_INT (v) + weight));
} else {
- g_hash_table_insert (word_table, *tmp, GINT_TO_POINTER (GPOINTER_TO_INT (v) + weight));
+ g_hash_table_insert (word_table,
+ *tmp,
+ GINT_TO_POINTER (GPOINTER_TO_INT (v) + weight));
}
}
}
@@ -512,12 +500,25 @@
return word_table;
}
-
-/* use this for unprocessed raw text */
GHashTable *
-tracker_parse_text (GHashTable *word_table, const char *txt, int weight, gboolean filter_words, gboolean delimit_words)
+tracker_parser_text (GHashTable *word_table,
+ const gchar *txt,
+ gint weight,
+ TrackerLanguage *language,
+ gint max_words_to_index,
+ gint max_word_length,
+ gint min_word_length,
+ gboolean filter_words,
+ gboolean delimit_words)
{
- int total_words;
+ const gchar *p;
+ gchar *word;
+ guint32 i;
+
+ /* Use this for unprocessed raw text */
+ gint total_words;
+
+ g_return_val_if_fail (language != NULL, NULL);
if (!word_table) {
word_table = g_hash_table_new (g_str_hash, g_str_equal);
@@ -530,16 +531,14 @@
return word_table;
}
- const char *p = txt;
-
- char *word = NULL;
- guint32 i = 0;
+ p = txt;
+ word = NULL;
+ i = 0;
if (text_needs_pango (txt)) {
/* CJK text does not need stemming or other treatment */
-
PangoLogAttr *attrs;
- guint nb_bytes, str_len, word_start;
+ guint nb_bytes, str_len, word_start;
nb_bytes = strlen (txt);
@@ -547,40 +546,50 @@
attrs = g_new0 (PangoLogAttr, str_len + 1);
- pango_get_log_attrs (txt, nb_bytes, 0, pango_language_from_string ("C"), attrs, str_len + 1);
+ pango_get_log_attrs (txt,
+ nb_bytes,
+ 0,
+ pango_language_from_string ("C"),
+ attrs,
+ str_len + 1);
word_start = 0;
for (i = 0; i < str_len + 1; i++) {
-
if (attrs[i].is_word_end) {
- char *start_word, *end_word;
+ gchar *start_word, *end_word;
start_word = g_utf8_offset_to_pointer (txt, word_start);
end_word = g_utf8_offset_to_pointer (txt, i);
if (start_word != end_word) {
-
- /* normalize word */
- char *s = g_utf8_casefold (start_word, end_word - start_word);
- if (!s) continue;
-
- char *index_word = g_utf8_normalize (s, -1, G_NORMALIZE_NFC);
+ /* Normalize word */
+ gchar *s;
+ gchar *index_word;
+
+ s = g_utf8_casefold (start_word, end_word - start_word);
+ if (!s) {
+ continue;
+ }
+
+ index_word = g_utf8_normalize (s, -1, G_NORMALIZE_NFC);
g_free (s);
- if (!index_word) continue;
+ if (!index_word) {
+ continue;
+ }
total_words++;
- if (total_words < tracker_config_get_max_words_to_index (tracker->config)) {
+ if (total_words < max_words_to_index) {
+ gint count;
- int count = GPOINTER_TO_INT (g_hash_table_lookup (word_table, index_word));
+ count = GPOINTER_TO_INT (g_hash_table_lookup (word_table, index_word));
g_hash_table_insert (word_table, index_word, GINT_TO_POINTER (count + weight));
if (count != 0) {
g_free (index_word);
}
-
} else {
g_free (index_word);
break;
@@ -596,26 +605,30 @@
}
g_free (attrs);
-
} else {
-
while (TRUE) {
i++;
- p = analyze_text (p, &word, filter_words, filter_words, delimit_words);
+ p = analyze_text (p,
+ language,
+ max_word_length,
+ min_word_length,
+ filter_words,
+ filter_words,
+ delimit_words,
+ &word);
if (word) {
-
total_words++;
- if (total_words < tracker_config_get_max_words_to_index (tracker->config)) {
-
- int count = GPOINTER_TO_INT (g_hash_table_lookup (word_table, word));
+ if (total_words < max_words_to_index) {
+ gint count;
+
+ count = GPOINTER_TO_INT (g_hash_table_lookup (word_table, word));
g_hash_table_insert (word_table, word, GINT_TO_POINTER (count + weight));
if (count != 0) {
g_free (word);
}
-
} else {
g_free (word);
break;
@@ -627,7 +640,16 @@
break;
}
}
-
}
+
return word_table;
}
+
+void
+tracker_parser_text_free (GHashTable *table)
+{
+ if (table) {
+ g_hash_table_foreach (table, delete_words, NULL);
+ g_hash_table_destroy (table);
+ }
+}
Modified: branches/indexer-split/src/trackerd/tracker-parser.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-parser.h (original)
+++ branches/indexer-split/src/trackerd/tracker-parser.h Wed May 14 10:10:31 2008
@@ -23,38 +23,50 @@
#include <glib.h>
+#include <libtracker-common/tracker-language.h>
+
G_BEGIN_DECLS
-/* tracker_parse_text
- *
- * function to parse supplied text and break into individual words and
- * maintain a count of no of occurances of the word multiplied by a
+/*
+ * Functions to parse supplied text and break into individual words and
+ * maintain a count of no of occurences of the word multiplied by a
* "weight" factor.
*
- * word_table - can be NULL. Contains the accumulated parsed words
+ * The word_table - can be NULL. It contains the accumulated parsed words
* with weighted word counts for the text (useful for indexing stuff
* line by line)
-
+ *
* text - the text to be parsed
* weight - used to multiply the count of a word's occurance to create
* a weighted rank score
*
- * returns the word_table.
+ * Returns the word_table.
*/
-GHashTable *tracker_parse_text (GHashTable *word_table,
- const char *txt,
- int weight,
- gboolean filter_words,
- gboolean delimit_words);
-GHashTable *tracker_parse_text_fast (GHashTable *word_table,
- const char *txt,
- int weight);
-char * tracker_parse_text_to_string (const char *txt,
- gboolean filter_words,
- gboolean filter_numbers,
- gboolean delimit);
-char ** tracker_parse_text_into_array (const char *text);
-void tracker_word_table_free (GHashTable *table);
+
+GHashTable *tracker_parser_text (GHashTable *word_table,
+ const gchar *txt,
+ gint weight,
+ TrackerLanguage *language,
+ gint max_words_to_index,
+ gint max_word_length,
+ gint min_word_length,
+ gboolean filter_words,
+ gboolean delimit_words);
+GHashTable *tracker_parser_text_fast (GHashTable *word_table,
+ const char *txt,
+ gint weight);
+gchar * tracker_parser_text_to_string (const gchar *txt,
+ TrackerLanguage *language,
+ gint max_word_length,
+ gint min_word_length,
+ gboolean filter_words,
+ gboolean filter_numbers,
+ gboolean delimit);
+gchar ** tracker_parser_text_into_array (const gchar *text,
+ TrackerLanguage *language,
+ gint max_word_length,
+ gint min_word_length);
+void tracker_parser_text_free (GHashTable *table);
G_END_DECLS
Modified: branches/indexer-split/src/trackerd/tracker-query-tree.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-query-tree.c (original)
+++ branches/indexer-split/src/trackerd/tracker-query-tree.c Wed May 14 10:10:31 2008
@@ -17,21 +17,35 @@
* Boston, MA 02110-1301, USA.
*/
-#include <glib-object.h>
+/* These defines are required to allow lrintf() without warnings when
+ * building.
+ */
+#define _ISOC9X_SOURCE 1
+#define _ISOC99_SOURCE 1
+
+#define __USE_ISOC9X 1
+#define __USE_ISOC99 1
+
#include <string.h>
#include <math.h>
#include <depot.h>
+
+#include <glib-object.h>
+
+#include <libtracker-common/tracker-config.h>
+
#include "tracker-query-tree.h"
#include "tracker-parser.h"
#include "tracker-utils.h"
#include "tracker-service-manager.h"
#define TRACKER_QUERY_TREE_GET_PRIVATE(o) (G_TYPE_INSTANCE_GET_PRIVATE ((o), TRACKER_TYPE_QUERY_TREE, TrackerQueryTreePrivate))
+
#define MAX_HIT_BUFFER 480000
#define SCORE_MULTIPLIER 100000
-typedef enum OperationType OperationType;
-typedef enum TreeNodeType TreeNodeType;
+typedef enum OperationType OperationType;
+typedef enum TreeNodeType TreeNodeType;
typedef struct TreeNode TreeNode;
typedef struct TrackerQueryTreePrivate TrackerQueryTreePrivate;
typedef struct ComposeHitsData ComposeHitsData;
@@ -44,23 +58,25 @@
};
struct TreeNode {
- TreeNode *left;
- TreeNode *right;
- OperationType op;
- gchar *term;
+ TreeNode *left;
+ TreeNode *right;
+ OperationType op;
+ gchar *term;
};
struct TrackerQueryTreePrivate {
- gchar *query_str;
- TreeNode *tree;
- Indexer *indexer;
- GArray *services;
+ gchar *query_str;
+ TreeNode *tree;
+ Indexer *indexer;
+ TrackerConfig *config;
+ TrackerLanguage *language;
+ GArray *services;
};
struct ComposeHitsData {
- OperationType op;
- GHashTable *other_table;
- GHashTable *dest_table;
+ OperationType op;
+ GHashTable *other_table;
+ GHashTable *dest_table;
};
struct SearchHitData {
@@ -72,22 +88,22 @@
PROP_0,
PROP_QUERY,
PROP_INDEXER,
+ PROP_CONFIG,
+ PROP_LANGUAGE,
PROP_SERVICES
};
-static void tracker_query_tree_class_init (TrackerQueryTreeClass *class);
-static void tracker_query_tree_init (TrackerQueryTree *tree);
-static void tracker_query_tree_finalize (GObject *object);
-
-static void tracker_query_tree_set_property (GObject *object,
- guint prop_id,
- const GValue *value,
- GParamSpec *pspec);
-static void tracker_query_tree_get_property (GObject *object,
- guint prop_id,
- GValue *value,
- GParamSpec *pspec);
-
+static void tracker_query_tree_class_init (TrackerQueryTreeClass *class);
+static void tracker_query_tree_init (TrackerQueryTree *tree);
+static void tracker_query_tree_finalize (GObject *object);
+static void tracker_query_tree_set_property (GObject *object,
+ guint prop_id,
+ const GValue *value,
+ GParamSpec *pspec);
+static void tracker_query_tree_get_property (GObject *object,
+ guint prop_id,
+ GValue *value,
+ GParamSpec *pspec);
G_DEFINE_TYPE (TrackerQueryTree, tracker_query_tree, G_TYPE_OBJECT)
@@ -114,6 +130,19 @@
"Indexer",
G_PARAM_READWRITE));
g_object_class_install_property (object_class,
+ PROP_CONFIG,
+ g_param_spec_object ("config",
+ "Config",
+ "Config",
+ tracker_config_get_type (),
+ G_PARAM_READWRITE));
+ g_object_class_install_property (object_class,
+ PROP_LANGUAGE,
+ g_param_spec_pointer ("language",
+ "Language",
+ "Language",
+ G_PARAM_READWRITE));
+ g_object_class_install_property (object_class,
PROP_SERVICES,
g_param_spec_pointer ("services",
"Services",
@@ -195,6 +224,14 @@
tracker_query_tree_set_indexer (TRACKER_QUERY_TREE (object),
g_value_get_pointer (value));
break;
+ case PROP_CONFIG:
+ tracker_query_tree_set_config (TRACKER_QUERY_TREE (object),
+ g_value_get_object (value));
+ break;
+ case PROP_LANGUAGE:
+ tracker_query_tree_set_language (TRACKER_QUERY_TREE (object),
+ g_value_get_pointer (value));
+ break;
case PROP_SERVICES:
tracker_query_tree_set_services (TRACKER_QUERY_TREE (object),
g_value_get_pointer (value));
@@ -221,6 +258,12 @@
case PROP_INDEXER:
g_value_set_pointer (value, priv->indexer);
break;
+ case PROP_CONFIG:
+ g_value_set_object (value, priv->config);
+ break;
+ case PROP_LANGUAGE:
+ g_value_set_pointer (value, priv->language);
+ break;
case PROP_SERVICES:
g_value_set_pointer (value, priv->services);
break;
@@ -230,21 +273,27 @@
}
TrackerQueryTree *
-tracker_query_tree_new (const gchar *query_str,
- Indexer *indexer,
- GArray *services)
+tracker_query_tree_new (const gchar *query_str,
+ Indexer *indexer,
+ TrackerConfig *config,
+ TrackerLanguage *language,
+ GArray *services)
{
g_return_val_if_fail (query_str != NULL, NULL);
g_return_val_if_fail (indexer != NULL, NULL);
+ g_return_val_if_fail (TRACKER_IS_CONFIG (config), NULL);
+ g_return_val_if_fail (language != NULL, NULL);
return g_object_new (TRACKER_TYPE_QUERY_TREE,
"query", query_str,
"indexer", indexer,
+ "config", config,
+ "language", language,
"services", services,
NULL);
}
-# if 0
+#if 0
static void
print_tree (TreeNode *node)
{
@@ -348,7 +397,13 @@
break;
default:
/* search term */
- parsed_str = tracker_parse_text_to_string (strings[i], TRUE, FALSE, FALSE);
+ parsed_str = tracker_parser_text_to_string (strings[i],
+ priv->language,
+ tracker_config_get_max_word_length (priv->config),
+ tracker_config_get_min_word_length (priv->config),
+ TRUE,
+ FALSE,
+ FALSE);
node = tree_node_leaf_new (g_strstrip (parsed_str));
g_queue_push_head (queue, node);
last_element_is_term = TRUE;
@@ -446,6 +501,67 @@
}
void
+tracker_query_tree_set_config (TrackerQueryTree *tree,
+ TrackerConfig *config)
+{
+ TrackerQueryTreePrivate *priv;
+
+ g_return_if_fail (TRACKER_IS_QUERY_TREE (tree));
+ g_return_if_fail (TRACKER_IS_CONFIG (config));
+
+ priv = TRACKER_QUERY_TREE_GET_PRIVATE (tree);
+
+ if (config) {
+ g_object_ref (config);
+ }
+
+ if (priv->config) {
+ g_object_unref (priv->config);
+ }
+
+ priv->config = config;
+
+ g_object_notify (G_OBJECT (tree), "config");
+}
+
+TrackerConfig *
+tracker_query_tree_get_config (TrackerQueryTree *tree)
+{
+ TrackerQueryTreePrivate *priv;
+
+ g_return_val_if_fail (TRACKER_IS_QUERY_TREE (tree), NULL);
+
+ priv = TRACKER_QUERY_TREE_GET_PRIVATE (tree);
+ return priv->config;
+}
+
+void
+tracker_query_tree_set_language (TrackerQueryTree *tree,
+ TrackerLanguage *language)
+{
+ TrackerQueryTreePrivate *priv;
+
+ g_return_if_fail (TRACKER_IS_QUERY_TREE (tree));
+ g_return_if_fail (language != NULL);
+
+ priv = TRACKER_QUERY_TREE_GET_PRIVATE (tree);
+ priv->language = language;
+
+ g_object_notify (G_OBJECT (tree), "language");
+}
+
+TrackerLanguage *
+tracker_query_tree_get_language (TrackerQueryTree *tree)
+{
+ TrackerQueryTreePrivate *priv;
+
+ g_return_val_if_fail (TRACKER_IS_QUERY_TREE (tree), NULL);
+
+ priv = TRACKER_QUERY_TREE_GET_PRIVATE (tree);
+ return priv->language;
+}
+
+void
tracker_query_tree_set_services (TrackerQueryTree *tree,
GArray *services)
{
Modified: branches/indexer-split/src/trackerd/tracker-query-tree.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-query-tree.h (original)
+++ branches/indexer-split/src/trackerd/tracker-query-tree.h Wed May 14 10:10:31 2008
@@ -26,6 +26,10 @@
G_BEGIN_DECLS
#include <glib-object.h>
+
+#include <libtracker-common/tracker-config.h>
+#include <libtracker-common/tracker-language.h>
+
#include "tracker-indexer.h"
#define TRACKER_TYPE_QUERY_TREE (tracker_query_tree_get_type())
@@ -62,13 +66,21 @@
GType tracker_query_tree_get_type (void);
TrackerQueryTree * tracker_query_tree_new (const gchar *query_str,
Indexer *indexer,
+ TrackerConfig *config,
+ TrackerLanguage *language,
GArray *services);
G_CONST_RETURN gchar *tracker_query_tree_get_query (TrackerQueryTree *tree);
void tracker_query_tree_set_query (TrackerQueryTree *tree,
const gchar *query_str);
Indexer * tracker_query_tree_get_indexer (TrackerQueryTree *tree);
void tracker_query_tree_set_indexer (TrackerQueryTree *tree,
- Indexer *indexer);
+ Indexer *indexer);
+TrackerConfig * tracker_query_tree_get_config (TrackerQueryTree *tree);
+void tracker_query_tree_set_config (TrackerQueryTree *tree,
+ TrackerConfig *config);
+TrackerLanguage * tracker_query_tree_get_language (TrackerQueryTree *tree);
+void tracker_query_tree_set_language (TrackerQueryTree *tree,
+ TrackerLanguage *language);
GArray * tracker_query_tree_get_services (TrackerQueryTree *tree);
void tracker_query_tree_set_services (TrackerQueryTree *tree,
GArray *services);
Modified: branches/indexer-split/src/trackerd/tracker-xesam-live-search.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-xesam-live-search.c (original)
+++ branches/indexer-split/src/trackerd/tracker-xesam-live-search.c Wed May 14 10:10:31 2008
@@ -19,12 +19,13 @@
* Boston, MA 02110-1301, USA.
*/
+#include <string.h>
+
#include "tracker-xesam-live-search.h"
#include "tracker-dbus-xesam.h"
#include "tracker-dbus.h"
#include "tracker-xesam.h"
-
struct _TrackerXesamLiveSearchPriv {
TrackerXesamSession *session;
gchar *search_id;
Modified: branches/indexer-split/src/trackerd/tracker-xesam-session.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-xesam-session.c (original)
+++ branches/indexer-split/src/trackerd/tracker-xesam-session.c Wed May 14 10:10:31 2008
@@ -19,8 +19,9 @@
* Boston, MA 02110-1301, USA.
*/
-#include "tracker-xesam.h"
+#include <string.h>
+#include "tracker-xesam.h"
struct _TrackerXesamSessionPriv {
GHashTable *searches;
Modified: branches/indexer-split/src/trackerd/tracker-xesam-session.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-xesam-session.h (original)
+++ branches/indexer-split/src/trackerd/tracker-xesam-session.h Wed May 14 10:10:31 2008
@@ -24,11 +24,8 @@
#include <glib.h>
#include <glib-object.h>
-#include <stdlib.h>
-#include <string.h>
#include "tracker-xesam.h"
-
#include "tracker-xesam-live-search.h"
#include "tracker-dbus-xesam.h"
@@ -43,8 +40,9 @@
#define TRACKER_TYPE_XESAM_STRV_ARRAY (dbus_g_type_get_collection ("GPtrArray", G_TYPE_STRV))
+typedef struct _TrackerXesamSession TrackerXesamSession;
typedef struct _TrackerXesamSessionClass TrackerXesamSessionClass;
-typedef struct _TrackerXesamSessionPriv TrackerXesamSessionPriv;
+typedef struct _TrackerXesamSessionPriv TrackerXesamSessionPriv;
struct _TrackerXesamSession {
GObject parent_instance;
Modified: branches/indexer-split/src/trackerd/tracker-xesam.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-xesam.c (original)
+++ branches/indexer-split/src/trackerd/tracker-xesam.c Wed May 14 10:10:31 2008
@@ -29,6 +29,14 @@
extern Tracker *tracker;
+static gboolean live_search_handler_running = FALSE;
+
+GQuark
+tracker_xesam_error_quark (void)
+{
+ return g_quark_from_static_string (TRACKER_XESAM_ERROR_DOMAIN);
+}
+
void
tracker_xesam_init (void)
{
@@ -86,7 +94,9 @@
}
TrackerXesamSession *
-tracker_xesam_get_session_for_search (const gchar *search_id, TrackerXesamLiveSearch **search_in, GError **error)
+tracker_xesam_get_session_for_search (const gchar *search_id,
+ TrackerXesamLiveSearch **search_in,
+ GError **error)
{
TrackerXesamSession *retval = NULL;
GList * sessions = g_hash_table_get_values (tracker->xesam_sessions);
@@ -119,7 +129,7 @@
TrackerXesamLiveSearch *
-tracker_xesam_get_live_search (const gchar *search_id, GError **error)
+tracker_xesam_get_live_search (const gchar *search_id, GError **error)
{
TrackerXesamLiveSearch *retval = NULL;
GList * sessions = g_hash_table_get_values (tracker->xesam_sessions);
@@ -208,8 +218,6 @@
return reason_to_live;
}
-static gboolean live_search_handler_running = FALSE;
-
static void
live_search_handler_destroy (gpointer data)
{
Modified: branches/indexer-split/src/trackerd/tracker-xesam.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-xesam.h (original)
+++ branches/indexer-split/src/trackerd/tracker-xesam.h Wed May 14 10:10:31 2008
@@ -22,8 +22,6 @@
#ifndef __TRACKERD_XESAM_H__
#define __TRACKERD_XESAM_H__
-typedef struct _TrackerXesamSession TrackerXesamSession;
-
#include "tracker-utils.h"
#include "tracker-dbus.h"
#include "tracker-xesam-session.h"
@@ -31,10 +29,8 @@
G_BEGIN_DECLS
-
-typedef enum {
- TRACKER_XESAM_ERROR = 1
-} TrackerXesamErrorDomain;
+#define TRACKER_XESAM_ERROR_DOMAIN "TrackerDBus"
+#define TRACKER_XESAM_ERROR tracker_dbus_error_quark()
typedef enum {
TRACKER_XESAM_ERROR_SEARCH_ID_NOT_REGISTERED = 1,
@@ -44,21 +40,21 @@
TRACKER_XESAM_ERROR_PROPERTY_NOT_SUPPORTED = 5,
} TrackerXesamError;
+GQuark tracker_xesam_error_quark (void);
+void tracker_xesam_init (void);
+TrackerXesamSession* tracker_xesam_create_session (TrackerDBusXesam *dbus_proxy,
+ gchar **session_id,
+ GError **error);
+void tracker_xesam_close_session (const gchar *session_id,
+ GError **error);
TrackerXesamSession* tracker_xesam_get_session (const gchar *session_id,
- GError **error);
+ GError **error);
TrackerXesamSession* tracker_xesam_get_session_for_search (const gchar *search_id,
- TrackerXesamLiveSearch **search_in,
- GError **error);
+ TrackerXesamLiveSearch **search_in,
+ GError **error);
TrackerXesamLiveSearch* tracker_xesam_get_live_search (const gchar *search_id,
- GError **error);
-TrackerXesamSession* tracker_xesam_create_session (TrackerDBusXesam *dbus_proxy,
- gchar **session_id,
- GError **error);
-void tracker_xesam_close_session (const gchar *session_id,
- GError **error);
-void tracker_xesam_init (void);
-void tracker_xesam_wakeup (guint32 last_id);
-
+ GError **error);
+void tracker_xesam_wakeup (guint32 last_id);
gchar * tracker_xesam_generate_unique_key (void);
G_END_DECLS
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
