tracker r1364 - in branches/indexer-split: . src/libtracker-common src/trackerd
- From: mr svn gnome org
- To: svn-commits-list gnome org
- Subject: tracker r1364 - in branches/indexer-split: . src/libtracker-common src/trackerd
- Date: Wed, 7 May 2008 16:08:29 +0100 (BST)
Author: mr
Date: Wed May 7 15:08:29 2008
New Revision: 1364
URL: http://svn.gnome.org/viewvc/tracker?rev=1364&view=rev
Log:
* src/libtracker-common/tracker-utils.h: Fixed #ifndef issue where
this utils file had the same name that the trackerd utils file
had.
* src/trackerd/Makefile.am:
* src/trackerd/tracker-db-email.c:
* src/trackerd/tracker-db-sqlite.[ch]:
* src/trackerd/tracker-db.c:
* src/trackerd/tracker-dbus-daemon.h:
* src/trackerd/tracker-dbus-search.c:
* src/trackerd/tracker-email-evolution.c:
* src/trackerd/tracker-email-thunderbird.c:
* src/trackerd/tracker-email.c:
* src/trackerd/tracker-indexer.c:
* src/trackerd/tracker-inotify.c:
* src/trackerd/tracker-metadata.c:
* src/trackerd/tracker-parser.c:
* src/trackerd/tracker-process-files.h:
* src/trackerd/tracker-utils.[ch]:
* src/trackerd/tracker-xesam.c:
* src/trackerd/trackerd.[ch]: Added trackerd.h for all stuff
related to the Tracker struct. Kept all other utils based
functions in tracker-utils.h. Fixed all the places where this
broke things based on everything being included from
tracker-utils.h and everwhere including that.
Added:
branches/indexer-split/src/trackerd/trackerd.h
Modified:
branches/indexer-split/ChangeLog
branches/indexer-split/src/libtracker-common/tracker-utils.h
branches/indexer-split/src/trackerd/Makefile.am
branches/indexer-split/src/trackerd/tracker-db-email.c
branches/indexer-split/src/trackerd/tracker-db-sqlite.c
branches/indexer-split/src/trackerd/tracker-db-sqlite.h
branches/indexer-split/src/trackerd/tracker-db.c
branches/indexer-split/src/trackerd/tracker-dbus-daemon.h
branches/indexer-split/src/trackerd/tracker-dbus-search.c
branches/indexer-split/src/trackerd/tracker-email-evolution.c
branches/indexer-split/src/trackerd/tracker-email-thunderbird.c
branches/indexer-split/src/trackerd/tracker-email.c
branches/indexer-split/src/trackerd/tracker-indexer.c
branches/indexer-split/src/trackerd/tracker-inotify.c
branches/indexer-split/src/trackerd/tracker-metadata.c
branches/indexer-split/src/trackerd/tracker-parser.c
branches/indexer-split/src/trackerd/tracker-process-files.h
branches/indexer-split/src/trackerd/tracker-utils.c
branches/indexer-split/src/trackerd/tracker-utils.h
branches/indexer-split/src/trackerd/tracker-xesam.c
branches/indexer-split/src/trackerd/trackerd.c
Modified: branches/indexer-split/src/libtracker-common/tracker-utils.h
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-utils.h (original)
+++ branches/indexer-split/src/libtracker-common/tracker-utils.h Wed May 7 15:08:29 2008
@@ -19,12 +19,12 @@
* Boston, MA 02110-1301, USA.
*/
-#ifndef __TRACKER_UTILS_H__
-#define __TRACKER_UTILS_H__
+#ifndef __LIBTRACKER_UTILS_H__
+#define __LIBTRACKER_UTILS_H__
gboolean tracker_is_empty_string (const char *str);
gchar * tracker_string_replace (const gchar *haystack,
gchar *needle,
gchar *replacement);
-#endif /* __TRACKER_UTILS_H__ */
+#endif /* __LIBTRACKER_UTILS_H__ */
Modified: branches/indexer-split/src/trackerd/Makefile.am
==============================================================================
--- branches/indexer-split/src/trackerd/Makefile.am (original)
+++ branches/indexer-split/src/trackerd/Makefile.am Wed May 7 15:08:29 2008
@@ -85,6 +85,7 @@
$(hal_sources) \
$(db_sources) \
trackerd.c \
+ trackerd.h \
tracker-apps.c \
tracker-apps.h \
tracker-db.c \
Modified: branches/indexer-split/src/trackerd/tracker-db-email.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-db-email.c (original)
+++ branches/indexer-split/src/trackerd/tracker-db-email.c Wed May 7 15:08:29 2008
@@ -28,6 +28,7 @@
#include <libtracker-common/tracker-type-utils.h>
#include "tracker-db-email.h"
+#include "tracker-parser.h"
#define MAX_ADDRESSES 255
Modified: branches/indexer-split/src/trackerd/tracker-db-sqlite.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-db-sqlite.c (original)
+++ branches/indexer-split/src/trackerd/tracker-db-sqlite.c Wed May 7 15:08:29 2008
@@ -61,11 +61,13 @@
#include "tracker-service-manager.h"
#include "tracker-query-tree.h"
#include "tracker-xesam.h"
+#include "trackerd.h"
-#define MAX_TEXT_BUFFER 65567
+#define MAX_TEXT_BUFFER 65567
#define MAX_COMPRESS_BUFFER 65565
-#define DEFAULT_PAGE_CACHE 64
-#define MIN_PAGE_CACHE 16
+#define DEFAULT_PAGE_CACHE 64
+#define MIN_PAGE_CACHE 16
+#define ZLIBBUFSIZ 8192
extern Tracker *tracker;
@@ -157,6 +159,181 @@
return result;
}
+static gchar *
+function_compress_string (const gchar *ptr,
+ gint size,
+ gint *compressed_size)
+{
+ z_stream zs;
+ gchar *buf, *swap;
+ unsigned char obuf[ZLIBBUFSIZ];
+ gint rv, asiz, bsiz, osiz;
+
+ if (size < 0) {
+ size = strlen (ptr);
+ }
+
+ zs.zalloc = Z_NULL;
+ zs.zfree = Z_NULL;
+ zs.opaque = Z_NULL;
+
+ if (deflateInit2 (&zs, 6, Z_DEFLATED, 15, 6, Z_DEFAULT_STRATEGY) != Z_OK) {
+ return NULL;
+ }
+
+ asiz = size + 16;
+
+ if (asiz < ZLIBBUFSIZ) {
+ asiz = ZLIBBUFSIZ;
+ }
+
+ if (!(buf = malloc (asiz))) {
+ deflateEnd (&zs);
+ return NULL;
+ }
+
+ bsiz = 0;
+ zs.next_in = (unsigned char *)ptr;
+ zs.avail_in = size;
+ zs.next_out = obuf;
+ zs.avail_out = ZLIBBUFSIZ;
+
+ while ((rv = deflate (&zs, Z_FINISH)) == Z_OK) {
+ osiz = ZLIBBUFSIZ - zs.avail_out;
+
+ if (bsiz + osiz > asiz) {
+ asiz = asiz * 2 + osiz;
+
+ if (!(swap = realloc (buf, asiz))) {
+ free (buf);
+ deflateEnd (&zs);
+ return NULL;
+ }
+
+ buf = swap;
+ }
+
+ memcpy (buf + bsiz, obuf, osiz);
+ bsiz += osiz;
+ zs.next_out = obuf;
+ zs.avail_out = ZLIBBUFSIZ;
+ }
+
+ if (rv != Z_STREAM_END) {
+ free (buf);
+ deflateEnd (&zs);
+ return NULL;
+ }
+
+ osiz = ZLIBBUFSIZ - zs.avail_out;
+
+ if (bsiz + osiz + 1 > asiz) {
+ asiz = asiz * 2 + osiz;
+
+ if (!(swap = realloc (buf, asiz))) {
+ free (buf);
+ deflateEnd (&zs);
+ return NULL;
+ }
+
+ buf = swap;
+ }
+
+ memcpy (buf + bsiz, obuf, osiz);
+ bsiz += osiz;
+ buf[bsiz] = '\0';
+
+ *compressed_size = bsiz;
+
+ deflateEnd (&zs);
+
+ return buf;
+}
+
+static gchar *
+function_uncompress_string (const gchar *ptr,
+ gint size,
+ gint *uncompressed_size)
+{
+ z_stream zs;
+ gchar *buf, *swap;
+ unsigned char obuf[ZLIBBUFSIZ];
+ gint rv, asiz, bsiz, osiz;
+
+ zs.zalloc = Z_NULL;
+ zs.zfree = Z_NULL;
+ zs.opaque = Z_NULL;
+
+ if (inflateInit2 (&zs, 15) != Z_OK) {
+ return NULL;
+ }
+
+ asiz = size * 2 + 16;
+
+ if (asiz < ZLIBBUFSIZ) {
+ asiz = ZLIBBUFSIZ;
+ }
+
+ if (!(buf = malloc (asiz))) {
+ inflateEnd (&zs);
+ return NULL;
+ }
+
+ bsiz = 0;
+ zs.next_in = (unsigned char *)ptr;
+ zs.avail_in = size;
+ zs.next_out = obuf;
+ zs.avail_out = ZLIBBUFSIZ;
+
+ while ((rv = inflate (&zs, Z_NO_FLUSH)) == Z_OK) {
+ osiz = ZLIBBUFSIZ - zs.avail_out;
+
+ if (bsiz + osiz >= asiz) {
+ asiz = asiz * 2 + osiz;
+
+ if (!(swap = realloc (buf, asiz))) {
+ free (buf);
+ inflateEnd (&zs);
+ return NULL;
+ }
+
+ buf = swap;
+ }
+
+ memcpy (buf + bsiz, obuf, osiz);
+ bsiz += osiz;
+ zs.next_out = obuf;
+ zs.avail_out = ZLIBBUFSIZ;
+ }
+
+ if (rv != Z_STREAM_END) {
+ free (buf);
+ inflateEnd (&zs);
+ return NULL;
+ }
+ osiz = ZLIBBUFSIZ - zs.avail_out;
+
+ if (bsiz + osiz >= asiz) {
+ asiz = asiz * 2 + osiz;
+
+ if (!(swap = realloc (buf, asiz))) {
+ free (buf);
+ inflateEnd (&zs);
+ return NULL;
+ }
+
+ buf = swap;
+ }
+
+ memcpy (buf + bsiz, obuf, osiz);
+ bsiz += osiz;
+ buf[bsiz] = '\0';
+ *uncompressed_size = bsiz;
+ inflateEnd (&zs);
+
+ return buf;
+}
+
/* unzips data */
static GValue
function_uncompress (TrackerDBInterface *interface,
@@ -174,7 +351,7 @@
return result;
}
- output = tracker_uncompress ((const gchar *) array->data, array->len, &len);
+ output = function_uncompress_string ((const gchar *) array->data, array->len, &len);
if (!output) {
g_warning ("Uncompress failed");
@@ -1507,7 +1684,7 @@
gint bytes_compressed;
FieldDef *def;
- compressed = tracker_compress (text, length, &bytes_compressed);
+ compressed = function_compress_string (text, length, &bytes_compressed);
if (compressed) {
tracker_debug ("compressed full text size of %d to %d", length, bytes_compressed);
@@ -5019,6 +5196,49 @@
g_free (str_value);
}
+static gint
+get_memory_usage (void)
+{
+#if defined(__linux__)
+ gint fd, length, mem = 0;
+ gchar buffer[8192];
+ gchar *stat_file;
+ gchar **terms;
+
+ stat_file = g_strdup_printf ("/proc/%d/stat", tracker->pid);
+ fd = open (stat_file, O_RDONLY);
+ g_free (stat_file);
+
+ if (fd ==-1) {
+ return 0;
+ }
+
+ length = read (fd, buffer, 8192);
+ buffer[length] = 0;
+ close (fd);
+
+ terms = g_strsplit (buffer, " ", -1);
+
+ if (terms) {
+ gint i;
+
+ for (i = 0; i < 24; i++) {
+ if (!terms[i]) {
+ break;
+ }
+
+ if (i==23) {
+ mem = 4 * atoi (terms[23]);
+ }
+ }
+ }
+
+ g_strfreev (terms);
+
+ return mem;
+#endif
+ return 0;
+}
gboolean
tracker_db_regulate_transactions (DBConnection *db_con, int interval)
@@ -5030,7 +5250,10 @@
if (tracker->index_count > 1) {
tracker_db_end_index_transaction (db_con);
tracker_db_start_index_transaction (db_con);
- tracker_log ("Current memory usage is %d, word count %d and hits %d", tracker_get_memory_usage (), tracker->word_count, tracker->word_detail_count);
+ tracker_log ("Current memory usage is %d, word count %d and hits %d",
+ get_memory_usage (),
+ tracker->word_count,
+ tracker->word_detail_count);
}
return TRUE;
@@ -5067,3 +5290,35 @@
return integrity_check;
}
+
+void
+tracker_free_metadata_field (FieldData *field_data)
+{
+ g_return_if_fail (field_data);
+
+ if (field_data->alias) {
+ g_free (field_data->alias);
+ }
+
+ if (field_data->where_field) {
+ g_free (field_data->where_field);
+ }
+
+ if (field_data->field_name) {
+ g_free (field_data->field_name);
+ }
+
+ if (field_data->select_field) {
+ g_free (field_data->select_field);
+ }
+
+ if (field_data->table_name) {
+ g_free (field_data->table_name);
+ }
+
+ if (field_data->id_field) {
+ g_free (field_data->id_field);
+ }
+
+ g_free (field_data);
+}
Modified: branches/indexer-split/src/trackerd/tracker-db-sqlite.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-db-sqlite.h (original)
+++ branches/indexer-split/src/trackerd/tracker-db-sqlite.h Wed May 7 15:08:29 2008
@@ -26,8 +26,9 @@
#include <libtracker-db/tracker-db-interface.h>
#include <libtracker-db/tracker-db-file-info.h>
-#include "tracker-utils.h"
#include "tracker-service-manager.h"
+#include "tracker-indexer.h"
+#include "tracker-utils.h"
typedef struct DBConnection DBConnection;
@@ -43,6 +44,49 @@
Indexer *word_index;
};
+typedef enum {
+ DATA_KEYWORD,
+ DATA_INDEX,
+ DATA_FULLTEXT,
+ DATA_STRING,
+ DATA_INTEGER,
+ DATA_DOUBLE,
+ DATA_DATE,
+ DATA_BLOB,
+ DATA_STRUCT,
+ DATA_LINK
+} DataTypes;
+
+typedef struct {
+ char *id;
+ DataTypes type;
+ char *field_name;
+ int weight;
+ guint embedded : 1;
+ guint multiple_values : 1;
+ guint delimited : 1;
+ guint filtered : 1;
+ guint store_metadata : 1;
+
+ GSList *child_ids; /* related child metadata ids */
+
+} FieldDef;
+
+typedef struct {
+ char *alias;
+ char *field_name;
+ char *select_field;
+ char *where_field;
+ char *table_name;
+ char *id_field;
+ DataTypes data_type;
+ guint multiple_values : 1;
+ guint is_select : 1;
+ guint is_condition : 1;
+ guint needs_join : 1;
+
+} FieldData;
+
gboolean tracker_db_needs_setup (void);
gboolean tracker_db_needs_data (void);
gboolean tracker_db_initialize (void);
@@ -319,4 +363,6 @@
TrackerDBResultSet *tracker_db_get_live_search_hit_count (DBConnection *db_con,
const gchar *search_id);
+void tracker_free_metadata_field (FieldData *field_data);
+
#endif
Modified: branches/indexer-split/src/trackerd/tracker-db.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-db.c (original)
+++ branches/indexer-split/src/trackerd/tracker-db.c Wed May 7 15:08:29 2008
@@ -406,6 +406,60 @@
}
static void
+notify_meta_data_available (void)
+{
+ if (!tracker->is_running) {
+ return;
+ }
+
+ /* If metadata thread is asleep then we just need to wake it
+ * up!
+ */
+ if (g_mutex_trylock (tracker->metadata_signal_mutex)) {
+ g_cond_signal (tracker->metadata_signal_cond);
+ g_mutex_unlock (tracker->metadata_signal_mutex);
+ return;
+ }
+
+ /* If busy - check if async queue has new stuff as we do not
+ * need to notify then.
+ */
+ if (g_async_queue_length (tracker->file_metadata_queue) > 1) {
+ return;
+ }
+
+ /* If metadata thread not in check phase then we need do
+ * nothing
+ */
+ if (g_mutex_trylock (tracker->metadata_check_mutex)) {
+ g_mutex_unlock (tracker->metadata_check_mutex);
+ return;
+ }
+
+ /* We are in check phase - we need to wait until either
+ * check_mutex is unlocked or until metadata thread is asleep
+ * then we awaken it.
+ */
+ while (TRUE) {
+ if (g_mutex_trylock (tracker->metadata_check_mutex)) {
+ g_mutex_unlock (tracker->metadata_check_mutex);
+ return;
+ }
+
+ if (g_mutex_trylock (tracker->metadata_signal_mutex)) {
+ g_cond_signal (tracker->metadata_signal_cond);
+ g_mutex_unlock (tracker->metadata_signal_mutex);
+ return;
+ }
+
+ g_thread_yield ();
+ g_usleep (10);
+
+ tracker_debug ("In check phase");
+ }
+}
+
+static void
make_pending_file (DBConnection *db_con, guint32 file_id, const char *uri, const char *moved_to_uri, const char *mime, int counter, TrackerDBAction action, gboolean is_directory, gboolean is_new, int service_type_id)
{
char *str_file_id, *str_action, *str_counter;
@@ -466,7 +520,7 @@
/* signal respective thread that data is available and awake it if its asleep */
if (action == TRACKER_DB_ACTION_EXTRACT_METADATA) {
- tracker_notify_meta_data_available ();
+ notify_meta_data_available ();
} else {
tracker_notify_file_data_available ();
}
@@ -517,7 +571,7 @@
tracker_db_insert_pending_file (db_con, info->file_id, info->uri, NULL, info->mime, 0, TRACKER_DB_ACTION_EXTRACT_METADATA, info->is_directory, info->is_new, info->service_type_id);
}
- tracker_notify_meta_data_available ();
+ notify_meta_data_available ();
}
static void
Modified: branches/indexer-split/src/trackerd/tracker-dbus-daemon.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-dbus-daemon.h (original)
+++ branches/indexer-split/src/trackerd/tracker-dbus-daemon.h Wed May 7 15:08:29 2008
@@ -29,6 +29,7 @@
#include <libtracker-common/tracker-config.h>
#include "tracker-db-sqlite.h"
+#include "trackerd.h"
#define TRACKER_DBUS_DAEMON_SERVICE "org.freedesktop.Tracker"
#define TRACKER_DBUS_DAEMON_PATH "/org/freedesktop/Tracker"
Modified: branches/indexer-split/src/trackerd/tracker-dbus-search.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-dbus-search.c (original)
+++ branches/indexer-split/src/trackerd/tracker-dbus-search.c Wed May 7 15:08:29 2008
@@ -29,6 +29,7 @@
#include "tracker-dbus.h"
#include "tracker-dbus-search.h"
+#include "tracker-parser.h"
#include "tracker-rdf-query.h"
#include "tracker-query-tree.h"
#include "tracker-indexer.h"
@@ -214,6 +215,304 @@
return max_hits;
}
+static const gchar *
+dbus_search_utf8_p_from_offset_skipping_decomp (const gchar *str,
+ gint offset)
+{
+ const gchar *p, *q;
+ gchar *casefold, *normal;
+
+ g_return_val_if_fail (str != NULL, NULL);
+
+ p = str;
+
+ while (offset > 0) {
+ q = g_utf8_next_char (p);
+ casefold = g_utf8_casefold (p, q - p);
+ normal = g_utf8_normalize (casefold, -1, G_NORMALIZE_NFC);
+ offset -= g_utf8_strlen (normal, -1);
+ g_free (casefold);
+ g_free (normal);
+ p = q;
+ }
+
+ return p;
+}
+
+static const char *
+dbus_search_utf8_strcasestr_array (const gchar *haystack,
+ gchar **needles)
+{
+ gsize needle_len;
+ gsize haystack_len;
+ const gchar *ret = NULL;
+ const gchar *needle;
+ gchar **array;
+ gchar *p;
+ gchar *casefold;
+ gchar *caseless_haystack;
+ gint i;
+
+ g_return_val_if_fail (haystack != NULL, NULL);
+
+ casefold = g_utf8_casefold (haystack, -1);
+ caseless_haystack = g_utf8_normalize (casefold, -1, G_NORMALIZE_NFC);
+ g_free (casefold);
+
+ if (!caseless_haystack) {
+ return NULL;
+ }
+
+ haystack_len = g_utf8_strlen (caseless_haystack, -1);
+
+ for (array = needles; *array; array++) {
+ needle = *array;
+ needle_len = g_utf8_strlen (needle, -1);
+
+ if (needle_len == 0) {
+ continue;
+ }
+
+ if (haystack_len < needle_len) {
+ continue;
+ }
+
+ p = (gchar *) caseless_haystack;
+ needle_len = strlen (needle);
+ i = 0;
+
+ while (*p) {
+ if ((strncmp (p, needle, needle_len) == 0)) {
+ ret = dbus_search_utf8_p_from_offset_skipping_decomp (haystack, i);
+ goto done;
+ }
+
+ p = g_utf8_next_char (p);
+ i++;
+ }
+ }
+
+done:
+ g_free (caseless_haystack);
+
+ return ret;
+}
+
+static gint
+dbus_search_get_word_break (const char *a)
+{
+ gchar **words;
+ gint value;
+
+ words = g_strsplit_set (a, "\t\n\v\f\r !\"#$%&'()*/<=>?[\\]^`{|}~+,.:;@\"[]" , -1);
+
+ if (!words) {
+ return 0;
+ }
+
+ value = strlen (words[0]);
+ g_strfreev (words);
+
+ return value;
+}
+
+
+static gboolean
+dbus_search_is_word_break (const char a)
+{
+ const gchar *breaks = "\t\n\v\f\r !\"#$%&'()*/<=>?[\\]^`{|}~+,.:;@\"[]";
+ gint i;
+
+ for (i = 0; breaks[i]; i++) {
+ if (a == breaks[i]) {
+ return TRUE;
+ }
+ }
+
+ return FALSE;
+}
+
+static char *
+dbus_search_highlight_terms (const gchar *text,
+ gchar **terms)
+{
+ GStrv p;
+ GString *s;
+ const gchar *str;
+ gchar *text_copy;
+ gint term_len;
+
+ if (!text || !terms) {
+ return NULL;
+ }
+
+ text_copy = g_strdup (text);
+
+ for (p = terms; *p; p++) {
+ const gchar *text_p;
+ gchar **single_term;
+
+ single_term = g_new (gchar*, 2);
+ single_term[0] = g_strdup (*p);
+ single_term[1] = NULL;
+
+ s = g_string_new ("");
+ text_p = text_copy;
+
+ while ((str = dbus_search_utf8_strcasestr_array (text_p, single_term))) {
+ gchar *pre_snip;
+ gchar *term;
+
+ pre_snip = g_strndup (text_p, (str - text_p));
+ term_len = dbus_search_get_word_break (str);
+ term = g_strndup (str, term_len);
+
+ text_p = str + term_len;
+ g_string_append_printf (s, "%s<b>%s</b>", pre_snip, term);
+
+ g_free (pre_snip);
+ g_free (term);
+ }
+
+ if (text_p) {
+ g_string_append (s, text_p);
+ }
+
+ g_strfreev (single_term);
+ g_free (text_copy);
+
+ text_copy = g_string_free (s, FALSE);
+ }
+
+ return text_copy;
+}
+
+gchar *
+dbus_search_get_snippet (const gchar *text,
+ gchar **terms,
+ gint length)
+{
+ const gchar *ptr = NULL;
+ const gchar *end_ptr;
+ const gchar *tmp;
+ gint i;
+ gint text_len;
+
+ if (!text || !terms) {
+ return NULL;
+ }
+
+ text_len = strlen (text);
+ ptr = dbus_search_utf8_strcasestr_array (text, terms);
+
+ if (ptr) {
+ gchar *snippet;
+ gchar *snippet_escaped;
+ gchar *snippet_highlighted;
+
+ tmp = ptr;
+ i = 0;
+
+ /* Get snippet before the matching term */
+ while ((ptr = g_utf8_prev_char (ptr)) && ptr >= text && i < length) {
+ if (*ptr == '\n') {
+ break;
+ }
+
+ i++;
+ }
+
+ /* Try to start beginning of snippet on a word break */
+ if (*ptr != '\n' && ptr > text) {
+ i = 0;
+
+ while (!dbus_search_is_word_break (*ptr) && i < (length / 2)) {
+ ptr = g_utf8_next_char (ptr);
+ i++;
+ }
+ }
+
+ ptr = g_utf8_next_char (ptr);
+
+ if (!ptr || ptr < text) {
+ return NULL;
+ }
+
+ end_ptr = tmp;
+ i = 0;
+
+ /* Get snippet after match */
+ while ((end_ptr = g_utf8_next_char (end_ptr)) &&
+ end_ptr <= text_len + text &&
+ i < length) {
+ i++;
+
+ if (*end_ptr == '\n') {
+ break;
+ }
+ }
+
+ while (end_ptr > text_len + text) {
+ end_ptr = g_utf8_prev_char (end_ptr);
+ }
+
+ /* Try to end snippet on a word break */
+ if (*end_ptr != '\n' && end_ptr < text_len + text) {
+ i=0;
+ while (!dbus_search_is_word_break (*end_ptr) && i < (length / 2)) {
+ end_ptr = g_utf8_prev_char (end_ptr);
+ i++;
+ }
+ }
+
+ if (!end_ptr || !ptr) {
+ return NULL;
+ }
+
+ snippet = g_strndup (ptr, end_ptr - ptr);
+ i = strlen (snippet);
+ snippet_escaped = g_markup_escape_text (snippet, i);
+ g_free (snippet);
+
+ snippet_highlighted = dbus_search_highlight_terms (snippet_escaped, terms);
+ g_free (snippet_escaped);
+
+ return snippet_highlighted;
+ }
+
+ ptr = text;
+ i = 0;
+
+ while ((ptr = g_utf8_next_char (ptr)) && ptr <= text_len + text && i < length) {
+ i++;
+
+ if (*ptr == '\n') {
+ break;
+ }
+ }
+
+ if (ptr > text_len + text) {
+ ptr = g_utf8_prev_char (ptr);
+ }
+
+ if (ptr) {
+ gchar *snippet;
+ gchar *snippet_escaped;
+ gchar *snippet_highlighted;
+
+ snippet = g_strndup (text, ptr - text);
+ snippet_escaped = g_markup_escape_text (snippet, ptr - text);
+ snippet_highlighted = dbus_search_highlight_terms (snippet_escaped, terms);
+
+ g_free (snippet);
+ g_free (snippet_escaped);
+
+ return snippet_highlighted;
+ } else {
+ return NULL;
+ }
+}
+
gboolean
tracker_dbus_search_get_hit_count (TrackerDBusSearch *object,
const gchar *service,
@@ -628,7 +927,7 @@
strv = tracker_parse_text_into_array (text);
if (strv && strv[0]) {
- snippet = tracker_get_snippet (text, strv, 120);
+ snippet = dbus_search_get_snippet (text, strv, 120);
}
g_strfreev (strv);
Modified: branches/indexer-split/src/trackerd/tracker-email-evolution.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-email-evolution.c (original)
+++ branches/indexer-split/src/trackerd/tracker-email-evolution.c Wed May 7 15:08:29 2008
@@ -46,6 +46,8 @@
#include "tracker-dbus-daemon.h"
#include "tracker-watch.h"
#include "tracker-status.h"
+#include "tracker-utils.h"
+#include "trackerd.h"
#define EVOLUTION_MAIL_DIR_S ".evolution/mail"
Modified: branches/indexer-split/src/trackerd/tracker-email-thunderbird.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-email-thunderbird.c (original)
+++ branches/indexer-split/src/trackerd/tracker-email-thunderbird.c Wed May 7 15:08:29 2008
@@ -29,6 +29,7 @@
#include "tracker-db-email.h"
#include "tracker-watch.h"
+#include "trackerd.h"
typedef enum
{
Modified: branches/indexer-split/src/trackerd/tracker-email.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-email.c (original)
+++ branches/indexer-split/src/trackerd/tracker-email.c Wed May 7 15:08:29 2008
@@ -26,6 +26,7 @@
#include "tracker-email.h"
#include "tracker-email-utils.h"
+#include "trackerd.h"
extern Tracker *tracker;
Modified: branches/indexer-split/src/trackerd/tracker-indexer.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-indexer.c (original)
+++ branches/indexer-split/src/trackerd/tracker-indexer.c Wed May 7 15:08:29 2008
@@ -29,6 +29,7 @@
#define INDEXFBP 32
#define SCORE_MULTIPLIER 100000
#define MAX_HIT_BUFFER 480000
+#define MAX_HITS_FOR_WORD 30000
#define MAX_INDEX_FILE_SIZE 2000000000
#define CREATE_INDEX \
Modified: branches/indexer-split/src/trackerd/tracker-inotify.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-inotify.c (original)
+++ branches/indexer-split/src/trackerd/tracker-inotify.c Wed May 7 15:08:29 2008
@@ -41,6 +41,7 @@
#include "tracker-watch.h"
#include "tracker-process-files.h"
+#include "tracker-utils.h"
#define INOTIFY_WATCH_LIMIT "/proc/sys/fs/inotify/max_user_watches"
Modified: branches/indexer-split/src/trackerd/tracker-metadata.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-metadata.c (original)
+++ branches/indexer-split/src/trackerd/tracker-metadata.c Wed May 7 15:08:29 2008
@@ -37,10 +37,10 @@
#include "tracker-metadata.h"
#include "tracker-utils.h"
#include "tracker-service-manager.h"
+#include "trackerd.h"
extern Tracker *tracker;
-
char *
tracker_metadata_get_text_file (const char *uri, const char *mime)
{
Modified: branches/indexer-split/src/trackerd/tracker-parser.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-parser.c (original)
+++ branches/indexer-split/src/trackerd/tracker-parser.c Wed May 7 15:08:29 2008
@@ -33,6 +33,7 @@
#include "tracker-parser.h"
#include "tracker-utils.h"
+#include "trackerd.h"
extern Tracker *tracker;
Modified: branches/indexer-split/src/trackerd/tracker-process-files.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-process-files.h (original)
+++ branches/indexer-split/src/trackerd/tracker-process-files.h Wed May 7 15:08:29 2008
@@ -25,6 +25,7 @@
#include <libtracker-db/tracker-db-file-info.h>
#include "tracker-utils.h"
+#include "trackerd.h"
/* Thread entry point */
gpointer tracker_process_files (gpointer data);
Modified: branches/indexer-split/src/trackerd/tracker-utils.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-utils.c (original)
+++ branches/indexer-split/src/trackerd/tracker-utils.c Wed May 7 15:08:29 2008
@@ -2,6 +2,7 @@
/*
* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
* Copyright (C) 2007, Michal Pryc (Michal Pryc Sun Com)
+ * Copyright (C) 2008, Nokia
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
@@ -20,52 +21,19 @@
*/
#include "config.h"
-#include <sys/time.h>
-#include <sys/resource.h>
-#include <sys/types.h>
-#include <sys/stat.h>
#include <sys/statvfs.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <limits.h>
-#include <zlib.h>
-#include <math.h>
-
-#ifdef OS_WIN32
-#include <conio.h>
-#include "mingw-compat.h"
-#else
-#include <sys/resource.h>
-#endif
-
-#include <glib/gprintf.h>
-#include <glib/gstdio.h>
-#include <glib/gpattern.h>
#include <libtracker-common/tracker-log.h>
#include <libtracker-common/tracker-config.h>
-#include <libtracker-common/tracker-os-dependant.h>
-#include <libtracker-common/tracker-file-utils.h>
-#include <libtracker-common/tracker-utils.h>
-
-#include "tracker-dbus.h"
#include "tracker-utils.h"
-#include "tracker-indexer.h"
-#include "tracker-process-files.h"
-
-extern Tracker *tracker;
-
-#define ZLIBBUFSIZ 8192
+#include "trackerd.h"
-static int info_allocated = 0;
-static int info_deallocated = 0;
+extern Tracker *tracker;
-char *
-tracker_get_radix_by_suffix (const char *str, const char *suffix)
+gchar *
+tracker_get_radix_by_suffix (const gchar *str,
+ const gchar *suffix)
{
g_return_val_if_fail (str, NULL);
g_return_val_if_fail (suffix, NULL);
@@ -77,123 +45,8 @@
}
}
-
-char *
-tracker_escape_metadata (const char *in)
-{
- if (!in) {
- return NULL;
- }
-
- GString *gs = g_string_new ("");
-
- for(; *in; in++) {
- if (*in == '|') {
- g_string_append_c (gs, 30);
- } else {
- g_string_append_c (gs, *in);
- }
- }
-
- return g_string_free (gs, FALSE);
-}
-
-
-
-char *
-tracker_unescape_metadata (const char *in)
-{
- if (!in) {
- return NULL;
- }
-
- GString *gs = g_string_new ("");
-
- for(; *in; in++) {
- if (*in == 30) {
- g_string_append_c (gs, '|');
- } else {
- g_string_append_c (gs, *in);
- }
- }
-
- return g_string_free (gs, FALSE);
-}
-
-char *
-tracker_format_search_terms (const char *str, gboolean *do_bool_search)
-{
- char *def_prefix;
- char **terms;
-
- *do_bool_search = FALSE;
-
- def_prefix = "+";
-
- if (strlen (str) < 3) {
- return g_strdup (str);
- }
-
- /* if already has quotes then do nothing */
- if (strchr (str, '"') || strchr (str, '*')) {
- *do_bool_search = TRUE;
- return g_strdup (str);
- }
-
- if (strstr (str, " or ")) {
- def_prefix = " ";
- }
-
- terms = g_strsplit (str, " ", -1);
-
- if (terms) {
- GString *search_term;
- char **st;
- char *prefix;
-
- search_term = g_string_new (" ");
-
- for (st = terms; *st; st++) {
-
- if (*st[0] == '-') {
- prefix = " ";
- } else {
- prefix = def_prefix;
- }
-
- if ((*st[0] != '-') && strchr (*st, '-')) {
- char *s;
-
- *do_bool_search = TRUE;
-
- s = g_strconcat ("\"", *st, "\"", NULL);
-
- g_string_append (search_term, s);
-
- g_free (s);
-
- } else {
- g_string_append_printf (search_term, " %s%s ", prefix, *st);
- }
- }
-
- g_strfreev (terms);
-
- return g_string_free (search_term, FALSE);
- }
-
- return g_strdup (str);
-}
-
-
-void
-tracker_print_object_allocations (void)
-{
- tracker_log ("Total allocations = %d, total deallocations = %d", info_allocated, info_deallocated);
-}
-
void
-tracker_throttle (int multiplier)
+tracker_throttle (gint multiplier)
{
gint throttle;
@@ -210,37 +63,38 @@
}
}
-
void
tracker_notify_file_data_available (void)
{
+ gint revs = 0;
+
if (!tracker->is_running) {
return;
}
- /* if file thread is asleep then we just need to wake it up! */
+ /* If file thread is asleep then we just need to wake it up! */
if (g_mutex_trylock (tracker->files_signal_mutex)) {
g_cond_signal (tracker->files_signal_cond);
g_mutex_unlock (tracker->files_signal_mutex);
return;
}
- /* if busy - check if async queue has new stuff as we do not need to notify then */
+ /* If busy - check if async queue has new stuff as we do not need to notify then */
if (g_async_queue_length (tracker->file_process_queue) > 1) {
return;
}
- /* if file thread not in check phase then we need do nothing */
+ /* If file thread not in check phase then we need do nothing */
if (g_mutex_trylock (tracker->files_check_mutex)) {
g_mutex_unlock (tracker->files_check_mutex);
return;
}
- int revs = 0;
-
- /* we are in check phase - we need to wait until either check_mutex is unlocked or file thread is asleep then awaken it */
+ /* We are in check phase - we need to wait until either
+ * check_mutex is unlocked or file thread is asleep then
+ * awaken it.
+ */
while (revs < 100000) {
-
if (g_mutex_trylock (tracker->files_check_mutex)) {
g_mutex_unlock (tracker->files_check_mutex);
return;
@@ -254,655 +108,22 @@
g_thread_yield ();
g_usleep (10);
- revs++;
- }
-}
-
-
-void
-tracker_notify_meta_data_available (void)
-{
- if (!tracker->is_running) {
- return;
- }
-
- /* if metadata thread is asleep then we just need to wake it up! */
- if (g_mutex_trylock (tracker->metadata_signal_mutex)) {
- g_cond_signal (tracker->metadata_signal_cond);
- g_mutex_unlock (tracker->metadata_signal_mutex);
- return;
- }
-
- /* if busy - check if async queue has new stuff as we do not need to notify then */
- if (g_async_queue_length (tracker->file_metadata_queue) > 1) {
- return;
- }
-
- /* if metadata thread not in check phase then we need do nothing */
- if (g_mutex_trylock (tracker->metadata_check_mutex)) {
- g_mutex_unlock (tracker->metadata_check_mutex);
- return;
- }
-
- /* we are in check phase - we need to wait until either check_mutex is unlocked or until metadata thread is asleep then we awaken it */
- while (TRUE) {
-
- if (g_mutex_trylock (tracker->metadata_check_mutex)) {
- g_mutex_unlock (tracker->metadata_check_mutex);
- return;
- }
-
- if (g_mutex_trylock (tracker->metadata_signal_mutex)) {
- g_cond_signal (tracker->metadata_signal_cond);
- g_mutex_unlock (tracker->metadata_signal_mutex);
- return;
- }
-
- g_thread_yield ();
- g_usleep (10);
- tracker_debug ("in check phase");
- }
-}
-
-char *
-tracker_compress (const char *ptr, int size, int *compressed_size)
-{
- z_stream zs;
- char *buf, *swap;
- unsigned char obuf[ZLIBBUFSIZ];
- int rv, asiz, bsiz, osiz;
- if (size < 0) size = strlen (ptr);
- zs.zalloc = Z_NULL;
- zs.zfree = Z_NULL;
- zs.opaque = Z_NULL;
-
- if (deflateInit2 (&zs, 6, Z_DEFLATED, 15, 6, Z_DEFAULT_STRATEGY) != Z_OK) {
- return NULL;
- }
-
- asiz = size + 16;
- if (asiz < ZLIBBUFSIZ) asiz = ZLIBBUFSIZ;
- if (!(buf = malloc (asiz))) {
- deflateEnd (&zs);
- return NULL;
- }
- bsiz = 0;
- zs.next_in = (unsigned char *)ptr;
- zs.avail_in = size;
- zs.next_out = obuf;
- zs.avail_out = ZLIBBUFSIZ;
- while ( (rv = deflate (&zs, Z_FINISH)) == Z_OK) {
- osiz = ZLIBBUFSIZ - zs.avail_out;
- if (bsiz + osiz > asiz) {
- asiz = asiz * 2 + osiz;
- if (!(swap = realloc (buf, asiz))) {
- free (buf);
- deflateEnd (&zs);
- return NULL;
- }
- buf = swap;
- }
- memcpy (buf + bsiz, obuf, osiz);
- bsiz += osiz;
- zs.next_out = obuf;
- zs.avail_out = ZLIBBUFSIZ;
- }
- if (rv != Z_STREAM_END) {
- free (buf);
- deflateEnd (&zs);
- return NULL;
- }
-
- osiz = ZLIBBUFSIZ - zs.avail_out;
-
- if (bsiz + osiz + 1 > asiz) {
- asiz = asiz * 2 + osiz;
-
- if (!(swap = realloc (buf, asiz))) {
- free (buf);
- deflateEnd (&zs);
- return NULL;
- }
- buf = swap;
- }
-
- memcpy (buf + bsiz, obuf, osiz);
- bsiz += osiz;
- buf[bsiz] = '\0';
-
- *compressed_size = bsiz;
-
- deflateEnd (&zs);
-
- return buf;
-}
-
-
-char *
-tracker_uncompress (const char *ptr, int size, int *uncompressed_size)
-{
- z_stream zs;
- char *buf, *swap;
- unsigned char obuf[ZLIBBUFSIZ];
- int rv, asiz, bsiz, osiz;
- zs.zalloc = Z_NULL;
- zs.zfree = Z_NULL;
- zs.opaque = Z_NULL;
-
- if (inflateInit2 (&zs, 15) != Z_OK) return NULL;
-
- asiz = size * 2 + 16;
- if (asiz < ZLIBBUFSIZ) asiz = ZLIBBUFSIZ;
- if (!(buf = malloc (asiz))) {
- inflateEnd (&zs);
- return NULL;
- }
- bsiz = 0;
- zs.next_in = (unsigned char *)ptr;
- zs.avail_in = size;
- zs.next_out = obuf;
- zs.avail_out = ZLIBBUFSIZ;
- while ( (rv = inflate (&zs, Z_NO_FLUSH)) == Z_OK) {
- osiz = ZLIBBUFSIZ - zs.avail_out;
- if (bsiz + osiz >= asiz) {
- asiz = asiz * 2 + osiz;
- if (!(swap = realloc (buf, asiz))) {
- free (buf);
- inflateEnd (&zs);
- return NULL;
- }
- buf = swap;
- }
- memcpy (buf + bsiz, obuf, osiz);
- bsiz += osiz;
- zs.next_out = obuf;
- zs.avail_out = ZLIBBUFSIZ;
- }
- if (rv != Z_STREAM_END) {
- free (buf);
- inflateEnd (&zs);
- return NULL;
- }
- osiz = ZLIBBUFSIZ - zs.avail_out;
- if (bsiz + osiz >= asiz) {
- asiz = asiz * 2 + osiz;
- if (!(swap = realloc (buf, asiz))) {
- free (buf);
- inflateEnd (&zs);
- return NULL;
- }
- buf = swap;
- }
- memcpy (buf + bsiz, obuf, osiz);
- bsiz += osiz;
- buf[bsiz] = '\0';
- *uncompressed_size = bsiz;
- inflateEnd (&zs);
- return buf;
-}
-
-
-static inline gboolean
-is_match (const char *a, const char *b)
-{
- int len = strlen (b);
-
- char *str1 = g_utf8_casefold (a, len);
- char *str2 = g_utf8_casefold (b, len);
-
- char *normal1 = g_utf8_normalize (str1, -1, G_NORMALIZE_NFC);
- char *normal2 = g_utf8_normalize (str2, -1, G_NORMALIZE_NFC);
-
- gboolean result = (strcmp (normal1, normal2) == 0);
-
- g_free (str1);
- g_free (str2);
- g_free (normal1);
- g_free (normal2);
-
- return result;
-}
-
-
-static const gchar *
-pointer_from_offset_skipping_decomp (const gchar *str, gint offset)
-{
- gchar *casefold, *normal;
- const gchar *p, *q;
-
- g_return_val_if_fail (str != NULL, NULL);
-
- p = str;
- while (offset > 0)
- {
- q = g_utf8_next_char (p);
- casefold = g_utf8_casefold (p, q - p);
- normal = g_utf8_normalize (casefold, -1, G_NORMALIZE_NFC);
- offset -= g_utf8_strlen (normal, -1);
- g_free (casefold);
- g_free (normal);
- p = q;
- }
- return p;
-}
-
-
-static const char *
-g_utf8_strcasestr_array (const gchar *haystack, gchar **needles)
-{
- gsize needle_len;
- gsize haystack_len;
- const char *ret = NULL, *needle;
- char **array;
- char *p;
- char *casefold;
- char *caseless_haystack;
- int i;
-
- g_return_val_if_fail (haystack != NULL, NULL);
-
- casefold = g_utf8_casefold (haystack, -1);
- caseless_haystack = g_utf8_normalize (casefold, -1, G_NORMALIZE_NFC);
- g_free (casefold);
-
- if (!caseless_haystack) {
- return NULL;
- }
-
- haystack_len = g_utf8_strlen (caseless_haystack, -1);
-
- for (array=needles; *array; array++)
- {
- needle = *array;
- needle_len = g_utf8_strlen (needle, -1);
-
- if (needle_len == 0) {
- continue;
- }
-
- if (haystack_len < needle_len) {
- continue;
- }
-
- p = (gchar *) caseless_haystack;
- needle_len = strlen (needle);
- i = 0;
-
- while (*p) {
-
- if ((strncmp (p, needle, needle_len) == 0)) {
- ret = pointer_from_offset_skipping_decomp (haystack, i);
- goto finally_1;
- }
- p = g_utf8_next_char (p);
- i++;
- }
- }
-
-finally_1:
- g_free (caseless_haystack);
-
- return ret;
-}
-
-
-const char *
-substring_utf8 (const char *a, const char *b)
-{
- const char *ptr, *found_ptr;
- gunichar c, lower, upper;
- int len;
- gboolean got_match = FALSE;
-
- len = strlen (b);
-
- c = g_utf8_get_char (b);
-
- lower = g_unichar_tolower (c);
- upper = g_unichar_toupper (c);
-
- ptr = a;
- found_ptr = a;
-
- /* check lowercase first */
- while (found_ptr) {
-
- found_ptr = g_utf8_strchr (ptr, -1, lower);
-
- if (found_ptr) {
- ptr = g_utf8_find_next_char (found_ptr, NULL);
- if (is_match (found_ptr, b)) {
- got_match = TRUE;
- break;
- }
- } else {
- break;
- }
- }
-
- if (!got_match) {
- ptr = a;
- found_ptr = a;
- while (found_ptr) {
-
- found_ptr = g_utf8_strchr (ptr, -1, upper);
-
- if (found_ptr) {
- ptr = g_utf8_find_next_char (found_ptr, NULL);
- if (is_match (found_ptr, b)) {
- break;
- }
- } else {
-
- }
- }
- }
-
- return found_ptr;
-}
-
-
-static int
-get_word_break (const char *a)
-{
- char **words = g_strsplit_set (a, "\t\n\v\f\r !\"#$%&'()*/<=>?[\\]^`{|}~+,.:;@\"[]" , -1);
-
- if (!words) return 0;
-
- int ret = strlen (words[0]);
-
- g_strfreev (words);
-
- return ret;
-}
-
-
-static gboolean
-is_word_break (const char a)
-{
- const char *breaks = "\t\n\v\f\r !\"#$%&'()*/<=>?[\\]^`{|}~+,.:;@\"[]";
- int i;
-
- for (i = 0; breaks[i]; i++) {
- if (a == breaks[i]) {
- return TRUE;
- }
- }
-
- return FALSE;
-}
-
-
-static char *
-highlight_terms (const char *str, char **terms)
-{
- const char *ptr;
- char *txt;
- GString *st;
- int term_length;
-
- if (!str || !terms) {
- return NULL;
- }
-
- char **array;
- txt = g_strdup (str);
-
- for (array = terms; *array; array++) {
- char **single_term;
-
- single_term = g_new( char *, 2);
- single_term[0] = g_strdup (*array);
- single_term[1] = NULL;
-
- st = g_string_new ("");
-
- const char *ptxt = txt;
-
- while ((ptr = g_utf8_strcasestr_array (ptxt, single_term))) {
- char *pre_snip, *term;
-
- pre_snip = g_strndup (ptxt, (ptr - ptxt));
-
- term_length = get_word_break (ptr);
-
- term = g_strndup (ptr, term_length);
-
- ptxt = ptr + term_length;
-
- g_string_append_printf (st, "%s<b>%s</b>", pre_snip, term);
-
- g_free (pre_snip);
- g_free (term);
- }
-
- if (ptxt) {
- g_string_append (st, ptxt);
-
- }
-
- g_strfreev (single_term);
- g_free (txt);
-
- txt = g_string_free (st, FALSE);
- }
-
- return txt;
-}
-
-
-char *
-tracker_get_snippet (const char *txt, char **terms, int length)
-{
- const char *ptr = NULL, *end_ptr, *tmp;
- int i, txt_len;
-
- if (!txt || !terms) {
- return NULL;
- }
-
- txt_len = strlen (txt);
-
- ptr = g_utf8_strcasestr_array (txt, terms);
-
- if (ptr) {
- tmp = ptr;
-
- i = 0;
-
- /* get snippet before the matching term */
- while ((ptr = g_utf8_prev_char (ptr)) && (ptr >= txt) && (i < length)) {
-
- if (*ptr == '\n') {
- break;
- }
- i++;
- }
-
- /* try to start beginning of snippet on a word break */
- if ((*ptr != '\n') && (ptr > txt)) {
- i=0;
- while (!is_word_break (*ptr) && (i<(length/2))) {
- ptr = g_utf8_next_char (ptr);
- i++;
- }
- }
-
- ptr = g_utf8_next_char (ptr);
-
- if (!ptr || ptr < txt) {
- return NULL;
- }
-
- end_ptr = tmp;
- i = 0;
-
- /* get snippet after match */
- while ((end_ptr = g_utf8_next_char (end_ptr)) && (end_ptr <= txt_len + txt) && (i < length)) {
- i++;
- if (*end_ptr == '\n') {
- break;
- }
- }
-
- while (end_ptr > txt_len + txt) {
- end_ptr = g_utf8_prev_char (end_ptr);
- }
-
- /* try to end snippet on a word break */
- if ((*end_ptr != '\n') && (end_ptr < txt_len + txt)) {
- i=0;
- while (!is_word_break (*end_ptr) && (i<(length/2))) {
- end_ptr = g_utf8_prev_char (end_ptr);
- i++;
- }
- }
-
- if (!end_ptr || !ptr) {
- return NULL;
- }
-
- char *snip, *esc_snip, *highlight_snip;
-
- snip = g_strndup (ptr, end_ptr - ptr);
-
- i = strlen (snip);
-
- esc_snip = g_markup_escape_text (snip, i);
-
- g_free (snip);
-
- highlight_snip = highlight_terms (esc_snip, terms);
-
- g_free (esc_snip);
-
- return highlight_snip;
- }
-
- ptr = txt;
- i = 0;
- while ((ptr = g_utf8_next_char (ptr)) && (ptr <= txt_len + txt) && (i < length)) {
- i++;
- if (*ptr == '\n') {
- break;
- }
- }
-
- if (ptr > txt_len + txt) {
- ptr = g_utf8_prev_char (ptr);
- }
-
- if (ptr) {
- char *snippet = g_strndup (txt, ptr - txt);
- char *esc_snippet = g_markup_escape_text (snippet, ptr - txt);
- char *highlight_snip = highlight_terms (esc_snippet, terms);
-
- g_free (snippet);
- g_free (esc_snippet);
-
- return highlight_snip;
- } else {
- return NULL;
+ revs++;
}
}
void
-tracker_add_metadata_to_table (GHashTable *meta_table, const gchar *key, const gchar *value)
+tracker_add_metadata_to_table (GHashTable *meta_table,
+ const gchar *key,
+ const gchar *value)
{
- GSList *list = g_hash_table_lookup (meta_table, (gchar *) key);
-
- list = g_slist_prepend (list, (gchar *) value);
+ GSList *list;
+ list = g_hash_table_lookup (meta_table, (gchar*) key);
+ list = g_slist_prepend (list, (gchar*) value);
g_hash_table_steal (meta_table, key);
-
- g_hash_table_insert (meta_table, (gchar *) key, list);
-}
-
-
-void
-tracker_free_metadata_field (FieldData *field_data)
-{
- g_return_if_fail (field_data);
-
- if (field_data->alias) {
- g_free (field_data->alias);
- }
-
- if (field_data->where_field) {
- g_free (field_data->where_field);
- }
-
- if (field_data->field_name) {
- g_free (field_data->field_name);
- }
-
- if (field_data->select_field) {
- g_free (field_data->select_field);
- }
-
- if (field_data->table_name) {
- g_free (field_data->table_name);
- }
-
- if (field_data->id_field) {
- g_free (field_data->id_field);
- }
-
- g_free (field_data);
-}
-
-
-
-
-int
-tracker_get_memory_usage (void)
-{
-
-
-#if defined(__linux__)
- int fd, length, mem = 0;
- char buffer[8192];
-
- char *stat_file = g_strdup_printf ("/proc/%d/stat", tracker->pid);
-
- fd = open (stat_file, O_RDONLY);
-
- g_free (stat_file);
-
- if (fd ==-1) {
- return 0;
- }
-
-
- length = read (fd, buffer, 8192);
-
- buffer[length] = 0;
-
- close (fd);
-
- char **terms = g_strsplit (buffer, " ", -1);
-
-
- if (terms) {
-
- int i;
- for (i=0; i < 24; i++) {
- if (!terms[i]) {
- break;
- }
-
- if (i==23) mem = 4 * atoi (terms[23]);
- }
- }
-
-
- g_strfreev (terms);
-
- return mem;
-
-#endif
- return 0;
+ g_hash_table_insert (meta_table, (gchar*) key, list);
}
void
@@ -912,22 +133,23 @@
return;
}
- tracker_log ("file changes to %s is pausing tracker", uri);
-
+ tracker_log ("File changes to:'%s' is causing tracker to pause...",
+ uri);
tracker->grace_period++;
}
gboolean
-tracker_low_diskspace (void)
+tracker_is_low_diskspace (void)
{
struct statvfs st;
gint low_disk_space_limit;
low_disk_space_limit = tracker_config_get_low_disk_space_limit (tracker->config);
- if (low_disk_space_limit < 1)
+ if (low_disk_space_limit < 1) {
return FALSE;
+ }
if (statvfs (tracker->data_dir, &st) == -1) {
static gboolean reported = 0;
@@ -951,7 +173,7 @@
{
return tracker->pause_manual ||
tracker_should_pause_on_battery () ||
- tracker_low_diskspace () ||
+ tracker_is_low_diskspace () ||
tracker_indexer_are_databases_too_big ();
}
Modified: branches/indexer-split/src/trackerd/tracker-utils.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-utils.h (original)
+++ branches/indexer-split/src/trackerd/tracker-utils.h Wed May 7 15:08:29 2008
@@ -1,5 +1,8 @@
-/* Tracker - indexer and metadata database engine
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
+/*
* Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
+ * Copyright (C) 2007, Michal Pryc (Michal Pryc Sun Com)
+ * Copyright (C) 2008, Nokia
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public
@@ -17,333 +20,25 @@
* Boston, MA 02110-1301, USA.
*/
-#ifndef _TRACKER_UTILS_H_
-#define _TRACKER_UTILS_H_
-
-extern char *type_array[];
-extern char *implemented_services[];
-extern char *file_service_array[] ;
-extern char *serice_index_array[];
-extern char *service_table_names[];
-extern char *service_metadata_table_names[];
-extern char *service_metadata_join_names[];
-
-#include "config.h"
-
-#include <time.h>
+#ifndef __TRACKERD_UTILS_H__
+#define __TRACKERD_UTILS_H__
#include <glib.h>
-#include <libtracker-db/tracker-db-action.h>
-
-#include "tracker-parser.h"
-#include "tracker-indexer.h"
-
-#define MAX_HITS_FOR_WORD 30000
-
-/* set merge limit default to 64MB */
-#define MERGE_LIMIT 671088649
-
-/* max default file pause time in ms = FILE_PAUSE_PERIOD * FILE_SCHEDULE_PERIOD */
-#define FILE_PAUSE_PERIOD 1
-#define FILE_SCHEDULE_PERIOD 300
-
-#define TRACKER_DB_VERSION_REQUIRED 13
-#define TRACKER_VERSION VERSION
-#define TRACKER_VERSION_INT 604
-
-/* default performance options */
-#define MAX_INDEX_TEXT_LENGTH 1048576
-#define MAX_PROCESS_QUEUE_SIZE 100
-#define MAX_EXTRACT_QUEUE_SIZE 500
-#define OPTIMIZATION_COUNT 10000
-#define MAX_WORDS_TO_INDEX 10000
-
-typedef struct {
- int id; /* word ID of the cached word */
- int count; /* cummulative count of the cached word */
-} CacheWord;
-
-typedef enum {
- DATA_KEYWORD,
- DATA_INDEX,
- DATA_FULLTEXT,
- DATA_STRING,
- DATA_INTEGER,
- DATA_DOUBLE,
- DATA_DATE,
- DATA_BLOB,
- DATA_STRUCT,
- DATA_LINK
-} DataTypes;
-
-
-typedef enum {
- DB_CATEGORY_FILES,
- DB_CATEGORY_EMAILS,
- DB_CATEGORY_USER
-} DBCategory;
-
-
-typedef enum {
- INDEX_CONFIG,
- INDEX_APPLICATIONS,
- INDEX_FILES,
- INDEX_WEBHISTORY,
- INDEX_CRAWL_FILES,
- INDEX_CONVERSATIONS,
- INDEX_EXTERNAL,
- INDEX_EMAILS,
- INDEX_FINISHED
-} IndexStatus;
-
-
-typedef struct {
- char *id;
- DataTypes type;
- char *field_name;
- int weight;
- guint embedded : 1;
- guint multiple_values : 1;
- guint delimited : 1;
- guint filtered : 1;
- guint store_metadata : 1;
-
- GSList *child_ids; /* related child metadata ids */
-
-} FieldDef;
-
-
-typedef struct {
- char *alias;
- char *field_name;
- char *select_field;
- char *where_field;
- char *table_name;
- char *id_field;
- DataTypes data_type;
- guint multiple_values : 1;
- guint is_select : 1;
- guint is_condition : 1;
- guint needs_join : 1;
-
-} FieldData;
-
-
-typedef struct {
- char *name;
- char *type;
-} ServiceInfo;
-
-
-typedef enum {
- EVENT_NOTHING,
- EVENT_SHUTDOWN,
- EVENT_DISABLE,
- EVENT_PAUSE,
- EVENT_CACHE_FLUSHED
-} LoopEvent;
-
-
-typedef struct {
- gchar *uri;
- time_t first_change_time;
- gint num_of_change;
-} FileChange;
-
-
-typedef struct {
-
- gboolean readonly;
-
- int pid;
-
- gpointer hal;
-
- gboolean reindex;
-
- gpointer config;
- gpointer language;
-
- /* config options */
- guint32 watch_limit;
-
- gboolean fatal_errors;
-
- gpointer index_db;
-
- /* data directories */
- char *data_dir;
- char *config_dir;
- char *root_dir;
- char *user_data_dir;
- char *sys_tmp_root_dir;
- char *email_attachements_dir;
- char *services_dir;
-
- /* performance and memory usage options */
- int max_index_text_length; /* max size of file's text contents to index */
- int max_process_queue_size;
- int max_extract_queue_size;
- int memory_limit;
- int thread_stack_size;
-
- /* HAL battery */
- char *battery_udi;
-
- /* pause/shutdown vars */
- gboolean shutdown;
- gboolean pause_manual;
- gboolean pause_battery;
- gboolean pause_io;
-
- /* indexing options */
- Indexer *file_index;
- Indexer *file_update_index;
- Indexer *email_index;
-
- guint32 merge_limit; /* size of index in MBs when merging is triggered -1 == no merging*/
- gboolean active_file_merge;
- gboolean active_email_merge;
-
- GHashTable *stop_words; /* table of stop words that are to be ignored by the parser */
-
- gboolean index_numbers;
- int index_number_min_length;
- gboolean strip_accents;
-
- gboolean first_time_index;
- gboolean first_flush;
- gboolean do_optimize;
-
- time_t index_time_start;
- int folders_count;
- int folders_processed;
- int mbox_count;
- int mbox_processed;
-
-
- const char *current_uri;
-
- IndexStatus index_status;
-
- int grace_period;
- gboolean request_waiting;
-
- char * xesam_dir;
-
- /* lookup tables for service and metadata IDs */
- GHashTable *metadata_table;
-
- /* email config options */
- GSList *additional_mboxes_to_index;
-
- int email_service_min;
- int email_service_max;
-
- /* nfs options */
- gboolean use_nfs_safe_locking; /* use safer but much slower external lock file when users home dir is on an nfs systems */
-
- /* Queue for recorad file changes */
- GQueue *file_change_queue;
- gboolean black_list_timer_active;
-
- /* progress info for merges */
- int merge_count;
- int merge_processed;
-
-
- /* application run time values */
- gboolean is_indexing;
- gboolean in_flush;
- gboolean in_merge;
- int index_count;
- int index_counter;
- int update_count;
-
- /* cache words before saving to word index */
- GHashTable *file_word_table;
- GHashTable *file_update_word_table;
- GHashTable *email_word_table;
-
- int word_detail_limit;
- int word_detail_count;
- int word_detail_min;
- int word_count;
- int word_update_count;
- int word_count_limit;
- int word_count_min;
- int flush_count;
-
- int file_update_count;
- int email_update_count;
-
- gboolean is_running;
- gboolean is_dir_scan;
- GMainLoop *loop;
-
- GMutex *log_access_mutex;
- char *log_file;
-
- GAsyncQueue *file_process_queue;
- GAsyncQueue *file_metadata_queue;
-
- GAsyncQueue *dir_queue;
-
- GMutex *files_check_mutex;
- GMutex *files_signal_mutex;
- GCond *files_signal_cond;
-
- GMutex *metadata_check_mutex;
- GMutex *metadata_signal_mutex;
- GCond *metadata_signal_cond;
-
- GHashTable *xesam_sessions;
-
-} Tracker;
-
-
-char * tracker_get_radix_by_suffix (const char *str, const char *suffix);
-
-char * tracker_escape_metadata (const char *in);
-char * tracker_unescape_metadata (const char *in);
-
-char * tracker_format_search_terms (const char *str, gboolean *do_bool_search);
-
-GSList * tracker_get_watch_root_dirs (void);
-
-void tracker_print_object_allocations (void);
-
-void tracker_throttle (int multiplier);
-
-void tracker_notify_file_data_available (void);
-void tracker_notify_meta_data_available (void);
-void tracker_notify_request_data_available (void);
-
-char * tracker_compress (const char *ptr, int size, int *compressed_size);
-char * tracker_uncompress (const char *ptr, int size, int *uncompressed_size);
-
-char * tracker_get_snippet (const char *txt, char **terms, int length);
-
-gboolean tracker_spawn (char **argv, int timeout, char **tmp_stdout, int *exit_status);
-
-void tracker_add_metadata_to_table (GHashTable *meta_table, const char *key, const char *value);
-
-void tracker_free_metadata_field (FieldData *field_data);
-
-int tracker_get_memory_usage (void);
-
-void tracker_add_io_grace (const char *uri);
-
-void free_file_change (FileChange **user_data);
-gboolean tracker_do_cleanup (const gchar *sig_msg);
-gboolean tracker_watch_dir (const gchar *uri);
-void tracker_scan_directory (const gchar *uri);
-
-gboolean tracker_low_diskspace (void);
+G_BEGIN_DECLS
-gboolean tracker_should_pause (void);
-gboolean tracker_should_pause_on_battery (void);
+gchar * tracker_get_radix_by_suffix (const gchar *str,
+ const gchar *suffix);
+void tracker_throttle (gint multiplier);
+void tracker_notify_file_data_available (void);
+void tracker_add_metadata_to_table (GHashTable *meta_table,
+ const char *key,
+ const char *value);
+void tracker_add_io_grace (const char *uri);
+gboolean tracker_is_diskspace_low (void);
+gboolean tracker_should_pause (void);
+gboolean tracker_should_pause_on_battery (void);
+G_END_DECLS
-#endif
+#endif /* __TRACKERD_UTILS_H__ */
Modified: branches/indexer-split/src/trackerd/tracker-xesam.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-xesam.c (original)
+++ branches/indexer-split/src/trackerd/tracker-xesam.c Wed May 7 15:08:29 2008
@@ -25,6 +25,7 @@
#include <libtracker-common/tracker-config.h>
#include "tracker-xesam.h"
+#include "trackerd.h"
extern Tracker *tracker;
Modified: branches/indexer-split/src/trackerd/trackerd.c
==============================================================================
--- branches/indexer-split/src/trackerd/trackerd.c (original)
+++ branches/indexer-split/src/trackerd/trackerd.c Wed May 7 15:08:29 2008
@@ -218,8 +218,6 @@
if (sig_msg) {
tracker_log ("Received signal '%s' so now shutting down", sig_msg);
-
- tracker_print_object_allocations ();
}
/* set kill timeout */
Added: branches/indexer-split/src/trackerd/trackerd.h
==============================================================================
--- (empty file)
+++ branches/indexer-split/src/trackerd/trackerd.h Wed May 7 15:08:29 2008
@@ -0,0 +1,266 @@
+/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
+/*
+ * Copyright (C) 2006, Mr Jamie McCracken (jamiemcc gnome org)
+ * Copyright (C) 2007, Michal Pryc (Michal Pryc Sun Com)
+ * Copyright (C) 2008, Nokia
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ */
+
+#ifndef __TRACKERD_H__
+#define __TRACKERD_H__
+
+extern char *type_array[];
+extern char *implemented_services[];
+extern char *file_service_array[] ;
+extern char *serice_index_array[];
+extern char *service_table_names[];
+extern char *service_metadata_table_names[];
+extern char *service_metadata_join_names[];
+
+#include "config.h"
+
+#include <time.h>
+
+#include <glib.h>
+
+#include <libtracker-db/tracker-db-action.h>
+
+#include "tracker-parser.h"
+#include "tracker-indexer.h"
+
+/* set merge limit default to 64MB */
+#define MERGE_LIMIT 671088649
+
+/* max default file pause time in ms = FILE_PAUSE_PERIOD * FILE_SCHEDULE_PERIOD */
+#define FILE_PAUSE_PERIOD 1
+#define FILE_SCHEDULE_PERIOD 300
+
+#define TRACKER_DB_VERSION_REQUIRED 13
+#define TRACKER_VERSION VERSION
+#define TRACKER_VERSION_INT 604
+
+/* default performance options */
+#define MAX_INDEX_TEXT_LENGTH 1048576
+#define MAX_PROCESS_QUEUE_SIZE 100
+#define MAX_EXTRACT_QUEUE_SIZE 500
+#define OPTIMIZATION_COUNT 10000
+#define MAX_WORDS_TO_INDEX 10000
+
+G_BEGIN_DECLS
+
+typedef struct {
+ int id; /* word ID of the cached word */
+ int count; /* cummulative count of the cached word */
+} CacheWord;
+
+typedef enum {
+ INDEX_CONFIG,
+ INDEX_APPLICATIONS,
+ INDEX_FILES,
+ INDEX_WEBHISTORY,
+ INDEX_CRAWL_FILES,
+ INDEX_CONVERSATIONS,
+ INDEX_EXTERNAL,
+ INDEX_EMAILS,
+ INDEX_FINISHED
+} IndexStatus;
+
+
+typedef struct {
+ char *name;
+ char *type;
+} ServiceInfo;
+
+
+typedef enum {
+ EVENT_NOTHING,
+ EVENT_SHUTDOWN,
+ EVENT_DISABLE,
+ EVENT_PAUSE,
+ EVENT_CACHE_FLUSHED
+} LoopEvent;
+
+
+typedef struct {
+ gchar *uri;
+ time_t first_change_time;
+ gint num_of_change;
+} FileChange;
+
+
+typedef struct {
+
+ gboolean readonly;
+
+ int pid;
+
+ gpointer hal;
+
+ gboolean reindex;
+
+ gpointer config;
+ gpointer language;
+
+ /* config options */
+ guint32 watch_limit;
+
+ gboolean fatal_errors;
+
+ gpointer index_db;
+
+ /* data directories */
+ char *data_dir;
+ char *config_dir;
+ char *root_dir;
+ char *user_data_dir;
+ char *sys_tmp_root_dir;
+ char *email_attachements_dir;
+ char *services_dir;
+
+ /* performance and memory usage options */
+ int max_index_text_length; /* max size of file's text contents to index */
+ int max_process_queue_size;
+ int max_extract_queue_size;
+ int memory_limit;
+ int thread_stack_size;
+
+ /* HAL battery */
+ char *battery_udi;
+
+ /* pause/shutdown vars */
+ gboolean shutdown;
+ gboolean pause_manual;
+ gboolean pause_battery;
+ gboolean pause_io;
+
+ /* indexing options */
+ Indexer *file_index;
+ Indexer *file_update_index;
+ Indexer *email_index;
+
+ guint32 merge_limit; /* size of index in MBs when merging is triggered -1 == no merging*/
+ gboolean active_file_merge;
+ gboolean active_email_merge;
+
+ GHashTable *stop_words; /* table of stop words that are to be ignored by the parser */
+
+ gboolean index_numbers;
+ int index_number_min_length;
+ gboolean strip_accents;
+
+ gboolean first_time_index;
+ gboolean first_flush;
+ gboolean do_optimize;
+
+ time_t index_time_start;
+ int folders_count;
+ int folders_processed;
+ int mbox_count;
+ int mbox_processed;
+
+
+ const char *current_uri;
+
+ IndexStatus index_status;
+
+ int grace_period;
+ gboolean request_waiting;
+
+ char * xesam_dir;
+
+ /* lookup tables for service and metadata IDs */
+ GHashTable *metadata_table;
+
+ /* email config options */
+ GSList *additional_mboxes_to_index;
+
+ int email_service_min;
+ int email_service_max;
+
+ /* nfs options */
+ gboolean use_nfs_safe_locking; /* use safer but much slower external lock file when users home dir is on an nfs systems */
+
+ /* Queue for recorad file changes */
+ GQueue *file_change_queue;
+ gboolean black_list_timer_active;
+
+ /* progress info for merges */
+ int merge_count;
+ int merge_processed;
+
+
+ /* application run time values */
+ gboolean is_indexing;
+ gboolean in_flush;
+ gboolean in_merge;
+ int index_count;
+ int index_counter;
+ int update_count;
+
+ /* cache words before saving to word index */
+ GHashTable *file_word_table;
+ GHashTable *file_update_word_table;
+ GHashTable *email_word_table;
+
+ int word_detail_limit;
+ int word_detail_count;
+ int word_detail_min;
+ int word_count;
+ int word_update_count;
+ int word_count_limit;
+ int word_count_min;
+ int flush_count;
+
+ int file_update_count;
+ int email_update_count;
+
+ gboolean is_running;
+ gboolean is_dir_scan;
+ GMainLoop *loop;
+
+ GMutex *log_access_mutex;
+ char *log_file;
+
+ GAsyncQueue *file_process_queue;
+ GAsyncQueue *file_metadata_queue;
+
+ GAsyncQueue *dir_queue;
+
+ GMutex *files_check_mutex;
+ GMutex *files_signal_mutex;
+ GCond *files_signal_cond;
+
+ GMutex *metadata_check_mutex;
+ GMutex *metadata_signal_mutex;
+ GCond *metadata_signal_cond;
+
+ GHashTable *xesam_sessions;
+} Tracker;
+
+GSList * tracker_get_watch_root_dirs (void);
+gboolean tracker_spawn (gchar **argv,
+ gint timeout,
+ gchar **tmp_stdout,
+ gint *exit_status);
+gboolean tracker_do_cleanup (const gchar *sig_msg);
+gboolean tracker_watch_dir (const gchar *uri);
+void tracker_scan_directory (const gchar *uri);
+void free_file_change (FileChange **user_data);
+
+G_END_DECLS
+
+#endif /* __TRACKERD_H__ */
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
