GNOME.org
 

tracker r1936 - in branches/indexer-split: . src/trackerd tests/trackerd

Author: ifrade
Date: Fri Jul 25 13:15:13 2008
New Revision: 1936
URL: http://svn.gnome.org/viewvc/tracker?rev=1936&view=rev

Log:
Added reload capabilities to trackerd/tracker-indexer

Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/src/trackerd/tracker-indexer.c
   branches/indexer-split/src/trackerd/tracker-indexer.h
   branches/indexer-split/tests/trackerd/Makefile.am
   branches/indexer-split/tests/trackerd/tracker-indexer-test.c

Modified: branches/indexer-split/src/trackerd/tracker-indexer.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-indexer.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-indexer.c	Fri Jul 25 13:15:13 2008
@@ -56,6 +56,8 @@
 	gchar         *name;
 	guint          min_bucket;
         guint          max_bucket;
+	
+	gboolean       reload;
 };
 
 static void tracker_indexer_class_init   (TrackerIndexerClass *class);
@@ -74,7 +76,8 @@
 	PROP_0,
 	PROP_NAME,
         PROP_MIN_BUCKET,
-	PROP_MAX_BUCKET
+	PROP_MAX_BUCKET,
+	PROP_RELOAD
 };
 
 G_DEFINE_TYPE (TrackerIndexer, tracker_indexer, G_TYPE_OBJECT)
@@ -114,6 +117,16 @@
 							   1000000, /* FIXME MAX_GUINT ?? */
 							   0,
 							   G_PARAM_READWRITE));
+	g_object_class_install_property (object_class,
+					 PROP_RELOAD,
+					 g_param_spec_int ("reload",
+							   "Reload the index file before read",
+							   "Reload the index file before read",
+							   0,
+							   1000000, /* FIXME MAX_GUINT ?? */
+							   0,
+							   G_PARAM_READWRITE));
+	
 	g_type_class_add_private (object_class, sizeof (TrackerIndexerPrivate));
 }
 
@@ -125,6 +138,7 @@
 	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
         
 	priv->word_mutex = g_mutex_new ();
+	priv->reload = TRUE;
 }
 
 static void
@@ -140,7 +154,7 @@
 
         g_mutex_lock (priv->word_mutex);
 
-	if (!dpclose (priv->word_index)) {
+	if (priv->word_index && !dpclose (priv->word_index)) {
 		g_message ("Index closure has failed, %s", dperrmsg (dpecode));
 	}
 
@@ -170,6 +184,11 @@
 		tracker_indexer_set_max_bucket (TRACKER_INDEXER (object),
 						g_value_get_int (value));
 		break;
+	case PROP_RELOAD:
+		tracker_indexer_set_reload (TRACKER_INDEXER (object),
+					    g_value_get_boolean (value));
+		break;
+
 	default:
 		G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
 	}
@@ -195,6 +214,10 @@
 	case PROP_MAX_BUCKET:
 		g_value_set_int (value, priv->max_bucket);
 		break;
+	case PROP_RELOAD:
+		g_value_set_boolean (value, priv->reload);
+		break;
+
 	default:
 		G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
 	}
@@ -244,40 +267,84 @@
 }
 
 
-static inline DEPOT *
-open_index (const gchar *filename,
-            gint         min_bucket_count,
-            gint         max_bucket_count)
-{
-	DEPOT *word_index = NULL;
-
-	g_return_val_if_fail (filename, NULL);
-
-	g_message ("Opening index:'%s'", filename);
-
-	word_index = dpopen (filename, 
-			     DP_OWRITER | DP_OCREAT | DP_ONOLCK, /* Should be DP_OREADER!!!! */
-			     max_bucket_count);
-
-	if (!word_index) {
-		g_critical ("Index was not closed properly, index:'%s', %s", 
-                            filename, 
-                            dperrmsg (dpecode));
-		g_message ("Attempting to repair...");
-
-		if (dprepair (filename)) {
-			word_index = dpopen (filename, 
-                                             DP_OWRITER | DP_OCREAT | DP_ONOLCK, /* Should be DP_OREADER!!!! */
-                                             max_bucket_count);
-		} else {
-			g_critical ("Index file is dead, it is suggested you remove "
-                                    "the indexe file:'%s' and restart trackerd",
-                                    filename);
-                        return NULL;
-		}
+void
+tracker_indexer_set_reload (TrackerIndexer *indexer,
+			    gboolean reload)
+{
+	TrackerIndexerPrivate *priv;
+
+	g_return_if_fail (TRACKER_IS_INDEXER (indexer));
+
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+	priv->reload = reload;
+}
+
+gboolean
+tracker_indexer_get_reload (TrackerIndexer *indexer)
+{
+	TrackerIndexerPrivate *priv;
+
+	g_return_val_if_fail (TRACKER_IS_INDEXER (indexer), FALSE);
+
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+	return priv->reload;
+}
+
+static inline gboolean
+tracker_indexer_open_index (TrackerIndexer *indexer)
+{
+	TrackerIndexerPrivate *priv = NULL;
+	gint                   bucket_count;
+        gint                   rec_count;
+
+	g_return_val_if_fail (TRACKER_IS_INDEXER (indexer), FALSE);
+
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+	g_return_val_if_fail (priv->name, FALSE);
+
+	g_message ("Opening index:'%s'", priv->name);
+
+	priv->word_index = dpopen (priv->name, 
+				   DP_OREADER | DP_ONOLCK, 
+				   priv->max_bucket);
+
+	if (!priv->word_index) {
+		g_debug ("Index doesnt exists or was not closed properly, index:'%s', %s", 
+			 priv->name, 
+			 dperrmsg (dpecode));
+
+		if (g_file_test (priv->name, G_FILE_TEST_EXISTS)) {
+
+			if (dprepair (priv->name)) {
+				priv->word_index = dpopen (priv->name, 
+							   DP_OREADER | DP_ONOLCK, 
+							   priv->max_bucket);
+			} else {
+				g_critical ("Corrupted index file %s. Remove and restart trackerd.", 
+					    priv->name);
+			}
+		} 
+	}
+
+
+	if (priv->word_index) {
+		dpsetalign (priv->word_index, 8);
+
+		/* Re optimize database if bucket count < rec count */
+		bucket_count = dpbnum (priv->word_index);
+		rec_count = dprnum (priv->word_index);
+		
+		g_message ("Bucket count (max is %d) is %d and record count is %d", 
+			   priv->max_bucket,
+			   bucket_count, 
+			   rec_count);
 	}
 
-	return word_index;
+	return (priv->word_index != NULL);
+
 }
 
 static inline gboolean 
@@ -431,8 +498,6 @@
 {
         TrackerIndexer        *indexer;
         TrackerIndexerPrivate *priv;
-	gint                   bucket_count;
-        gint                   rec_count;
 
 	indexer = g_object_new (TRACKER_TYPE_INDEXER,
 				"name", filename,
@@ -440,25 +505,36 @@
 				"max-bucket", max_bucket,
                                 NULL);
 
-        priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
 
-	priv->word_index = open_index (filename,
-                                       min_bucket,
-                                       max_bucket);
-	dpsetalign (priv->word_index, 8);
-
-	/* Re optimize database if bucket count < rec count */
-	bucket_count = dpbnum (priv->word_index);
-	rec_count = dprnum (priv->word_index);
-
-	g_message ("Bucket count (max is %d) is %d and record count is %d", 
-                   max_bucket,
-                   bucket_count, 
-                   rec_count);
+	priv->reload = !tracker_indexer_open_index (indexer);
        
         return indexer;
 }
 
+static gboolean
+indexer_is_up_to_date (TrackerIndexer *indexer)
+{
+        TrackerIndexerPrivate *priv;
+
+        g_return_val_if_fail (TRACKER_IS_INDEXER (indexer), 0);
+
+        priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
+
+	if (priv->reload || !priv->word_index) {
+		if (priv->word_index) {
+			dpclose (priv->word_index);
+			priv->word_index = NULL;
+		}
+		priv->reload = !tracker_indexer_open_index (indexer);
+		g_message ("Reloading the index %s", priv->name);
+	}
+
+	return !priv->reload;
+
+}
+
+
 guint32
 tracker_indexer_get_size (TrackerIndexer *indexer)
 {
@@ -467,6 +543,9 @@
 
         g_return_val_if_fail (TRACKER_IS_INDEXER (indexer), 0);
 
+	if (!indexer_is_up_to_date (indexer)) {
+		return 0;
+	}
         priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
 
         g_mutex_lock (priv->word_mutex);
@@ -492,6 +571,10 @@
         g_return_val_if_fail (TRACKER_IS_INDEXER (indexer), NULL);
         g_return_val_if_fail (term != NULL, NULL);
 
+	if (!indexer_is_up_to_date (indexer)) {
+		return NULL;
+	}
+
 	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
 
 	winner_str = g_strdup (term);
@@ -559,6 +642,10 @@
 
 	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
 
+	if (!indexer_is_up_to_date (indexer)) {
+		return NULL;
+	}
+
 	g_mutex_lock (priv->word_mutex);
 
 	details = NULL;
@@ -582,7 +669,10 @@
 	return details;
 }
 
-/* Use to delete dud hits for a word - dud_list is a list of
+/*
+ * UNUSED
+ *
+ *  Use to delete dud hits for a word - dud_list is a list of
  * TrackerSearchHit structs.
  */
 gboolean
@@ -599,6 +689,10 @@
 	g_return_val_if_fail (word, FALSE);
 	g_return_val_if_fail (dud_list, FALSE);
 
+	if (!indexer_is_up_to_date (indexer)) {
+		return TRUE;
+	}
+
 	priv = TRACKER_INDEXER_GET_PRIVATE (indexer);
 	
 	/* Check if existing record is there  */

Modified: branches/indexer-split/src/trackerd/tracker-indexer.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-indexer.h	(original)
+++ branches/indexer-split/src/trackerd/tracker-indexer.h	Fri Jul 25 13:15:13 2008
@@ -50,26 +50,30 @@
 };
 
 
-GType                      tracker_indexer_get_type                      (void);
-TrackerIndexer *           tracker_indexer_new                           (const gchar               *name,
-									  gint                       min_bucket,
-									  gint                       max_bucket);
-void                       tracker_indexer_set_name                      (TrackerIndexer            *indexer,
-									  const gchar               *name);
-void                       tracker_indexer_set_min_bucket                (TrackerIndexer            *indexer,
-									  gint                       min_bucket);
-void                       tracker_indexer_set_max_bucket                (TrackerIndexer            *indexer,
-									  gint                       max_bucket);
-guint32                    tracker_indexer_get_size                      (TrackerIndexer            *indexer);
-char *                     tracker_indexer_get_suggestion                (TrackerIndexer            *indexer,
-									  const gchar               *term,
-									  gint                       maxdist);
-TrackerIndexItem *         tracker_indexer_get_word_hits                 (TrackerIndexer            *indexer,
-									  const gchar               *word,
-									  guint                     *count);
-gboolean                   tracker_indexer_remove_dud_hits               (TrackerIndexer            *indexer,
-									  const gchar               *word,
-									  GSList                    *dud_list);
+GType             tracker_indexer_get_type        (void);
+TrackerIndexer *  tracker_indexer_new             (const gchar    *name,
+						   gint            min_bucket,
+						   gint            max_bucket);
+void              tracker_indexer_set_name        (TrackerIndexer *indexer,
+						   const gchar    *name);
+void              tracker_indexer_set_min_bucket  (TrackerIndexer *indexer,
+						   gint            min_bucket);
+void              tracker_indexer_set_max_bucket  (TrackerIndexer *indexer,
+						   gint            max_bucket);
+void              tracker_indexer_set_reload      (TrackerIndexer *indexer,
+						   gboolean        reload);
+gboolean          tracker_indexer_get_reload      (TrackerIndexer *indexer);
+guint32           tracker_indexer_get_size        (TrackerIndexer *indexer);
+char *            tracker_indexer_get_suggestion  (TrackerIndexer *indexer,
+						   const gchar    *term,
+						   gint            maxdist);
+TrackerIndexItem *tracker_indexer_get_word_hits   (TrackerIndexer *indexer,
+						   const gchar    *word,
+						   guint          *count);
+gboolean          tracker_indexer_remove_dud_hits (TrackerIndexer *indexer,
+						   const gchar    *word,
+						   GSList         *dud_list);
+
 
 G_END_DECLS
 

Modified: branches/indexer-split/tests/trackerd/Makefile.am
==============================================================================
--- branches/indexer-split/tests/trackerd/Makefile.am	(original)
+++ branches/indexer-split/tests/trackerd/Makefile.am	Fri Jul 25 13:15:13 2008
@@ -14,6 +14,7 @@
 	$(GTHREAD_CFLAGS) \
 	$(GLIB2_CFLAGS) \
 	$(QDBM_CFLAGS) \
+	$(GIO_GFLAGS) \
 	$(DBUS_CFLAGS)
 
 if HAVE_XESAM_GLIB
@@ -54,5 +55,6 @@
 	$(GTHREAD_LIBS) \
 	$(GLIB2_LIBS) \
 	$(DBUS_LIBS) \
+	$(GIO_LIBS) \
 	$(QDBM_LIBS) 
 

Modified: branches/indexer-split/tests/trackerd/tracker-indexer-test.c
==============================================================================
--- branches/indexer-split/tests/trackerd/tracker-indexer-test.c	(original)
+++ branches/indexer-split/tests/trackerd/tracker-indexer-test.c	Fri Jul 25 13:15:13 2008
@@ -1,6 +1,7 @@
 #include <glib.h>
 #include <glib/gtestutils.h>
 #include <tracker-test-helpers.h>
+#include <gio/gio.h>
 
 #include "tracker-indexer.h"
 
@@ -16,6 +17,8 @@
         indexer = tracker_indexer_new ("./example.index", 
                                        DEFAULT_MIN_BUCKET_COUNT,
                                        DEFAULT_MAX_BUCKET_COUNT);
+        g_assert (!tracker_indexer_get_reload (indexer));
+
         suggestion = tracker_indexer_get_suggestion (indexer, "Thiz", 9);
 
         g_assert (tracker_test_helpers_cmpstr_equal (suggestion, "this"));
@@ -25,6 +28,76 @@
         g_object_unref (indexer);
 }
 
+static void
+test_reloading ()
+{
+        TrackerIndexer   *indexer;
+        guint              count;
+        TrackerIndexItem *hits;
+
+        indexer = tracker_indexer_new ("./example.index", 
+                                       DEFAULT_MIN_BUCKET_COUNT,
+                                       DEFAULT_MAX_BUCKET_COUNT);
+
+        tracker_indexer_set_reload (indexer, TRUE);
+        g_assert (tracker_indexer_get_reload (indexer)); /* Trivial check of get/set */
+
+        if (g_test_trap_fork (0, G_TEST_TRAP_SILENCE_STDERR)) {
+                hits = tracker_indexer_get_word_hits (indexer, "this", &count);
+                g_free (hits);
+        }
+        g_test_trap_assert_stderr ("*Reloading the index ./example.index*");
+
+}
+
+static void
+test_bad_index ()
+{
+        TrackerIndexer *indexer;
+        guint            count;
+
+        indexer = tracker_indexer_new ("unknown-index",
+                                       DEFAULT_MIN_BUCKET_COUNT,
+                                       DEFAULT_MAX_BUCKET_COUNT);
+
+        /* Reload true: it cannot open the index */
+        g_assert (tracker_indexer_get_reload (indexer));
+
+        /* Return NULL, the index cannot reload the file */
+        g_assert (!tracker_indexer_get_word_hits (indexer, "this", &count));
+
+        /* Return NULL, the index cannot reload the file */
+        g_assert (!tracker_indexer_get_suggestion (indexer, "Thiz", 9));
+
+}
+
+static void
+test_created_file_in_the_mean_time ()
+{
+        TrackerIndexer *indexer;
+        GFile          *good, *bad;
+        guint           count;
+
+        indexer = tracker_indexer_new ("./unknown-index",
+                                       DEFAULT_MIN_BUCKET_COUNT,
+                                       DEFAULT_MAX_BUCKET_COUNT);
+
+        /* Reload true: it cannot open the index */
+        g_assert (tracker_indexer_get_reload (indexer));
+
+        good = g_file_new_for_path ("./example.index");
+        bad = g_file_new_for_path ("./unknown-index");
+
+        g_file_copy (good, bad, G_FILE_COPY_OVERWRITE, NULL, NULL, NULL, NULL);
+
+        /* Now the first operation reload the index */
+        g_assert (tracker_indexer_get_word_hits (indexer, "this", &count));
+        
+        /* Reload false: It is already reloaded */
+        g_assert (!tracker_indexer_get_reload (indexer));
+
+        g_file_delete (bad, NULL, NULL);
+}
 
 
 int
@@ -40,6 +113,10 @@
 
         g_test_add_func ("/trackerd/tracker-indexer/get_suggestion",
                          test_get_suggestion );
+        g_test_add_func ("/trackerd/tracker-indexer/reloading",
+                         test_reloading );
+        g_test_add_func ("/trackerd/tracker-indexer/bad_index",
+                         test_bad_index );
 
         result = g_test_run ();
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]