tracker r1674 - in branches/indexer-split: . src/libtracker-common src/trackerd

[mr svn gnome org]

Author: mr
Date: Fri Jun 13 10:23:03 2008
New Revision: 1674
URL: http://svn.gnome.org/viewvc/tracker?rev=1674&view=rev

Log:
	* src/libtracker-common/tracker-file-utils.[ch]:
	* src/trackerd/tracker-crawler.c: Moved
	tracker_path_list_filter_duplicates() to the file utils.

	* src/trackerd/tracker-db.c: 
	* src/trackerd/tracker-email-kmail.c:
	* src/trackerd/tracker-process-files.c: Removed all
	tracker_monitor_() calls, this shouldn't be done from here.


Modified:
   branches/indexer-split/ChangeLog
   branches/indexer-split/src/libtracker-common/tracker-file-utils.c
   branches/indexer-split/src/libtracker-common/tracker-file-utils.h
   branches/indexer-split/src/trackerd/tracker-crawler.c
   branches/indexer-split/src/trackerd/tracker-db.c
   branches/indexer-split/src/trackerd/tracker-email-kmail.c
   branches/indexer-split/src/trackerd/tracker-monitor.c
   branches/indexer-split/src/trackerd/tracker-monitor.h
   branches/indexer-split/src/trackerd/tracker-process-files.c

Modified: branches/indexer-split/src/libtracker-common/tracker-file-utils.c
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-file-utils.c	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-file-utils.c	Fri Jun 13 10:23:03 2008
@@ -467,3 +467,72 @@
 	g_slist_foreach (dirs_to_remove, (GFunc) g_free, NULL);
 	g_slist_free (dirs_to_remove);
 }
+
+GSList *
+tracker_path_list_filter_duplicates (GSList *roots)
+{
+	GSList *checked_roots = NULL;
+	GSList *l1, *l2;
+
+	/* ONLY HERE do we add separators on each location we check.
+	 * The reason for this is that these locations are user
+	 * entered in the configuration and we need to make sure we
+	 * don't include the same location more than once.
+	 */
+
+	for (l1 = roots; l1; l1 = l1->next) {
+		gchar    *path;
+		gboolean  should_add = TRUE;
+
+		if (!g_str_has_suffix (l1->data, G_DIR_SEPARATOR_S)) {
+			path = g_strconcat (l1->data, G_DIR_SEPARATOR_S, NULL);
+		} else {
+			path = g_strdup (l1->data);
+		}
+
+		l2 = checked_roots;
+
+		while (l2 && should_add) {
+			/* If the new path exists as a lower level
+			 * path or is the same as an existing checked
+			 * root we disgard it, it will be checked
+			 * anyway. 
+			 */
+			if (g_str_has_prefix (path, l2->data)) {
+				should_add = FALSE;
+			} 
+
+			/* If the new path exists as a higher level
+			 * path to one already in the checked roots,
+			 * we remove the checked roots version
+			 */
+			if (g_str_has_prefix (l2->data, path)) {
+				checked_roots = g_slist_remove_link (checked_roots, l2);
+				g_free (l2->data);
+				l2 = checked_roots;
+				continue;
+			}
+
+			l2 = l2->next;
+		}
+		
+		if (should_add) {
+			checked_roots = g_slist_prepend (checked_roots, path);
+			continue;
+		} 
+		
+		g_free (path);
+	}
+
+	checked_roots = g_slist_reverse (checked_roots);
+
+#ifdef TESTING
+	g_debug ("Using the following roots to crawl:");
+
+	for (l1 = checked_roots; l1; l1 = l1->next) {
+		g_debug ("  %s", (gchar*) l1->data);
+	}
+#endif /* TESTING */
+
+	return checked_roots;
+}

Modified: branches/indexer-split/src/libtracker-common/tracker-file-utils.h
==============================================================================
--- branches/indexer-split/src/libtracker-common/tracker-file-utils.h	(original)
+++ branches/indexer-split/src/libtracker-common/tracker-file-utils.h	Fri Jun 13 10:23:03 2008
@@ -24,20 +24,20 @@
 
 #include <glib.h>
 
-gint     tracker_file_open          (const gchar *uri,
-				     gboolean     readahead);
-void     tracker_file_close         (gint         fd,
-				     gboolean     no_longer_needed);
-gboolean tracker_file_unlink        (const gchar *uri);
-gboolean tracker_file_is_valid      (const gchar *uri);
-gboolean tracker_file_is_directory  (const gchar *uri);
-gboolean tracker_file_is_indexable  (const gchar *uri);
-guint32  tracker_file_get_size      (const gchar *uri);
-gint32   tracker_file_get_mtime     (const gchar *uri);
-gchar *  tracker_file_get_mime_type (const gchar *uri);
-gchar *  tracker_file_get_vfs_path  (const gchar *uri);
-gchar *  tracker_file_get_vfs_name  (const gchar *uri);
-
-void     tracker_path_remove        (const gchar *uri);
+gint     tracker_file_open                   (const gchar *uri,
+					      gboolean     readahead);
+void     tracker_file_close                  (gint         fd,
+					      gboolean     no_longer_needed);
+gboolean tracker_file_unlink                 (const gchar *uri);
+gboolean tracker_file_is_valid               (const gchar *uri);
+gboolean tracker_file_is_directory           (const gchar *uri);
+gboolean tracker_file_is_indexable           (const gchar *uri);
+guint32  tracker_file_get_size               (const gchar *uri);
+gint32   tracker_file_get_mtime              (const gchar *uri);
+gchar *  tracker_file_get_mime_type          (const gchar *uri);
+gchar *  tracker_file_get_vfs_path           (const gchar *uri);
+gchar *  tracker_file_get_vfs_name           (const gchar *uri);
+void     tracker_path_remove                 (const gchar *uri);
+GSList * tracker_path_list_filter_duplicates (GSList      *roots);
 
 #endif /* __LIBTRACKER_COMMON_FILE_UTILS_H__ */

Modified: branches/indexer-split/src/trackerd/tracker-crawler.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-crawler.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-crawler.c	Fri Jun 13 10:23:03 2008
@@ -24,12 +24,14 @@
 
 #include <gio/gio.h>
 
-#include <libtracker-common/tracker-utils.h>
 #include <libtracker-common/tracker-dbus.h>
+#include <libtracker-common/tracker-file-utils.h>
+#include <libtracker-common/tracker-utils.h>
 
 #include "tracker-crawler.h"
 #include "tracker-dbus.h"
 #include "tracker-indexer-client.h"
+#include "tracker-monitor.h"
 
 #define GET_PRIV(obj) (G_TYPE_INSTANCE_GET_PRIVATE ((obj), TRACKER_TYPE_CRAWLER, TrackerCrawlerPriv))
 
@@ -653,7 +655,7 @@
 	GFileEnumerator *enumerator;
 	GFileInfo       *info;
 	GFile           *parent, *child;
-	gchar           *relative_path;
+	gchar           *path;
 
 	crawler = TRACKER_CRAWLER (user_data);
 	parent = G_FILE (file);
@@ -670,30 +672,30 @@
 	     info && crawler->priv->running;
 	     info = g_file_enumerator_next_file (enumerator, NULL, NULL)) {
 		child = g_file_get_child (parent, g_file_info_get_name (info));
-		relative_path = g_file_get_path (child);
+		path = g_file_get_path (child);
 		
-		if (path_should_be_ignored (crawler, relative_path)) {
+		if (path_should_be_ignored (crawler, path)) {
 			crawler->priv->files_ignored++;
 #ifdef TESTING
 			g_debug ("Ignored:'%s' (%d)",  
-				 relative_path, 
+				 path, 
 				 crawler->priv->enumerations); 
 #endif /* TESTING */
-			g_free (relative_path);
+			g_free (path);
 		} else {
 			crawler->priv->files_found++;
 #ifdef TESTING
 			g_debug ("Found  :'%s' (%d)", 
-				 relative_path, 
+				 path, 
 				 crawler->priv->enumerations);
 #endif /* TESTING */
 
 			if (g_file_info_get_file_type (info) == G_FILE_TYPE_DIRECTORY) {
 				file_enumerate (crawler, child);
-				g_free (relative_path);
+				g_free (path);
 			} else {
 				g_async_queue_push (crawler->priv->files,
-						    relative_path);
+						    path);
 			}
 		}	
 
@@ -718,6 +720,8 @@
 {
 	file_enumerators_increment (crawler);
 
+	tracker_monitor_add (file);
+
 	g_file_enumerate_children_async (file, 
 					 "*",
 					 G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
@@ -736,11 +740,6 @@
 	gchar **        files;
 	gboolean        running;
 	gint            length;
-#if 0
-	gint            items;
-	GPtrArray      *ptr_array;
-	gint            i;
-#endif
 
 	crawler = user_data;
 
@@ -787,76 +786,6 @@
 	return TRUE;
 }
 
-static GSList *
-check_roots_dont_coinside (TrackerCrawler *crawler,
-			   GSList         *roots)
-{
-	GSList *checked_roots = NULL;
-	GSList *l1, *l2;
-
-	/* ONLY HERE do we add separators on each location we check.
-	 * The reason for this is that these locations are user
-	 * entered in the configuration and we need to make sure we
-	 * don't include the same location more than once.
-	 */
-
-	for (l1 = roots; l1; l1 = l1->next) {
-		gchar    *path;
-		gboolean  should_add = TRUE;
-
-		if (!g_str_has_suffix (l1->data, G_DIR_SEPARATOR_S)) {
-			path = g_strconcat (l1->data, G_DIR_SEPARATOR_S, NULL);
-		} else {
-			path = g_strdup (l1->data);
-		}
-
-		l2 = checked_roots;
-
-		while (l2 && should_add) {
-			/* If the new path exists as a lower level
-			 * path or is the same as an existing checked
-			 * root we disgard it, it will be checked
-			 * anyway. 
-			 */
-			if (g_str_has_prefix (path, l2->data)) {
-				should_add = FALSE;
-			} 
-
-			/* If the new path exists as a higher level
-			 * path to one already in the checked roots,
-			 * we remove the checked roots version
-			 */
-			if (g_str_has_prefix (l2->data, path)) {
-				checked_roots = g_slist_remove_link (checked_roots, l2);
-				g_free (l2->data);
-				l2 = checked_roots;
-				continue;
-			}
-
-			l2 = l2->next;
-		}
-		
-		if (should_add) {
-			checked_roots = g_slist_prepend (checked_roots, path);
-			continue;
-		} 
-		
-		g_free (path);
-	}
-
-	checked_roots = g_slist_reverse (checked_roots);
-
-#ifdef TESTING
-	g_debug ("Using the following roots to crawl:");
-
-	for (l1 = checked_roots; l1; l1 = l1->next) {
-		g_debug ("  %s", (gchar*) l1->data);
-	}
-#endif /* TESTING */
-
-	return checked_roots;
-}
-
 void
 tracker_crawler_start (TrackerCrawler *crawler)
 {
@@ -886,7 +815,7 @@
         config_roots = tracker_config_get_crawl_directory_roots (priv->config);
 	if (config_roots) {
 		/* Make sure none of the roots co-inside each other */
-		roots = check_roots_dont_coinside (crawler, config_roots);
+		roots = tracker_path_list_filter_duplicates (config_roots);
 	}
 
 	if (!roots) {

Modified: branches/indexer-split/src/trackerd/tracker-db.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-db.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-db.c	Fri Jun 13 10:23:03 2008
@@ -555,13 +555,19 @@
 		const gchar        *moved_from_uri, 
 		const gchar        *moved_to_uri)
 {
+	/* FIXME: the monitor updates should not be done here, -mr */
+
+#if 0
 	/* Stop watching old dir, start watching new dir */
 	tracker_monitor_remove (moved_from_uri, TRUE);
+#endif
 		
 	tracker_db_file_move (iface, moved_from_uri, moved_to_uri);
 	directory_move_files (iface, moved_from_uri, moved_to_uri);
 
+#if 0
 	tracker_monitor_add (moved_to_uri);
+#endif
 }
 
 static gint 

Modified: branches/indexer-split/src/trackerd/tracker-email-kmail.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-email-kmail.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-email-kmail.c	Fri Jun 13 10:23:03 2008
@@ -663,7 +663,7 @@
                     (( in_imap_dir && dir_name[0] == '.' && g_str_has_suffix (dir_name, ".directory")) ||
                       !in_imap_dir )
                     ) {
-                        if (!tracker_monitor_is_watched (dir_path)) {
+                        if (!tracker_monitor_is_watched_by_string (dir_path)) {
                                 /* if we are in a maildir directory, we will only index emails in directory "cur" */
                                 gchar *dir_cur = g_build_filename (dir_path, "cur", NULL);
 

Modified: branches/indexer-split/src/trackerd/tracker-monitor.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-monitor.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-monitor.c	Fri Jun 13 10:23:03 2008
@@ -18,7 +18,9 @@
  * Boston, MA  02110-1301, USA.
  */
 
-#include <gio/gio.h>
+#include <string.h>
+
+#include <libtracker-common/tracker-file-utils.h>
 
 #include "tracker-monitor.h"
 
@@ -44,9 +46,9 @@
 	}
 	
 	if (!monitors) {
-		monitors = g_hash_table_new_full (g_str_hash,
-						  g_str_equal,
-						  g_free,
+		monitors = g_hash_table_new_full (g_file_hash,
+						  (GEqualFunc) g_file_equal,
+						  (GDestroyNotify) g_object_unref,
 						  (GDestroyNotify) g_file_monitor_cancel);
 	}
 
@@ -67,16 +69,69 @@
 	}
 }
 
+static const gchar *
+monitor_event_to_string (GFileMonitorEvent event_type)
+{
+	switch (event_type) {
+	case G_FILE_MONITOR_EVENT_CHANGED:
+		return "G_FILE_MONITOR_EVENT_CHANGED";
+	case G_FILE_MONITOR_EVENT_CHANGES_DONE_HINT:
+		return "G_FILE_MONITOR_EVENT_CHANGES_DONE_HINT";
+	case G_FILE_MONITOR_EVENT_DELETED:
+		return "G_FILE_MONITOR_EVENT_DELETED";
+	case G_FILE_MONITOR_EVENT_CREATED:
+		return "G_FILE_MONITOR_EVENT_CREATED";
+	case G_FILE_MONITOR_EVENT_ATTRIBUTE_CHANGED:
+		return "G_FILE_MONITOR_EVENT_ATTRIBUTE_CHANGED";
+	case G_FILE_MONITOR_EVENT_PRE_UNMOUNT:
+		return "G_FILE_MONITOR_EVENT_PRE_UNMOUNT";
+	case G_FILE_MONITOR_EVENT_UNMOUNTED:
+		return "G_FILE_MONITOR_EVENT_UNMOUNTED";
+	}
+
+	return "unknown";
+}
+
+static void
+monitor_event_cb (GFileMonitor     *monitor,
+		  GFile            *file,
+		  GFile            *other_file,
+		  GFileMonitorEvent event_type,
+		  gpointer          user_data)  
+{
+	gchar *str1;
+	gchar *str2;
+
+	str1 = g_file_get_path (file);
+	
+	if (other_file) {
+		str2 = g_file_get_path (other_file);
+	} else {
+		str2 = g_strdup ("");
+	}
+
+	g_message ("Received monitor event:%d->'%s' for file:'%s' and other file:'%s'",
+		   event_type,
+		   monitor_event_to_string (event_type),
+		   str1,
+		   str2);
+		   
+	g_free (str1);
+	g_free (str2);
+}
+
 gboolean
-tracker_monitor_add (const gchar *path)
+tracker_monitor_add (GFile *file)
 {
-	GFile        *file;
 	GFileMonitor *monitor;
+	GSList       *ignored_roots;
+	GSList       *l;
 	GError       *error = NULL;
+	gchar        *path;
 
-	g_return_val_if_fail (path != NULL, FALSE);
+	g_return_val_if_fail (G_IS_FILE (file), FALSE);
 
-	if (g_hash_table_lookup (monitors, path)) {
+	if (g_hash_table_lookup (monitors, file)) {
 		return TRUE;
 	}
 
@@ -88,63 +143,101 @@
 		return FALSE;
 	}
 
+	path = g_file_get_path (file);
+
+	ignored_roots = tracker_config_get_no_watch_directory_roots (config);
+	/* Check this location isn't excluded in the config */
+	for (l = ignored_roots; l; l = l->next) {
+		if (strcmp (path, l->data) == 0) {
+			g_message ("Not adding montior for:'%s', path is in config ignore list",
+				   path);
+			g_free (path);
+			return FALSE;
+		}
+	}
+
 	/* We don't check if a file exists or not since we might want
 	 * to monitor locations which don't exist yet.
+	 *
+	 * Also, we assume ALL paths passed are directories.
 	 */
-	file = g_file_new_for_path (path);
 	monitor = g_file_monitor_directory (file,
 					    G_FILE_MONITOR_WATCH_MOUNTS,
 					    NULL,
 					    &error);
-	g_object_unref (file);
 
 	if (error) {
 		g_warning ("Could not add monitor for path:'%s', %s", 
 			   path, 
 			   error->message);
+		g_free (path);
 		g_error_free (error);
 		return FALSE;
 	}
 
-	g_debug ("Added monitor for:'%s', total monitors:%d", 
-		 path,
-		 g_hash_table_size (monitors));
+	g_signal_connect (monitor, "changed",
+			  G_CALLBACK (monitor_event_cb),
+			  NULL);
 
 	g_hash_table_insert (monitors, 
-			     g_strdup (path), 
+			     g_object_ref (file), 
 			     monitor);
+
+	g_message ("Added monitor for:'%s', total monitors:%d", 
+		   path,
+		   g_hash_table_size (monitors));
+	
+	g_free (path);
 	
 	return TRUE;
 }
 
 gboolean
-tracker_monitor_remove (const gchar *path,
-			gboolean     delete_subdirs)
+tracker_monitor_remove (GFile    *file,
+			gboolean  delete_subdirs)
 {
 	GFileMonitor *monitor;
+	gchar        *path;
 
-	g_return_val_if_fail (path != NULL, FALSE);
+	g_return_val_if_fail (G_IS_FILE (file), FALSE);
 
-	monitor = g_hash_table_lookup (monitors, path);
+	monitor = g_hash_table_lookup (monitors, file);
 	if (!monitor) {
 		return TRUE;
 	}
 
-	g_hash_table_remove (monitors, path);
+	g_hash_table_remove (monitors, file);
 
-	g_debug ("Removed monitor for:'%s', total monitors:%d", 
-		 path,
-		 g_hash_table_size (monitors));
+	path = g_file_get_path (file);
+	g_message ("Removed monitor for:'%s', total monitors:%d", 
+		   path,
+		   g_hash_table_size (monitors));
+	g_free (path);
 
 	return TRUE;
 }
 
 gboolean
-tracker_monitor_is_watched (const gchar *path)
+tracker_monitor_is_watched (GFile *file)
 {
+	g_return_val_if_fail (G_IS_FILE (file), FALSE);
+
+	return g_hash_table_lookup (monitors, file) != NULL;
+}
+
+gboolean
+tracker_monitor_is_watched_by_string (const gchar *path)
+{
+	GFile    *file;
+	gboolean  watched;
+
 	g_return_val_if_fail (path != NULL, FALSE);
 
-	return g_hash_table_lookup (monitors, path) != NULL;
+	file = g_file_new_for_path (path);
+	watched = g_hash_table_lookup (monitors, file) != NULL;
+	g_object_unref (file);
+
+	return watched;
 }
 
 gint

Modified: branches/indexer-split/src/trackerd/tracker-monitor.h
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-monitor.h	(original)
+++ branches/indexer-split/src/trackerd/tracker-monitor.h	Fri Jun 13 10:23:03 2008
@@ -22,19 +22,21 @@
 #ifndef __TRACKERD_MONITOR_H__
 #define __TRACKERD_MONITOR_H__
 
-#include <libtracker-common/tracker-config.h>
+#include <gio/gio.h>
 
-#include "tracker-db.h"
+#include <libtracker-common/tracker-config.h>
 
 G_BEGIN_DECLS
 
-gboolean tracker_monitor_init       (TrackerConfig *config);
-void     tracker_monitor_shutdown   (void);
-gboolean tracker_monitor_add        (const gchar   *path);
-gboolean tracker_monitor_remove     (const gchar   *path,
-				     gboolean       delete_subdirs);
-gboolean tracker_monitor_is_watched (const gchar   *path);
-gint     tracker_monitor_get_count  (void);
+gboolean tracker_monitor_init                 (TrackerConfig *config);
+void     tracker_monitor_shutdown             (void);
+gboolean tracker_monitor_add                  (GFile         *file);
+gboolean tracker_monitor_remove               (GFile         *file,
+					       gboolean       delete_subdirs);
+gboolean tracker_monitor_is_watched           (GFile         *file);
+gboolean tracker_monitor_is_watched_by_string (const gchar   *path);
+gint     tracker_monitor_get_count            (void);
+
 
 G_END_DECLS
 

Modified: branches/indexer-split/src/trackerd/tracker-process-files.c
==============================================================================
--- branches/indexer-split/src/trackerd/tracker-process-files.c	(original)
+++ branches/indexer-split/src/trackerd/tracker-process-files.c	Fri Jun 13 10:23:03 2008
@@ -251,7 +251,10 @@
                                 continue;
                         }
                         
+#if 0
+                        /* Done in the crawler module now */
                         tracker_monitor_add (dir);
+#endif
 		}
 
                 for (l = list; l; l = l->next) {
@@ -398,7 +401,10 @@
 
 	tracker_db_directory_delete (iface, info->file_id, info->uri);
 
+#if 0
+        /* Done in the crawler module now */
 	tracker_monitor_remove (info->uri, TRUE);
+#endif
 
 	g_message ("Deleting directory:'%s' and subdirs", info->uri);
 }
@@ -681,7 +687,10 @@
                 break;
                 
         case TRACKER_DB_ACTION_DIRECTORY_MOVED_FROM:
-                tracker_db_directory_move (iface, info->uri, info->moved_to_uri);
+#if 0
+                /* We should be sending this crap to the indexer */
+                tracker_db_directory_move (iface, info->uri, info->moved_to_uri); 
+#endif
                 need_index = FALSE;
                 break;