tracker r3026 - in trunk: . data/db src/libtracker-data src/libtracker-db src/tracker-indexer src/trackerd

[carlosg svn gnome org]

Author: carlosg
Date: Fri Mar  6 15:16:15 2009
New Revision: 3026
URL: http://svn.gnome.org/viewvc/tracker?rev=3026&view=rev

Log:
2009-03-06  Carlos Garnacho  <carlos imendio com>

	Clean up deleted elements from dbs/index on tracker-indexer idle time.
	This is the equivalent to dud hits removal, which trackerd can't
	perform since it opens the index files as readonly. Now services are
	just marked as disabled on deletion. So the indexer cleans them up
	slowly, after all elements have been processed, and before it shuts
	down.

	* data/db/sqlite-service.sql: Create the new table for deleted
	elements.
	* data/db/sqlite-stored-procs.sql: Add stored procedures for disabling
	services and dealing with the DeletedServices table.

	* src/libtracker-data/tracker-data-query.[ch]
	(tracker_data_query_metadata_fields): Do not have service ID as a
	string argument.
	(tracker_data_query_first_removed_service): New function to retrieve a
	deleted file.

	* src/trackerd/tracker-metadata.c (tracker_metadata_get): Adapt to API
	change in tracker-data-query.[ch]

	* src/libtracker-data/tracker-data-search.c
	(tracker_data_search_text): Remove code to clean up dud hits.

	* src/libtracker-data/tracker-data-update.[ch]
	(tracker_data_update_disable_service): Add function to disable a
	service.
	(tracker_data_update_delete_service): Also remove the service from the
	DeletedServices table.

	* src/libtracker-db/tracker-db-index.[ch] (tracker_db_index_finalize):
	Ensure the index is opened before flushing.
	(tracker_db_index_remove_dud_hits): Removed.

	* src/tracker-indexer/tracker-indexer.[ch] (cleanup_task_func)
	(cleanup_task_start) (cleanup_task_stop) (state_check): Add "cleanup"
	state
	(check_started): Also ensure the cleanup task is stopped.
	(check_finished): Launch the cleanup task from here.
	(item_remove): Renamed to item_mark_for_removal(). No longer delete
	things from the databases, just mark them as disabled. Queue children
	to be removed in the file queue.
	(item_erase): Added, actually delete the item from databases and
	index.
	(item_move): Erase immediately in case the dest path already exists.
	(tracker_indexer_get_running) (tracker_indexer_get_stoppable):
	Separate these a bit since the indexer now also must be paused during
	cleanup.

	* src/tracker-indexer/tracker-main.c (quit_timeout_cb): Use
	tracker_indexer_get_stoppable() to get to know whether the indexer
	should be stopped.

	* src/trackerd/tracker-dbus.c (indexer_name_owner_changed)
	(initialize_indexer_presence) (dbus_register_names): Listen for
	changes in the tracker-indexer presence...
	(dbus_request_new_cb): ... in order to pause it here if necessary,
	since now it should always be paused.

Modified:
   trunk/ChangeLog
   trunk/data/db/sqlite-service.sql
   trunk/data/db/sqlite-stored-procs.sql
   trunk/src/libtracker-data/tracker-data-query.c
   trunk/src/libtracker-data/tracker-data-query.h
   trunk/src/libtracker-data/tracker-data-search.c
   trunk/src/libtracker-data/tracker-data-update.c
   trunk/src/libtracker-data/tracker-data-update.h
   trunk/src/libtracker-db/tracker-db-index.c
   trunk/src/libtracker-db/tracker-db-index.h
   trunk/src/tracker-indexer/tracker-indexer.c
   trunk/src/tracker-indexer/tracker-indexer.h
   trunk/src/tracker-indexer/tracker-main.c
   trunk/src/trackerd/tracker-dbus.c
   trunk/src/trackerd/tracker-metadata.c

Modified: trunk/data/db/sqlite-service.sql
==============================================================================
--- trunk/data/db/sqlite-service.sql	(original)
+++ trunk/data/db/sqlite-service.sql	Fri Mar  6 15:16:15 2009
@@ -85,3 +85,8 @@
 
 CREATE INDEX ServiceNumericMetaDataCompoundIndex ON ServiceNumericMetaData (ServiceID, MetaDataID, MetaDataValue);
 
+
+CREATE TABLE DeletedServices
+(
+        ID      Integer primary key not null
+);

Modified: trunk/data/db/sqlite-stored-procs.sql
==============================================================================
--- trunk/data/db/sqlite-stored-procs.sql	(original)
+++ trunk/data/db/sqlite-stored-procs.sql	Fri Mar  6 15:16:15 2009
@@ -45,6 +45,11 @@
 MoveService                    UPDATE Services SET Path = ?, Name = ? WHERE Path = ? AND Name = ?;
 MoveServiceChildren            UPDATE Services SET Path = replace (Path, ?, ?) WHERE Path = ? OR Path LIKE (? || '/%');
 
+DisableService                 UPDATE Services SET Enabled = 0 WHERE ID = ?;
+MarkServiceForRemoval          INSERT INTO DeletedServices (ID) VALUES (?);
+UnmarkServiceForRemoval        DELETE FROM DeletedServices WHERE ID = ?;
+GetFirstRemovedFile            SELECT ID FROM DeletedServices LIMIT 1;
+
 DeleteContent                  DELETE FROM ServiceContents WHERE ServiceID = ? AND MetadataId = ?;
 DeleteService1                 DELETE FROM Services WHERE ID = ?;
 DeleteServiceRecursively       DELETE FROM Services WHERE Path = ? OR Path LIKE (? || '/%');

Modified: trunk/src/libtracker-data/tracker-data-query.c
==============================================================================
--- trunk/src/libtracker-data/tracker-data-query.c	(original)
+++ trunk/src/libtracker-data/tracker-data-query.c	Fri Mar  6 15:16:15 2009
@@ -230,20 +230,25 @@
  */
 G_CONST_RETURN gchar *
 tracker_data_query_service_type_by_id (TrackerDBInterface *iface,
-				       const gchar	  *service_id)
+				       guint32             service_id)
 {
 	TrackerDBResultSet *result_set;
 	gint		    service_type_id;
+	gchar              *service_id_str;
 	const gchar	   *result = NULL;
 
 	g_return_val_if_fail (TRACKER_IS_DB_INTERFACE (iface), NULL);
-	g_return_val_if_fail (service_id != NULL, NULL);
+	g_return_val_if_fail (service_id > 0, NULL);
+
+	service_id_str = tracker_guint32_to_string (service_id);
 
 	result_set = tracker_data_manager_exec_proc (iface,
 					   "GetFileByID",
-					   service_id,
+					   service_id_str,
 					   NULL);
 
+	g_free (service_id_str);
+
 	if (result_set) {
 		tracker_db_result_set_get (result_set, 3, &service_type_id, -1);
 		g_object_unref (result_set);
@@ -729,3 +734,28 @@
 	return contents;
 }
 
+gboolean
+tracker_data_query_first_removed_service (TrackerDBInterface *iface,
+					  guint32            *service_id)
+{
+	TrackerDBResultSet *result_set;
+
+	result_set = tracker_db_interface_execute_procedure (iface, NULL,
+							     "GetFirstRemovedFile",
+							     NULL);
+
+	if (result_set) {
+		guint32 id;
+
+		tracker_db_result_set_get (result_set, 0, &id, -1);
+		g_object_unref (result_set);
+
+		if (service_id) {
+			*service_id = id;
+		}
+
+		return TRUE;
+	}
+
+	return FALSE;
+}

Modified: trunk/src/libtracker-data/tracker-data-query.h
==============================================================================
--- trunk/src/libtracker-data/tracker-data-query.h	(original)
+++ trunk/src/libtracker-data/tracker-data-query.h	Fri Mar  6 15:16:15 2009
@@ -73,10 +73,15 @@
 GHashTable *         tracker_data_query_service_children      (TrackerService      *service,
 							       const gchar         *dirname);
 
+/* Deleted files */
+gboolean             tracker_data_query_first_removed_service (TrackerDBInterface  *iface,
+							       guint32             *service_id);
+
+
 /* Service API */
 G_CONST_RETURN gchar *
                      tracker_data_query_service_type_by_id    (TrackerDBInterface  *iface,
-							       const gchar         *service_id);
+							       guint32              service_id);
 
 /* Files API */
 guint32              tracker_data_query_file_id               (const gchar         *service_type,

Modified: trunk/src/libtracker-data/tracker-data-search.c
==============================================================================
--- trunk/src/libtracker-data/tracker-data-search.c	(original)
+++ trunk/src/libtracker-data/tracker-data-search.c	Fri Mar  6 15:16:15 2009
@@ -59,7 +59,6 @@
 	gint		     count;
 	const gchar	    *procedure;
 	GArray		    *services = NULL;
-	GSList		    *duds = NULL;
 	guint		     i = 0;
 
 	g_return_val_if_fail (TRACKER_IS_DB_INTERFACE (iface), NULL);
@@ -159,9 +158,6 @@
 
 			g_free (path);
 			g_object_unref (result_set);
-		} else {
-			g_message ("Dud hit for search detected");
-			duds = g_slist_prepend (duds, &rank);
 		}
 	}
 
@@ -169,29 +165,6 @@
 		tracker_db_interface_end_transaction (iface);
 	}
 
-	/* Delete duds */
-	if (duds) {
-		TrackerDBIndex *file_index;
-		TrackerDBIndex *email_index;
-		GSList	       *words, *w;
-
-		words = tracker_query_tree_get_words (tree);
-		file_index = tracker_db_index_manager_get_index (TRACKER_DB_INDEX_FILE);
-		email_index = tracker_db_index_manager_get_index (TRACKER_DB_INDEX_EMAIL);
-
-		for (w = words; w; w = w->next) {
-			tracker_db_index_remove_dud_hits (file_index,
-							  (const gchar *) w->data,
-							  duds);
-			tracker_db_index_remove_dud_hits (email_index,
-							  (const gchar *) w->data,
-							  duds);
-		}
-
-		g_slist_free (words);
-		g_slist_free (duds);
-	}
-
 	g_object_unref (tree);
 	g_array_free (hits, TRUE);
 

Modified: trunk/src/libtracker-data/tracker-data-update.c
==============================================================================
--- trunk/src/libtracker-data/tracker-data-update.c	(original)
+++ trunk/src/libtracker-data/tracker-data-update.c	Fri Mar  6 15:16:15 2009
@@ -170,6 +170,34 @@
 }
 
 void
+tracker_data_update_disable_service (TrackerService *service,
+				     guint32         service_id)
+{
+	TrackerDBInterface *iface;
+	gchar *service_id_str;
+
+	g_return_if_fail (TRACKER_IS_SERVICE (service));
+	g_return_if_fail (service_id >= 1);
+
+	iface = tracker_db_manager_get_db_interface_by_type (tracker_service_get_name (service),
+							     TRACKER_DB_CONTENT_TYPE_METADATA);
+
+	service_id_str = tracker_guint32_to_string (service_id);
+
+	tracker_db_interface_execute_procedure (iface,
+						NULL,
+						"DisableService",
+						service_id_str,
+						NULL);
+	tracker_db_interface_execute_procedure (iface,
+						NULL,
+						"MarkServiceForRemoval",
+						service_id_str,
+						NULL);
+	g_free (service_id_str);
+}
+
+void
 tracker_data_update_delete_service (TrackerService *service,
 				    guint32	    service_id)
 {
@@ -192,6 +220,12 @@
 						service_id_str,
 						NULL);
 
+	/* Delete from cleanup maintenance table */
+	tracker_db_interface_execute_procedure (iface,
+						NULL,
+						"UnmarkServiceForRemoval",
+						service_id_str,
+						NULL);
 	g_free (service_id_str);
 }
 

Modified: trunk/src/libtracker-data/tracker-data-update.h
==============================================================================
--- trunk/src/libtracker-data/tracker-data-update.h	(original)
+++ trunk/src/libtracker-data/tracker-data-update.h	Fri Mar  6 15:16:15 2009
@@ -40,6 +40,8 @@
 							 const gchar         *dirname,
 							 const gchar         *basename,
 							 GHashTable          *metadata);
+void     tracker_data_update_disable_service            (TrackerService      *service,
+							 guint32              service_id);
 void     tracker_data_update_delete_service             (TrackerService      *service,
 							 guint32              service_id);
 void     tracker_data_update_delete_service_recursively (TrackerService      *service,

Modified: trunk/src/libtracker-db/tracker-db-index.c
==============================================================================
--- trunk/src/libtracker-db/tracker-db-index.c	(original)
+++ trunk/src/libtracker-db/tracker-db-index.c	Fri Mar  6 15:16:15 2009
@@ -192,8 +192,11 @@
 	indez = TRACKER_DB_INDEX (object);
 	priv = TRACKER_DB_INDEX_GET_PRIVATE (indez);
 
-	tracker_db_index_flush_sync (indez);
-	tracker_db_index_close (indez);
+	if (!priv->readonly) {
+		tracker_db_index_open (indez);
+		tracker_db_index_flush_sync (indez);
+		tracker_db_index_close (indez);
+	}
 
 	if (priv->idle_flush_id) {
 		g_source_remove (priv->idle_flush_id);
@@ -1256,95 +1259,3 @@
 
 	return priv->overloaded;
 }
-
-/*
- * UNUSED
- *
- *  Use to delete dud hits for a word - dud_list is a list of
- * TrackerSearchHit structs.
- */
-gboolean
-tracker_db_index_remove_dud_hits (TrackerDBIndex *indez,
-				  const gchar	 *word,
-				  GSList	 *dud_list)
-{
-	TrackerDBIndexPrivate *priv;
-	gchar		      *tmp;
-	gint		       tsiz;
-	gboolean	       retval = FALSE;
-
-	g_return_val_if_fail (indez != NULL, FALSE);
-	g_return_val_if_fail (word != NULL, FALSE);
-	g_return_val_if_fail (dud_list != NULL, FALSE);
-
-	if (!check_index_is_up_to_date (indez)) {
-		return TRUE;
-	}
-
-	priv = TRACKER_DB_INDEX_GET_PRIVATE (indez);
-
-	g_return_val_if_fail (priv->readonly == FALSE, FALSE);
-	g_return_val_if_fail (priv->index != NULL, FALSE);
-
-	/* Check if existing record is there  */
-	tmp = dpget (priv->index,
-		     word,
-		     -1,
-		     0,
-		     MAX_HIT_BUFFER,
-		     &tsiz);
-
-	if (!tmp) {
-		return FALSE;
-	}
-
-	if (tsiz >= (int) sizeof (TrackerDBIndexItem)) {
-		TrackerDBIndexItem *details;
-		gint		    wi, i, pnum;
-
-		details = (TrackerDBIndexItem *) tmp;
-		pnum = tsiz / sizeof (TrackerDBIndexItem);
-		wi = 0;
-
-		for (i = 0; i < pnum; i++) {
-			GSList *lst;
-
-			for (lst = dud_list; lst; lst = lst->next) {
-				TrackerDBIndexItemRank *rank = lst->data;
-
-				if (!rank) {
-					continue;
-				}
-
-				if (details[i].id == rank->service_id) {
-					gint k;
-
-					/* Shift all subsequent
-					 * records in array down one
-					 * place.
-					 */
-					for (k = i + 1; k < pnum; k++) {
-						details[k - 1] = details[k];
-					}
-
-					/* Make size of array one size
-					 * smaller.
-					 */
-					tsiz -= sizeof (TrackerDBIndexItem);
-					pnum--;
-
-					break;
-				}
-			}
-		}
-
-		dpput (priv->index, word, -1, (gchar *) details, tsiz, DP_DOVER);
-
-		retval = TRUE;
-	}
-
-	g_free (tmp);
-
-
-	return retval;
-}

Modified: trunk/src/libtracker-db/tracker-db-index.h
==============================================================================
--- trunk/src/libtracker-db/tracker-db-index.h	(original)
+++ trunk/src/libtracker-db/tracker-db-index.h	Fri Mar  6 15:16:15 2009
@@ -90,9 +90,7 @@
 						      gint	      weight);
 gboolean            tracker_db_index_get_flushing    (TrackerDBIndex *indez);
 gboolean            tracker_db_index_get_overloaded  (TrackerDBIndex *indez);
-gboolean	    tracker_db_index_remove_dud_hits (TrackerDBIndex *index,
-						      const gchar    *word,
-						      GSList	     *dud_list);
+
 
 G_END_DECLS
 

Modified: trunk/src/tracker-indexer/tracker-indexer.c
==============================================================================
--- trunk/src/tracker-indexer/tracker-indexer.c	(original)
+++ trunk/src/tracker-indexer/tracker-indexer.c	Fri Mar  6 15:16:15 2009
@@ -139,6 +139,7 @@
 	guint pause_for_duration_id;
 	guint signal_status_id;
 	guint flush_id;
+	guint cleanup_task_id;
 
 	guint items_indexed;
 	guint items_processed;
@@ -178,6 +179,7 @@
 	TRACKER_INDEXER_STATE_INDEX_OVERLOADED = 1 << 0,
 	TRACKER_INDEXER_STATE_PAUSED	= 1 << 1,
 	TRACKER_INDEXER_STATE_STOPPED	= 1 << 2,
+	TRACKER_INDEXER_STATE_CLEANUP   = 1 << 3
 };
 
 enum {
@@ -197,16 +199,14 @@
 };
 
 static gboolean process_func	       (gpointer	     data);
+static gboolean cleanup_task_func      (gpointer             user_data);
+
 static void	state_set_flags        (TrackerIndexer	    *indexer,
 					TrackerIndexerState  state);
 static void	state_unset_flags      (TrackerIndexer	    *indexer,
 					TrackerIndexerState  state);
 static void	state_check	       (TrackerIndexer	    *indexer);
 
-static void     item_remove            (TrackerIndexer      *indexer,
-					PathInfo	    *info,
-					const gchar         *dirname,
-					const gchar         *basename);
 static void     check_finished         (TrackerIndexer      *indexer,
 					gboolean             interrupted);
 
@@ -391,9 +391,11 @@
 schedule_flush (TrackerIndexer *indexer,
 		gboolean	immediately)
 {
+#if 0
 	if (indexer->private->state != 0) {
 		return;
 	}
+#endif
 
 	if (immediately) {
 		/* No need to wait for flush timeout */
@@ -536,6 +538,10 @@
 		g_source_remove (priv->signal_status_id);
 	}
 
+	if (priv->cleanup_task_id) {
+		g_source_remove (priv->cleanup_task_id);
+	}
+
 	if (priv->timer) {
 		g_timer_destroy (priv->timer);
 	}
@@ -712,7 +718,9 @@
 	}
 
 	indexer->private->interrupted = FALSE;
-	state_unset_flags (indexer, TRACKER_INDEXER_STATE_STOPPED);
+	state_unset_flags (indexer,
+			   TRACKER_INDEXER_STATE_STOPPED |
+			   TRACKER_INDEXER_STATE_CLEANUP);
 
 	if (indexer->private->timer) {
 		g_timer_destroy (indexer->private->timer);
@@ -770,6 +778,9 @@
 	indexer->private->items_indexed = 0;
 	indexer->private->items_to_index = 0;
 	indexer->private->subelements_processed = 0;
+
+	/* Setup clean up task */
+	state_set_flags (indexer, TRACKER_INDEXER_STATE_CLEANUP);
 }
 
 static void
@@ -826,6 +837,39 @@
 }
 
 static void
+cleanup_task_start (TrackerIndexer *indexer)
+{
+	TrackerIndexerPrivate *priv;
+
+	priv = indexer->private;
+
+	if (priv->cleanup_task_id == 0) {
+		priv->cleanup_task_id = g_timeout_add (500, cleanup_task_func, indexer);
+
+		/* Open indexes */
+		tracker_db_index_open (indexer->private->file_index);
+		tracker_db_index_open (indexer->private->email_index);
+	}
+}
+
+static void
+cleanup_task_stop (TrackerIndexer *indexer)
+{
+	TrackerIndexerPrivate *priv;
+
+	priv = indexer->private;
+
+	if (priv->cleanup_task_id != 0) {
+		g_source_remove (priv->cleanup_task_id);
+		priv->cleanup_task_id = 0;
+
+		/* close indexes */
+		tracker_db_index_close (indexer->private->file_index);
+		tracker_db_index_close (indexer->private->email_index);
+	}
+}
+
+static void
 tracker_indexer_load_modules (TrackerIndexer *indexer)
 {
 	TrackerIndexerPrivate *priv;
@@ -1687,6 +1731,76 @@
 }
 
 static void
+item_erase (TrackerIndexer *indexer,
+	    TrackerService *service,
+	    guint32         service_id)
+{
+	gchar *content, *metadata;
+	guint32 service_type_id;
+	TrackerDataMetadata *data_metadata;
+
+	service_type_id = tracker_service_get_id (service);
+
+	/* Get mime type and remove thumbnail from thumbnailerd */
+	data_metadata = tracker_data_query_metadata (service, service_id, TRUE);
+
+	if (data_metadata) {
+		const gchar *path, *mime_type;
+		GFile *file;
+		gchar *uri;
+
+		/* TODO URI branch: this is a URI conversion */
+		path = tracker_data_metadata_lookup (data_metadata, "File:NameDelimited");
+		file = g_file_new_for_path (path);
+		uri = g_file_get_uri (file);
+		g_object_unref (file);
+
+		mime_type = tracker_data_metadata_lookup (data_metadata, "File:Mime");
+		tracker_thumbnailer_remove (uri, mime_type);
+
+		tracker_data_metadata_free (data_metadata);
+		g_free (uri);
+	}
+
+	/* Get content, unindex the words and delete the contents */
+	content = tracker_data_query_content (service, service_id);
+
+	if (content) {
+		unindex_text_with_parsing (indexer,
+					   service_id,
+					   service_type_id,
+					   content,
+					   1000);
+		g_free (content);
+		tracker_data_update_delete_content (service, service_id);
+	}
+
+	/* Get metadata from DB to remove it from the index */
+	metadata = tracker_data_query_parsed_metadata (service, service_id);
+	unindex_text_no_parsing (indexer,
+				 service_id,
+				 service_type_id,
+				 metadata,
+				 1000);
+	g_free (metadata);
+
+	/* The weight depends on metadata, but a number high enough
+	 * force deletion.
+	 */
+	metadata = tracker_data_query_unparsed_metadata (service, service_id);
+	unindex_text_with_parsing (indexer,
+				   service_id,
+				   service_type_id,
+				   metadata,
+				   1000);
+	g_free (metadata);
+
+	/* Delete service */
+	tracker_data_update_delete_all_metadata (service, service_id);
+	tracker_data_update_delete_service (service, service_id);
+}
+
+static void
 item_move (TrackerIndexer  *indexer,
 	   PathInfo	   *info,
 	   const gchar	   *dirname,
@@ -1695,7 +1809,8 @@
 	TrackerService *service;
 	TrackerDataMetadata *old_metadata;
 	gchar *path, *source_path;
-	guint32 service_id;
+	gchar *dest_dirname, *dest_basename;
+	guint32 service_id, dest_service_id;
 	GHashTable *children = NULL;
 
 	service = get_service_for_file (info->module_file, info->module);
@@ -1736,6 +1851,23 @@
 		return;
 	}
 
+	tracker_file_get_path_and_name (path, &dest_dirname, &dest_basename);
+
+	/* Check whether destination path already existed */
+	if (tracker_data_query_service_exists (service,
+					       dest_dirname,
+					       dest_basename,
+					       &dest_service_id,
+					       NULL)) {
+		g_message ("Destination file '%s' already existed in database, removing", path);
+
+		/* Item has to be deleted from the database immediately */
+		item_erase (indexer, service, dest_service_id);
+	}
+
+	g_free (dest_dirname);
+	g_free (dest_basename);
+
 	if (info->recurse && strcmp (tracker_service_get_name (service), "Folders") == 0) {
 		children = tracker_data_query_service_children (service, source_path);
 	}
@@ -1744,32 +1876,16 @@
 	old_metadata = tracker_data_query_metadata (service, service_id, TRUE);
 
 	if (!tracker_data_update_move_service (service, source_path, path)) {
-		gchar *dest_dirname, *dest_basename;
-
-		/* Move operation failed, which means the dest path
-		 * corresponded to an indexed file, remove any info
-		 * related to it.
-		 */
-
-		g_message ("Destination file '%s' already existed in database, removing", path);
-
-		tracker_file_get_path_and_name (path, &dest_dirname, &dest_basename);
-		item_remove (indexer, info, dest_dirname, dest_basename);
-
-		g_free (dest_dirname);
-		g_free (dest_basename);
-
-		if (!tracker_data_update_move_service (service, source_path, path)) {
-			/* It failed again, no point in trying anymore */
-			g_free (path);
-			g_free (source_path);
+		g_critical ("Moving item could not be done for unknown reasons");
 
-			if (old_metadata) {
-				tracker_data_metadata_free (old_metadata);
-			}
+		g_free (path);
+		g_free (source_path);
 
-			return;
+		if (old_metadata) {
+			tracker_data_metadata_free (old_metadata);
 		}
+
+		return;
 	}
 
 	/* Update item being moved */
@@ -1814,21 +1930,17 @@
 
 
 static void
-item_remove (TrackerIndexer *indexer,
-	     PathInfo	    *info,
-	     const gchar    *dirname,
-	     const gchar    *basename)
+item_mark_for_removal (TrackerIndexer *indexer,
+		       PathInfo	      *info,
+		       const gchar    *dirname,
+		       const gchar    *basename)
 {
 	TrackerService *service;
-	TrackerDataMetadata *data_metadata;
-#if 0
-	gchar *content;
-	gchar *metadata;
-#endif
 	gchar *path;
 	gchar *mount_point = NULL;
 	const gchar *service_type;
 	guint service_id, service_type_id;
+	GHashTable *children = NULL;
 
 	g_debug ("Removing item: '%s/%s' (no metadata was given by module)", 
 		 dirname, 
@@ -1859,73 +1971,34 @@
 	/* This is needed in a few places. */
 	path = g_build_path (G_DIR_SEPARATOR_S, dirname, basename, NULL);
 
-	/* Get mime type and remove thumbnail from thumbnailerd */
-	data_metadata = tracker_data_query_metadata (service, service_id, TRUE);
-
-	if (data_metadata) {
-		GFile *file;
-		const gchar *mime_type;
-		gchar *uri;
+	if (info->recurse && strcmp (tracker_service_get_name (service), "Folders") == 0) {
+		children = tracker_data_query_service_children (service, path);
+	}
 
-		/* TODO URI branch: this is a URI conversion */
-		file = g_file_new_for_path (path);
-		uri = g_file_get_uri (file);
-		g_object_unref (file);
-		
-		mime_type = tracker_data_metadata_lookup (data_metadata, "File:Mime");
-		tracker_thumbnailer_remove (uri, mime_type);
+	tracker_data_update_disable_service (service, service_id);
 
-		tracker_data_metadata_free (data_metadata);
-		g_free (uri);
-	} else {
-		g_message ("Could not get mime type to remove thumbnail for:'%s'",
-			   path);
-	}
+	if (children) {
+		GHashTableIter iter;
+		gpointer key, value;
 
-	tracker_data_update_delete_content (service, service_id);
+		g_hash_table_iter_init (&iter, children);
 
-#if 0
-	/* Get content, unindex the words and delete the contents */
-	content = tracker_data_query_content (service, service_id);
-	if (content) {
-		unindex_text_with_parsing (indexer,
-					   service_id,
-					   service_type_id,
-					   content,
-					   1000);
-		g_free (content);
-		tracker_data_update_delete_content (service, service_id);
-	}
+		/* Queue children to be removed */
+		while (g_hash_table_iter_next (&iter, &key, &value)) {
+			PathInfo *child_info;
+			const gchar *child_name;
+			GFile *child_file;
 
-	/* Get metadata from DB to remove it from the index */
-	metadata = tracker_data_query_parsed_metadata (service,
-						       service_id);
-	unindex_text_no_parsing (indexer,
-				 service_id,
-				 service_type_id,
-				 metadata,
-				 1000);
-	g_free (metadata);
+			child_name = (const gchar *) value;
+			child_file = g_file_get_child (info->file, child_name);
 
-	/* The weight depends on metadata, but a number high enough
-	 * force deletion.
-	 */
-	metadata = tracker_data_query_unparsed_metadata (service,
-							 service_id);
-	unindex_text_with_parsing (indexer,
-				   service_id,
-				   service_type_id,
-				   metadata,
-				   1000);
-	g_free (metadata);
-#endif
+			child_info = path_info_new (info->module, child_file, NULL, TRUE);
+			add_file (indexer, child_info);
 
-	/* Delete service */
-	tracker_data_update_delete_service (service, service_id);
-	tracker_data_update_delete_all_metadata (service, service_id);
+			g_object_unref (child_file);
+		}
 
-	if (info->recurse && strcmp (service_type, "Folders") == 0) {
-		tracker_data_update_delete_service_recursively (service, path);
+		g_hash_table_destroy (children);
 	}
 
 	if (tracker_hal_path_is_on_removable_device (indexer->private->hal,
@@ -2199,7 +2272,7 @@
 
 static gboolean
 should_change_index_for_file (TrackerIndexer *indexer,
-			      PathInfo	  *info,
+			      PathInfo        *info,
 			      const gchar	  *dirname,
 			      const gchar	  *basename)
 {
@@ -2344,7 +2417,7 @@
 			item_add_or_update (indexer, info, dirname, basename, metadata);
 			g_object_unref (metadata);
 		} else {
-			item_remove (indexer, info, dirname, basename);
+			item_mark_for_removal (indexer, info, dirname, basename);
 		}
 	}
 
@@ -2471,6 +2544,44 @@
 }
 
 static gboolean
+cleanup_task_func (gpointer user_data)
+{
+	TrackerIndexer *indexer;
+	TrackerIndexerPrivate *priv;
+	TrackerService *service;
+	guint32 id;
+
+	indexer = (TrackerIndexer *) user_data;
+	priv = indexer->private;
+
+	if (indexer->private->idle_id) {
+		/* Sanity check, do not index and clean up at the same time */
+		indexer->private->cleanup_task_id = 0;
+		return FALSE;
+	}
+
+	if (tracker_data_query_first_removed_service (priv->file_metadata, &id)) {
+		g_debug ("Cleanup: Deleting service '%d' from files", id);
+		service = tracker_ontology_get_service_by_name ("Files");
+		item_erase (indexer, service, id);
+
+		return TRUE;
+	} else if (tracker_data_query_first_removed_service (priv->email_metadata, &id)) {
+		g_debug ("Cleanup: Deleting service '%d' from emails", id);
+		service = tracker_ontology_get_service_by_name ("Emails");
+		item_erase (indexer, service, id);
+
+		return TRUE;
+	}
+
+	g_debug ("Cleanup: No elements left, exiting");
+
+	state_unset_flags (indexer, TRACKER_INDEXER_STATE_CLEANUP);
+
+	return FALSE;
+}
+
+static gboolean
 process_func (gpointer data)
 {
 	TrackerIndexer *indexer;
@@ -2552,8 +2663,29 @@
 
 	state = indexer->private->state;
 
-	return ((state & TRACKER_INDEXER_STATE_PAUSED) == 0 &&
-		(state & TRACKER_INDEXER_STATE_STOPPED) == 0);
+	if ((state & TRACKER_INDEXER_STATE_PAUSED) != 0) {
+		return FALSE;
+	}
+
+	if ((state & TRACKER_INDEXER_STATE_CLEANUP) == 0 &&
+	    (state & TRACKER_INDEXER_STATE_STOPPED) != 0) {
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+gboolean
+tracker_indexer_get_stoppable (TrackerIndexer *indexer)
+{
+	TrackerIndexerState state;
+
+	g_return_val_if_fail (TRACKER_IS_INDEXER (indexer), FALSE);
+
+	state = indexer->private->state;
+
+	return ((state & TRACKER_INDEXER_STATE_STOPPED) != 0 ||
+		(state & TRACKER_INDEXER_STATE_PAUSED) != 0);
 }
 
 static void
@@ -2579,8 +2711,18 @@
 			g_source_remove (indexer->private->idle_id);
 			indexer->private->idle_id = 0;
 		}
+
+		if ((state & TRACKER_INDEXER_STATE_CLEANUP) != 0) {
+			/* Cleanup stage is only modified by paused */
+			if ((state & TRACKER_INDEXER_STATE_PAUSED) != 0) {
+				cleanup_task_stop (indexer);
+			} else {
+				cleanup_task_start (indexer);
+			}
+		}
 	} else {
 		signal_status_timeout_start (indexer);
+		cleanup_task_stop (indexer);
 
 		if (indexer->private->idle_id == 0) {
 			indexer->private->idle_id = g_idle_add (process_func, indexer);
@@ -2604,6 +2746,9 @@
 	if (state & TRACKER_INDEXER_STATE_STOPPED) {
 		s = g_string_append (s, "STOPPED | ");
 	}
+	if (state & TRACKER_INDEXER_STATE_CLEANUP) {
+		s = g_string_append (s, "CLEANUP | ");
+	}
 
 	s->str[s->len - 3] = '\0';
 

Modified: trunk/src/tracker-indexer/tracker-indexer.h
==============================================================================
--- trunk/src/tracker-indexer/tracker-indexer.h	(original)
+++ trunk/src/tracker-indexer/tracker-indexer.h	Fri Mar  6 15:16:15 2009
@@ -81,6 +81,8 @@
 gboolean        tracker_indexer_get_running         (TrackerIndexer         *indexer);
 void            tracker_indexer_set_running         (TrackerIndexer         *indexer,
 						     gboolean                running);
+gboolean        tracker_indexer_get_stoppable       (TrackerIndexer         *indexer);
+
 void            tracker_indexer_stop                (TrackerIndexer         *indexer);
 void            tracker_indexer_process_all         (TrackerIndexer         *indexer);
 void            tracker_indexer_process_modules     (TrackerIndexer         *indexer,

Modified: trunk/src/tracker-indexer/tracker-main.c
==============================================================================
--- trunk/src/tracker-indexer/tracker-main.c	(original)
+++ trunk/src/tracker-indexer/tracker-main.c	Fri Mar  6 15:16:15 2009
@@ -242,7 +242,7 @@
 
 	indexer = TRACKER_INDEXER (user_data);
 
-	if (!tracker_indexer_get_running (indexer)) {
+	if (tracker_indexer_get_stoppable (indexer)) {
 		g_message ("Indexer is still not running after %d seconds, quitting...",
 			   QUIT_TIMEOUT);
 		g_main_loop_quit (main_loop);

Modified: trunk/src/trackerd/tracker-dbus.c
==============================================================================
--- trunk/src/trackerd/tracker-dbus.c	(original)
+++ trunk/src/trackerd/tracker-dbus.c	Fri Mar  6 15:16:15 2009
@@ -53,11 +53,16 @@
 
 #define INDEXER_PAUSE_TIME_FOR_REQUESTS 5 /* seconds */
 
+#define TRACKER_INDEXER_SERVICE   "org.freedesktop.Tracker.Indexer"
+#define TRACKER_INDEXER_PATH      "/org/freedesktop/Tracker/Indexer"
+#define TRACKER_INDEXER_INTERFACE "org.freedesktop.Tracker.Indexer"
+
 static DBusGConnection *connection;
 static DBusGProxy      *gproxy;
 static DBusGProxy      *proxy_for_indexer;
 static GSList	       *objects;
 static guint		indexer_resume_timeout_id;
+static gboolean         indexer_available;
 
 static gboolean
 dbus_register_service (DBusGProxy  *proxy,
@@ -108,10 +113,34 @@
 }
 
 static void
-dbus_name_owner_changed (gpointer  data,
-			 GClosure *closure)
+indexer_name_owner_changed (DBusGProxy   *proxy,
+			    const char   *name,
+			    const char   *prev_owner,
+			    const char   *new_owner,
+			    TrackerXesam *self)
+{
+	if (strcmp (name, TRACKER_INDEXER_SERVICE) == 0) {
+		if (!new_owner || !*new_owner) {
+			g_debug ("Indexer no longer present");
+			indexer_available = FALSE;
+		} else {
+			g_debug ("Indexer has become present");
+			indexer_available = TRUE;
+		}
+	}
+}
+
+static void
+initialize_indexer_presence (DBusGProxy *proxy)
 {
-	g_object_unref (data);
+	gchar *owner;
+
+	if (org_freedesktop_DBus_get_name_owner (gproxy, TRACKER_INDEXER_SERVICE, &owner, NULL)) {
+		indexer_available = (owner != NULL);
+		g_free (owner);
+	} else {
+		indexer_available = FALSE;
+	}
 }
 
 static gboolean
@@ -146,6 +175,17 @@
 					    DBUS_PATH_DBUS,
 					    DBUS_INTERFACE_DBUS);
 
+	/* Register signals to know about tracker-indexer presence */
+	dbus_g_proxy_add_signal (gproxy, "NameOwnerChanged",
+				 G_TYPE_STRING, G_TYPE_STRING,
+				 G_TYPE_STRING, G_TYPE_INVALID);
+
+	dbus_g_proxy_connect_signal (gproxy, "NameOwnerChanged",
+				     G_CALLBACK (indexer_name_owner_changed),
+				     NULL, NULL);
+
+	initialize_indexer_presence (gproxy);
+
 	/* Register the service name for org.freedesktop.Tracker */
 	if (!dbus_register_service (gproxy, TRACKER_DAEMON_SERVICE)) {
 		return FALSE;
@@ -205,9 +245,8 @@
 		return;
 	}
 
-	/* Don't try to pause unless we are in particular states */
-	if (status != TRACKER_STATUS_INDEXING) {
-		g_message ("New DBus request, not pausing indexer, not in indexing state");
+	if (!indexer_available) {
+		g_message ("New DBus request, not pausing indexer, since it's not there");
 		return;
 	}
 
@@ -412,7 +451,7 @@
 		dbus_g_proxy_connect_signal (gproxy, "NameOwnerChanged",
 					     G_CALLBACK (tracker_xesam_name_owner_changed),
 					     g_object_ref (G_OBJECT (object)),
-					     dbus_name_owner_changed);
+					     (GClosureNotify) g_object_unref);
 	}
 
 	/* Reverse list since we added objects at the top each time */
@@ -446,9 +485,9 @@
 	if (!proxy_for_indexer) {
 		/* Get proxy for Service / Path / Interface of the indexer */
 		proxy_for_indexer = dbus_g_proxy_new_for_name (connection,
-							       "org.freedesktop.Tracker.Indexer",
-							       "/org/freedesktop/Tracker/Indexer",
-							       "org.freedesktop.Tracker.Indexer");
+							       TRACKER_INDEXER_SERVICE,
+							       TRACKER_INDEXER_PATH,
+							       TRACKER_INDEXER_INTERFACE);
 
 		if (!proxy_for_indexer) {
 			g_critical ("Couldn't create a DBusGProxy to the indexer service");

Modified: trunk/src/trackerd/tracker-metadata.c
==============================================================================
--- trunk/src/trackerd/tracker-metadata.c	(original)
+++ trunk/src/trackerd/tracker-metadata.c	Fri Mar  6 15:16:15 2009
@@ -77,7 +77,8 @@
 	TrackerDBResultSet  *result_set;
 	guint		     request_id;
 	const gchar         *service_result;
-	gchar		    *service_id;
+	guint32              service_id;
+	gchar		    *service_id_str;
 	guint		     i;
 	gchar		   **values;
 	GError		    *actual_error = NULL;
@@ -104,8 +105,9 @@
 		return;
 	}
 
-	service_id = tracker_data_query_file_id_as_string (service_type, uri);
-	if (!service_id) {
+	service_id = tracker_data_query_file_id (service_type, uri);
+
+	if (service_id <= 0) {
 		tracker_dbus_request_failed (request_id,
 					     &actual_error,
 					     "Service URI '%s' not found",
@@ -138,7 +140,6 @@
 
 	service_result = tracker_data_query_service_type_by_id (iface, service_id);
 	if (!service_result) {
-	       g_free (service_id);
 	       tracker_dbus_request_failed (request_id,
 					    &actual_error,
 					    "Service type can not be found for entity '%s'",
@@ -148,7 +149,10 @@
 	       return;
 	}
 
-	result_set = tracker_data_query_metadata_fields (iface, service_result, service_id, keys);
+	service_id_str = tracker_guint_to_string (service_id);
+	result_set = tracker_data_query_metadata_fields (iface, service_result, service_id_str, keys);
+	g_free (service_id_str);
+
 	if (result_set) {
 		values = tracker_dbus_query_result_columns_to_strv (result_set, -1, -1, TRUE);
 		g_object_unref (result_set);
@@ -166,7 +170,6 @@
 
 	dbus_g_method_return (context, values);
 	g_strfreev (values);
-	g_free (service_id);
 
 	tracker_dbus_request_success (request_id);
 }