[Evolution-hackers] Support in Tracker for ultra-new Evolution installs that use SQLite for the summary format

[Philip Van Hoof <spam pvanhoof be>]

This patch makes ultra-new Evolution installs work again with Tracker.

There's one problem and that is that the query will only find E-mails in
the INBOX folder. You can easily find the Query and figure out what the
problem is:

The design that Carlos made assumes that for each folder there's a
"summary" file. In the new Evolution cache format there's just one
"folders.db" for each account.

I could do a generated UNION select after first doing "select * from
folders" on folders.db and then generating a query that includes all
folders. I just have not done this for now and instead I'm just using
INBOX and I'm neglecting the other folders.

This is NOT the same as the proposal that I am doing at (a). This is
instead a ad-hoc solution for the new situation (Evolution using SQLite
for the summaries). I find this solution rather nasty, to be honest.

(a) http://live.gnome.org/Evolution/Metadata

For Carlos: I have also fixed a serious problem in evolution-pop.c,
which is by the way unaffected by Evolution's changes (and works, if you
just apply the patch that I included in this larger patch). The POP
support's get_message_metadata was not returning metadata. 

This was crashing my tracker-indexer (as seemingly my compiler was
putting "return 0x2" where the return was omitted, and the memory I have
at 0x2 didn't dereference TrackerModuleMetadata's members very well).

Please review and/or rework the patch.

-- 
Philip Van Hoof, freelance software developer
home: me at pvanhoof dot be 
gnome: pvanhoof at gnome dot org 
http://pvanhoof.be/blog
http://codeminded.be

Index: src/tracker-indexer/modules/evolution-pop.c
===================================================================
--- src/tracker-indexer/modules/evolution-pop.c	(revision 2701)
+++ src/tracker-indexer/modules/evolution-pop.c	(working copy)
@@ -360,6 +360,8 @@
 	}
 
 	g_list_free (list);
+
+	return metadata;
 }
 
 static TrackerModuleMetadata *
Index: src/tracker-indexer/modules/evolution.c
===================================================================
--- src/tracker-indexer/modules/evolution.c	(revision 2701)
+++ src/tracker-indexer/modules/evolution.c	(working copy)
@@ -70,7 +70,8 @@
 	    strchr (basename, '.') == NULL) {
 		type = MAIL_STORAGE_LOCAL;
 	} else if (g_str_has_prefix (path, imap_dir) &&
-		   strcmp (basename, "summary") == 0) {
+		   (strcmp (basename, "summary") == 0 ||
+		    strcmp (basename, "folders.db") == 0)) {
 		type = MAIL_STORAGE_IMAP;
 	}
 
Index: src/tracker-indexer/modules/Makefile.am
===================================================================
--- src/tracker-indexer/modules/Makefile.am	(revision 2701)
+++ src/tracker-indexer/modules/Makefile.am	(working copy)
@@ -16,7 +16,8 @@
 	$(GIO_CFLAGS)							\
 	$(GLIB2_CFLAGS)							\
 	$(GCONF_CFLAGS)							\
-	$(GMIME_CFLAGS)
+	$(GMIME_CFLAGS)							\
+	$(SQLITE3_CFLAGS)
 
 indexer_modules_LTLIBRARIES = 						\
 	libtracker-module-applications.la				\
@@ -67,6 +68,7 @@
 	$(GMODULE_LIBS)							\
 	$(GLIB2_LIBS)							\
 	$(GCONF_LIBS)							\
-	$(GMIME_LIBS)
+	$(GMIME_LIBS)							\
+	$(SQLITE3_LIBS)
 
 endif
Index: src/tracker-indexer/modules/evolution-imap.c
===================================================================
--- src/tracker-indexer/modules/evolution-imap.c	(revision 2701)
+++ src/tracker-indexer/modules/evolution-imap.c	(working copy)
@@ -128,7 +128,18 @@
         g_free (file->imap_dir);
         g_free (file->cur_message_uid);
 
-        fclose (file->summary);
+	if (file->db) {
+		sqlite3_close (file->db);
+	}
+
+	if (file->stmt) {
+		sqlite3_finalize (file->stmt);
+	}
+
+	if (file->summary) {
+		fclose (file->summary);
+	}
+
 	close (file->fd);
 
         G_OBJECT_CLASS (tracker_evolution_imap_file_parent_class)->finalize (object);
@@ -494,25 +505,62 @@
                                            NULL);
 
         path = g_file_get_path (tracker_module_file_get_file (file));
-        self->fd = tracker_file_open (path, TRUE);
-        g_free (path);
 
-        if (self->fd == -1) {
-                return;
-        }
+	self->db = NULL;
+	self->summary = NULL;
+	self->stmt = NULL;
 
-        self->summary = fdopen (self->fd, "r");
-        self->n_messages = read_summary_header (self->summary);
-        self->cur_message = 1;
+	if (g_str_has_suffix (path, ".db")) {
+		/* New SQLite based format */
+		sqlite3_stmt *stmt;
+		gint result = SQLITE_OK;
 
-        if (self->n_messages > 0) {
-                /* save current message uid */
-                read_summary (self->summary,
-                              SUMMARY_TYPE_STRING, &self->cur_message_uid, /* message uid */
-                              -1);
-        }
+		sqlite3_open (path, &self->db);
 
-        ensure_imap_accounts ();
+		sqlite3_prepare_v2 (self->db, "select saved_count from folders where folder_name = 'INBOX'", 
+				    -1, &stmt, NULL);
+
+		result = sqlite3_step (stmt);
+		self->n_messages = sqlite3_column_int (stmt, 0);
+		self->cur_message = 1;
+
+		if (self->n_messages > 0) {
+			sqlite3_prepare_v2 (self->db, "SELECT uid, deleted, attachment, dsent, subject, mail_from, mail_to, mail_cc, mlist FROM INBOX", 
+					    -1, &self->stmt, NULL);
+			if (self->stmt) {
+				result = sqlite3_step (self->stmt);
+				self->cur_message_uid = g_strdup (sqlite3_column_text (self->stmt, 0));
+				self->cur_message = 1;
+			}
+		}
+
+		/* success, start using the new format */
+		goto ensure;
+
+	}
+
+	/* Old format */
+	self->fd = tracker_file_open (path, TRUE);
+	g_free (path);
+
+	if (self->fd == -1) {
+		return;
+	}
+
+	self->summary = fdopen (self->fd, "r");
+	self->n_messages = read_summary_header (self->summary);
+	self->cur_message = 1;
+
+	if (self->n_messages > 0) {
+		/* save current message uid */
+		read_summary (self->summary,
+			      SUMMARY_TYPE_STRING, &self->cur_message_uid, /* message uid */
+			      -1);
+	}
+
+	ensure:
+
+	ensure_imap_accounts ();
 }
 
 static const gchar *
@@ -536,7 +584,12 @@
 	gchar *path, *prefix, *message_path;
 
         path = g_file_get_path (tracker_module_file_get_file (file));
-	prefix = g_strndup (path, strlen (path) - strlen ("summary"));
+
+	if (g_str_has_suffix (path, "folders.db"))
+		prefix = g_strndup (path, strlen (path) - strlen ("folders.db"));
+	else
+		prefix = g_strndup (path, strlen (path) - strlen ("summary"));
+
         g_free (path);
 
         message_path = g_strconcat (prefix, uid, ".", NULL);
@@ -882,37 +935,49 @@
 	GList *list, *l;
 	gboolean deleted;
 
-        self = TRACKER_EVOLUTION_IMAP_FILE (file);
+	self = TRACKER_EVOLUTION_IMAP_FILE (file);
 
-	if (!read_summary (self->summary,
-			   SUMMARY_TYPE_UINT32, &flags, /* flags */
-			   -1)) {
-		return NULL;
-	}
+	if (self->stmt) {
+		deleted = sqlite3_column_int (self->stmt, 1);
+		/* hasattach = sqlite3_column_int (self->stmt, 2); */
+		date = g_strdup (sqlite3_column_text (self->stmt, 3));
+		subject = g_strdup (sqlite3_column_text (self->stmt, 4));
+		from = g_strdup (sqlite3_column_text (self->stmt, 5));
+		to = g_strdup (sqlite3_column_text (self->stmt, 6));
+		cc = g_strdup (sqlite3_column_text (self->stmt, 7));
 
-	deleted = ((flags & EVOLUTION_MESSAGE_JUNK) != 0 ||
-		   (flags & EVOLUTION_MESSAGE_DELETED) != 0);
+	} else {
+		if (!read_summary (self->summary,
+				   SUMMARY_TYPE_UINT32, &flags, /* flags */
+				   -1)) {
+			return NULL;
+		}
 
-	subject = NULL;
-	from = NULL;
-	to = NULL;
-	cc = NULL;
+		deleted = ((flags & EVOLUTION_MESSAGE_JUNK) != 0 ||
+			   (flags & EVOLUTION_MESSAGE_DELETED) != 0);
 
-	if (!read_summary (self->summary,
-			   SUMMARY_TYPE_UINT32, NULL,	  /* size */
-			   SUMMARY_TYPE_TIME_T, NULL,	  /* date sent */
-			   SUMMARY_TYPE_TIME_T, &t,	  /* date received */
-			   SUMMARY_TYPE_STRING, &subject, /* subject */
-			   SUMMARY_TYPE_STRING, &from,	  /* from */
-			   SUMMARY_TYPE_STRING, &to,	  /* to */
-			   SUMMARY_TYPE_STRING, &cc,	  /* cc */
-			   SUMMARY_TYPE_STRING, NULL,	  /* mlist */
-			   -1)) {
-		g_free (subject);
-		g_free (from);
-		g_free (to);
-		g_free (cc);
-		return NULL;
+
+		subject = NULL;
+		from = NULL;
+		to = NULL;
+		cc = NULL;
+
+		if (!read_summary (self->summary,
+				   SUMMARY_TYPE_UINT32, NULL,	  /* size */
+				   SUMMARY_TYPE_TIME_T, NULL,	  /* date sent */
+				   SUMMARY_TYPE_TIME_T, &t,	  /* date received */
+				   SUMMARY_TYPE_STRING, &subject, /* subject */
+				   SUMMARY_TYPE_STRING, &from,	  /* from */
+				   SUMMARY_TYPE_STRING, &to,	  /* to */
+				   SUMMARY_TYPE_STRING, &cc,	  /* cc */
+				   SUMMARY_TYPE_STRING, NULL,	  /* mlist */
+				   -1)) {
+			g_free (subject);
+			g_free (from);
+			g_free (to);
+			g_free (cc);
+			return NULL;
+		}
 	}
 
 	if (!deleted) {
@@ -946,67 +1011,70 @@
 	g_free (to);
 	g_free (cc);
 
-	if (!read_summary (self->summary,
-			   SUMMARY_TYPE_INT32, NULL,
-			   SUMMARY_TYPE_INT32, NULL,
-			   SUMMARY_TYPE_UINT32, &count,
-			   -1)) {
-		goto corruption;
-	}
+	if (!self->stmt) {
+		if (!read_summary (self->summary,
+				   SUMMARY_TYPE_INT32, NULL,
+				   SUMMARY_TYPE_INT32, NULL,
+				   SUMMARY_TYPE_UINT32, &count,
+				   -1)) {
+			goto corruption;
+		}
 
-	/* references */
-	for (i = 0; i < count; i++) {
-		if (read_summary (self->summary,
-				  SUMMARY_TYPE_INT32, NULL,
-				  SUMMARY_TYPE_INT32, NULL,
-				  -1)) {
-			continue;
+		/* references */
+		for (i = 0; i < count; i++) {
+			if (read_summary (self->summary,
+					  SUMMARY_TYPE_INT32, NULL,
+					  SUMMARY_TYPE_INT32, NULL,
+					  -1)) {
+				continue;
+			}
+
+			goto corruption;
 		}
 
-		goto corruption;
-	}
+		if (!read_summary (self->summary, SUMMARY_TYPE_UINT32, &count, -1)) {
+			goto corruption;
+		}
 
-	if (!read_summary (self->summary, SUMMARY_TYPE_UINT32, &count, -1)) {
-		goto corruption;
-	}
+		/* user flags */
+		for (i = 0; i < count; i++) {
+			if (read_summary (self->summary, SUMMARY_TYPE_STRING, NULL, -1)) {
+				continue;
+			}
 
-	/* user flags */
-	for (i = 0; i < count; i++) {
-		if (read_summary (self->summary, SUMMARY_TYPE_STRING, NULL, -1)) {
-			continue;
+			goto corruption;
 		}
 
-		goto corruption;
-	}
+		if (!read_summary (self->summary, SUMMARY_TYPE_UINT32, &count, -1)) {
+			goto corruption;
+		}
 
-	if (!read_summary (self->summary, SUMMARY_TYPE_UINT32, &count, -1)) {
-		goto corruption;
-	}
+		/* user tags */
+		for (i = 0; i < count; i++) {
+			if (read_summary (self->summary,
+					  SUMMARY_TYPE_STRING, NULL,
+					  SUMMARY_TYPE_STRING, NULL,
+					  -1)) {
+				continue;
+			}
 
-	/* user tags */
-	for (i = 0; i < count; i++) {
-		if (read_summary (self->summary,
-				  SUMMARY_TYPE_STRING, NULL,
-				  SUMMARY_TYPE_STRING, NULL,
-				  -1)) {
-			continue;
+			goto corruption;
 		}
 
-		goto corruption;
-	}
+		/* server flags */
+		if (!read_summary (self->summary,
+				   SUMMARY_TYPE_UINT32, NULL,
+				   -1)) {
+			goto corruption;
+		}
 
-	/* server flags */
-	if (!read_summary (self->summary,
-			   SUMMARY_TYPE_UINT32, NULL,
-			   -1)) {
-		goto corruption;
+		skip_content_info (self->summary);
 	}
 
-	skip_content_info (self->summary);
-
 	return metadata;
 
-corruption:
+	corruption:
+
 	/* assume corruption */
 	if (metadata) {
 		g_object_unref (metadata);
@@ -1130,9 +1198,14 @@
         self->cur_message_uid = NULL;
 
         /* save current message uid */
-        read_summary (self->summary,
-                      SUMMARY_TYPE_STRING, &self->cur_message_uid, /* message uid */
-                      -1);
+	if (self->stmt) {
+		self->cur_message_uid = g_strdup (sqlite3_column_text (self->stmt, 0));
+		sqlite3_step (self->stmt);
+	} else {
+		read_summary (self->summary,
+			      SUMMARY_TYPE_STRING, &self->cur_message_uid, /* message uid */
+			      -1);
+	}
 
         self->cur_message++;
 
Index: src/tracker-indexer/modules/evolution-imap.h
===================================================================
--- src/tracker-indexer/modules/evolution-imap.h	(revision 2701)
+++ src/tracker-indexer/modules/evolution-imap.h	(working copy)
@@ -27,6 +27,7 @@
 
 #include <glib.h>
 #include <glib/gstdio.h>
+#include <sqlite3.h>
 
 #include <tracker-indexer/tracker-module-file.h>
 
@@ -56,6 +57,8 @@
 
         GList *mime_parts;
         GList *current_mime_part;
+	sqlite3 *db;
+	sqlite3_stmt *stmt;
 };
 
 struct TrackerEvolutionImapFileClass {