GNOME.org
 

Re: [Evolution-hackers] [Tracker] Support in Tracker for ultra-new Evolution installs that use SQLite for the summary format

On Fri, 2008-12-12 at 01:59 +0100, Philip Van Hoof wrote:
> On Thu, 2008-12-11 at 23:58 +0200, Ivan Frade wrote:
> > hi philip,
> > 
> >  short comment (the tablet is not the best device to review patches):
> > can we call this module "evolution224" and keep also the old module?
> > we could choose between one or other with a flag in compilation time
> > (the distributions will choose the correct for them)
> 
> With this patch the code will detect the format and automatically choose
> the right way to parse both.
> 

This version of the patch adds support for IMAP subfolders and fixes the
fact that in the last patch the "email://" URI was generated wrong (it
still contained the "folders.db" string-piece).


> >  regards,
> > 
> > Ivan
> > 
> > On 12/11/08, Philip Van Hoof <spam pvanhoof be> wrote:
> > > This patch makes ultra-new Evolution installs work again with Tracker.
> > >
> > > There's one problem and that is that the query will only find E-mails in
> > > the INBOX folder. You can easily find the Query and figure out what the
> > > problem is:
> > >
> > > The design that Carlos made assumes that for each folder there's a
> > > "summary" file. In the new Evolution cache format there's just one
> > > "folders.db" for each account.
> > >
> > > I could do a generated UNION select after first doing "select * from
> > > folders" on folders.db and then generating a query that includes all
> > > folders. I just have not done this for now and instead I'm just using
> > > INBOX and I'm neglecting the other folders.
> > >
> > > This is NOT the same as the proposal that I am doing at (a). This is
> > > instead a ad-hoc solution for the new situation (Evolution using SQLite
> > > for the summaries). I find this solution rather nasty, to be honest.
> > >
> > > (a) http://live.gnome.org/Evolution/Metadata
> > >
> > > For Carlos: I have also fixed a serious problem in evolution-pop.c,
> > > which is by the way unaffected by Evolution's changes (and works, if you
> > > just apply the patch that I included in this larger patch). The POP
> > > support's get_message_metadata was not returning metadata.
> > >
> > > This was crashing my tracker-indexer (as seemingly my compiler was
> > > putting "return 0x2" where the return was omitted, and the memory I have
> > > at 0x2 didn't dereference TrackerModuleMetadata's members very well).
> > >
> > > Please review and/or rework the patch.
> > >
> > > --
> > > Philip Van Hoof, freelance software developer
> > > home: me at pvanhoof dot be
> > > gnome: pvanhoof at gnome dot org
> > > http://pvanhoof.be/blog
> > > http://codeminded.be
> > >
> > 
-- 
Philip Van Hoof, freelance software developer
home: me at pvanhoof dot be 
gnome: pvanhoof at gnome dot org 
http://pvanhoof.be/blog
http://codeminded.be

Index: src/tracker-indexer/modules/evolution-pop.c
===================================================================
--- src/tracker-indexer/modules/evolution-pop.c	(revision 2701)
+++ src/tracker-indexer/modules/evolution-pop.c	(working copy)
@@ -360,6 +360,8 @@
 	}
 
 	g_list_free (list);
+
+	return metadata;
 }
 
 static TrackerModuleMetadata *
Index: src/tracker-indexer/modules/evolution.c
===================================================================
--- src/tracker-indexer/modules/evolution.c	(revision 2701)
+++ src/tracker-indexer/modules/evolution.c	(working copy)
@@ -70,7 +70,9 @@
 	    strchr (basename, '.') == NULL) {
 		type = MAIL_STORAGE_LOCAL;
 	} else if (g_str_has_prefix (path, imap_dir) &&
-		   strcmp (basename, "summary") == 0) {
+		   (strcmp (basename, "summary") == 0 ||
+		    strcmp (basename, "folders.db") == 0) ||
+		    strcmp (basename, "cmeta") == 0) {
 		type = MAIL_STORAGE_IMAP;
 	}
 
Index: src/tracker-indexer/modules/Makefile.am
===================================================================
--- src/tracker-indexer/modules/Makefile.am	(revision 2701)
+++ src/tracker-indexer/modules/Makefile.am	(working copy)
@@ -16,7 +16,8 @@
 	$(GIO_CFLAGS)							\
 	$(GLIB2_CFLAGS)							\
 	$(GCONF_CFLAGS)							\
-	$(GMIME_CFLAGS)
+	$(GMIME_CFLAGS)							\
+	$(SQLITE3_CFLAGS)
 
 indexer_modules_LTLIBRARIES = 						\
 	libtracker-module-applications.la				\
@@ -67,6 +68,7 @@
 	$(GMODULE_LIBS)							\
 	$(GLIB2_LIBS)							\
 	$(GCONF_LIBS)							\
-	$(GMIME_LIBS)
+	$(GMIME_LIBS)							\
+	$(SQLITE3_LIBS)
 
 endif
Index: src/tracker-indexer/modules/evolution-imap.c
===================================================================
--- src/tracker-indexer/modules/evolution-imap.c	(revision 2701)
+++ src/tracker-indexer/modules/evolution-imap.c	(working copy)
@@ -128,7 +128,18 @@
         g_free (file->imap_dir);
         g_free (file->cur_message_uid);
 
-        fclose (file->summary);
+	if (file->db) {
+		sqlite3_close (file->db);
+	}
+
+	if (file->stmt) {
+		sqlite3_finalize (file->stmt);
+	}
+
+	if (file->summary) {
+		fclose (file->summary);
+	}
+
 	close (file->fd);
 
         G_OBJECT_CLASS (tracker_evolution_imap_file_parent_class)->finalize (object);
@@ -482,10 +493,88 @@
 }
 
 static void
+get_folder_info (const gchar *path, const gchar *imap_dir, gchar **out_foldersdb, gchar **out_mailbox)
+{
+	gchar *mailbox = NULL;
+	gchar *n1path = g_strdup (path);
+	gchar *n2path = g_strdup (path);
+	gchar *ptr = strstr (n1path, imap_dir);
+	gboolean isset = FALSE;
+
+	*out_foldersdb = NULL;
+
+	if (ptr) {
+		ptr += strlen (imap_dir);
+
+		if (*ptr == G_DIR_SEPARATOR)
+			ptr++;
+
+		gchar *account_path = strchr (ptr, G_DIR_SEPARATOR);
+
+		if (account_path) {
+			gchar *foldersdb_path;
+
+			*account_path = '\0';
+			account_path = n1path;
+
+			foldersdb_path = g_build_filename (account_path,
+						    "folders.db", NULL); 
+
+			if (g_file_test (foldersdb_path, G_FILE_TEST_EXISTS)) {
+				gchar *folders_path;
+
+				folders_path = g_build_filename (account_path,
+						    "folders", G_DIR_SEPARATOR_S, NULL); 
+
+				ptr = strstr (n2path, folders_path);
+				if (ptr) {
+					guint i = 0, t = 0, y;
+					guint step_a = strlen ("subfolders" G_DIR_SEPARATOR_S);
+					guint step_b = strlen (G_DIR_SEPARATOR_S "cmeta");
+
+					ptr += strlen (folders_path);
+					y = strlen (ptr);
+
+					while (i < y) {
+
+						if (ptr[i] == 's' && g_str_has_prefix (ptr+i, "subfolders" G_DIR_SEPARATOR_S)) {
+							i += step_a;
+						} else if (ptr[i] == '/' && g_str_has_prefix (ptr+i, G_DIR_SEPARATOR_S "cmeta")) {
+							i += step_b;
+						} else {
+							ptr[t] = ptr[i];
+							i++;
+							t++;
+						}
+
+					}
+					if (t < y)
+						ptr[t] = '\0';
+
+					isset = TRUE;
+					mailbox = g_strdup (ptr);
+					*out_foldersdb = foldersdb_path;
+
+				} 
+			} else
+				g_free (foldersdb_path);
+		}
+	}
+
+	g_free (n1path);
+	g_free (n2path);
+
+	*out_mailbox = mailbox;
+	if (!isset)
+		*out_foldersdb = g_strdup (path);
+}
+
+static void
 tracker_evolution_imap_file_initialize (TrackerModuleFile *file)
 {
         TrackerEvolutionImapFile *self;
         gchar *path;
+	gchar *mailbox = NULL;
 
         self = TRACKER_EVOLUTION_IMAP_FILE (file);
 
@@ -494,25 +583,85 @@
                                            NULL);
 
         path = g_file_get_path (tracker_module_file_get_file (file));
-        self->fd = tracker_file_open (path, TRUE);
-        g_free (path);
 
-        if (self->fd == -1) {
-                return;
-        }
+	self->db = NULL;
+	self->summary = NULL;
+	self->stmt = NULL;
 
-        self->summary = fdopen (self->fd, "r");
-        self->n_messages = read_summary_header (self->summary);
-        self->cur_message = 1;
+	if (g_str_has_suffix (path, "cmeta")) {
+		gchar *new_path = NULL;
 
-        if (self->n_messages > 0) {
-                /* save current message uid */
-                read_summary (self->summary,
-                              SUMMARY_TYPE_STRING, &self->cur_message_uid, /* message uid */
-                              -1);
-        }
+		get_folder_info (path, self->imap_dir, &new_path, &mailbox);
 
-        ensure_imap_accounts ();
+		if (new_path) {
+			g_free (path);
+			path = new_path;
+		}
+	}
+
+	if (g_str_has_suffix (path, ".db")) {
+		/* New SQLite based format */
+		sqlite3_stmt *stmt;
+		gint result = SQLITE_OK;
+
+		if (sqlite3_open_v2 (path, &self->db, SQLITE_OPEN_READONLY, NULL) == SQLITE_OK) {
+			gchar *sql = g_strdup_printf ("select saved_count from folders where folder_name = '%s'", 
+						      mailbox ? mailbox : "INBOX");
+
+			sqlite3_prepare_v2 (self->db, sql , -1, &stmt, NULL);
+
+			g_free (sql);
+
+			result = sqlite3_step (stmt);
+			self->n_messages = sqlite3_column_int (stmt, 0);
+			self->cur_message = 1;
+
+			if (self->n_messages > 0) {
+
+				sql = g_strdup_printf ("SELECT uid, deleted, attachment, dsent, subject, mail_from, mail_to, mail_cc, mlist FROM '%s'",
+							      mailbox ? mailbox : "INBOX");
+
+				sqlite3_prepare_v2 (self->db, sql, -1, &self->stmt, NULL);
+				if (self->stmt) {
+					result = sqlite3_step (self->stmt);
+					self->cur_message_uid = g_strdup (sqlite3_column_text (self->stmt, 0));
+					self->cur_message = 1;
+				}
+
+				g_free (sql);
+			}
+		}
+
+		/* Even if we don't succeed, the existence of a folders.db file
+		 * means that the code below just wont work anyway. */
+
+		goto ensure;
+	}
+
+	g_free (mailbox);
+
+	/* Old format */
+	self->fd = tracker_file_open (path, TRUE);
+	g_free (path);
+
+	if (self->fd == -1) {
+		return;
+	}
+
+	self->summary = fdopen (self->fd, "r");
+	self->n_messages = read_summary_header (self->summary);
+	self->cur_message = 1;
+
+	if (self->n_messages > 0) {
+		/* save current message uid */
+		read_summary (self->summary,
+			      SUMMARY_TYPE_STRING, &self->cur_message_uid, /* message uid */
+			      -1);
+	}
+
+	ensure:
+
+	ensure_imap_accounts ();
 }
 
 static const gchar *
@@ -536,7 +685,12 @@
 	gchar *path, *prefix, *message_path;
 
         path = g_file_get_path (tracker_module_file_get_file (file));
-	prefix = g_strndup (path, strlen (path) - strlen ("summary"));
+
+	if (g_str_has_suffix (path, "folders.db"))
+		prefix = g_strndup (path, strlen (path) - strlen ("folders.db"));
+	else
+		prefix = g_strndup (path, strlen (path) - strlen ("summary"));
+
         g_free (path);
 
         message_path = g_strconcat (prefix, uid, ".", NULL);
@@ -676,6 +830,8 @@
                 subdirs = tracker_string_remove (subdirs, "/folders/");
                 subdirs = tracker_string_remove (subdirs, "/subfolders");
                 subdirs = tracker_string_remove (subdirs, "/summary");
+                subdirs = tracker_string_remove (subdirs, "/folders.db");
+                subdirs = tracker_string_remove (subdirs, "/cmeta");
 
                 uri = g_strdup_printf ("email://%s/%s;uid=%s",
                                        (gchar *) g_hash_table_lookup (accounts, k->data),
@@ -882,37 +1038,49 @@
 	GList *list, *l;
 	gboolean deleted;
 
-        self = TRACKER_EVOLUTION_IMAP_FILE (file);
+	self = TRACKER_EVOLUTION_IMAP_FILE (file);
 
-	if (!read_summary (self->summary,
-			   SUMMARY_TYPE_UINT32, &flags, /* flags */
-			   -1)) {
-		return NULL;
-	}
+	if (self->stmt) {
+		deleted = sqlite3_column_int (self->stmt, 1);
+		/* hasattach = sqlite3_column_int (self->stmt, 2); */
+		date = g_strdup (sqlite3_column_text (self->stmt, 3));
+		subject = g_strdup (sqlite3_column_text (self->stmt, 4));
+		from = g_strdup (sqlite3_column_text (self->stmt, 5));
+		to = g_strdup (sqlite3_column_text (self->stmt, 6));
+		cc = g_strdup (sqlite3_column_text (self->stmt, 7));
 
-	deleted = ((flags & EVOLUTION_MESSAGE_JUNK) != 0 ||
-		   (flags & EVOLUTION_MESSAGE_DELETED) != 0);
+	} else {
+		if (!read_summary (self->summary,
+				   SUMMARY_TYPE_UINT32, &flags, /* flags */
+				   -1)) {
+			return NULL;
+		}
 
-	subject = NULL;
-	from = NULL;
-	to = NULL;
-	cc = NULL;
+		deleted = ((flags & EVOLUTION_MESSAGE_JUNK) != 0 ||
+			   (flags & EVOLUTION_MESSAGE_DELETED) != 0);
 
-	if (!read_summary (self->summary,
-			   SUMMARY_TYPE_UINT32, NULL,	  /* size */
-			   SUMMARY_TYPE_TIME_T, NULL,	  /* date sent */
-			   SUMMARY_TYPE_TIME_T, &t,	  /* date received */
-			   SUMMARY_TYPE_STRING, &subject, /* subject */
-			   SUMMARY_TYPE_STRING, &from,	  /* from */
-			   SUMMARY_TYPE_STRING, &to,	  /* to */
-			   SUMMARY_TYPE_STRING, &cc,	  /* cc */
-			   SUMMARY_TYPE_STRING, NULL,	  /* mlist */
-			   -1)) {
-		g_free (subject);
-		g_free (from);
-		g_free (to);
-		g_free (cc);
-		return NULL;
+
+		subject = NULL;
+		from = NULL;
+		to = NULL;
+		cc = NULL;
+
+		if (!read_summary (self->summary,
+				   SUMMARY_TYPE_UINT32, NULL,	  /* size */
+				   SUMMARY_TYPE_TIME_T, NULL,	  /* date sent */
+				   SUMMARY_TYPE_TIME_T, &t,	  /* date received */
+				   SUMMARY_TYPE_STRING, &subject, /* subject */
+				   SUMMARY_TYPE_STRING, &from,	  /* from */
+				   SUMMARY_TYPE_STRING, &to,	  /* to */
+				   SUMMARY_TYPE_STRING, &cc,	  /* cc */
+				   SUMMARY_TYPE_STRING, NULL,	  /* mlist */
+				   -1)) {
+			g_free (subject);
+			g_free (from);
+			g_free (to);
+			g_free (cc);
+			return NULL;
+		}
 	}
 
 	if (!deleted) {
@@ -946,67 +1114,70 @@
 	g_free (to);
 	g_free (cc);
 
-	if (!read_summary (self->summary,
-			   SUMMARY_TYPE_INT32, NULL,
-			   SUMMARY_TYPE_INT32, NULL,
-			   SUMMARY_TYPE_UINT32, &count,
-			   -1)) {
-		goto corruption;
-	}
+	if (!self->stmt) {
+		if (!read_summary (self->summary,
+				   SUMMARY_TYPE_INT32, NULL,
+				   SUMMARY_TYPE_INT32, NULL,
+				   SUMMARY_TYPE_UINT32, &count,
+				   -1)) {
+			goto corruption;
+		}
 
-	/* references */
-	for (i = 0; i < count; i++) {
-		if (read_summary (self->summary,
-				  SUMMARY_TYPE_INT32, NULL,
-				  SUMMARY_TYPE_INT32, NULL,
-				  -1)) {
-			continue;
+		/* references */
+		for (i = 0; i < count; i++) {
+			if (read_summary (self->summary,
+					  SUMMARY_TYPE_INT32, NULL,
+					  SUMMARY_TYPE_INT32, NULL,
+					  -1)) {
+				continue;
+			}
+
+			goto corruption;
 		}
 
-		goto corruption;
-	}
+		if (!read_summary (self->summary, SUMMARY_TYPE_UINT32, &count, -1)) {
+			goto corruption;
+		}
 
-	if (!read_summary (self->summary, SUMMARY_TYPE_UINT32, &count, -1)) {
-		goto corruption;
-	}
+		/* user flags */
+		for (i = 0; i < count; i++) {
+			if (read_summary (self->summary, SUMMARY_TYPE_STRING, NULL, -1)) {
+				continue;
+			}
 
-	/* user flags */
-	for (i = 0; i < count; i++) {
-		if (read_summary (self->summary, SUMMARY_TYPE_STRING, NULL, -1)) {
-			continue;
+			goto corruption;
 		}
 
-		goto corruption;
-	}
+		if (!read_summary (self->summary, SUMMARY_TYPE_UINT32, &count, -1)) {
+			goto corruption;
+		}
 
-	if (!read_summary (self->summary, SUMMARY_TYPE_UINT32, &count, -1)) {
-		goto corruption;
-	}
+		/* user tags */
+		for (i = 0; i < count; i++) {
+			if (read_summary (self->summary,
+					  SUMMARY_TYPE_STRING, NULL,
+					  SUMMARY_TYPE_STRING, NULL,
+					  -1)) {
+				continue;
+			}
 
-	/* user tags */
-	for (i = 0; i < count; i++) {
-		if (read_summary (self->summary,
-				  SUMMARY_TYPE_STRING, NULL,
-				  SUMMARY_TYPE_STRING, NULL,
-				  -1)) {
-			continue;
+			goto corruption;
 		}
 
-		goto corruption;
-	}
+		/* server flags */
+		if (!read_summary (self->summary,
+				   SUMMARY_TYPE_UINT32, NULL,
+				   -1)) {
+			goto corruption;
+		}
 
-	/* server flags */
-	if (!read_summary (self->summary,
-			   SUMMARY_TYPE_UINT32, NULL,
-			   -1)) {
-		goto corruption;
+		skip_content_info (self->summary);
 	}
 
-	skip_content_info (self->summary);
-
 	return metadata;
 
-corruption:
+	corruption:
+
 	/* assume corruption */
 	if (metadata) {
 		g_object_unref (metadata);
@@ -1130,9 +1301,14 @@
         self->cur_message_uid = NULL;
 
         /* save current message uid */
-        read_summary (self->summary,
-                      SUMMARY_TYPE_STRING, &self->cur_message_uid, /* message uid */
-                      -1);
+	if (self->stmt) {
+		self->cur_message_uid = g_strdup (sqlite3_column_text (self->stmt, 0));
+		sqlite3_step (self->stmt);
+	} else {
+		read_summary (self->summary,
+			      SUMMARY_TYPE_STRING, &self->cur_message_uid, /* message uid */
+			      -1);
+	}
 
         self->cur_message++;
 
Index: src/tracker-indexer/modules/evolution-imap.h
===================================================================
--- src/tracker-indexer/modules/evolution-imap.h	(revision 2701)
+++ src/tracker-indexer/modules/evolution-imap.h	(working copy)
@@ -27,6 +27,7 @@
 
 #include <glib.h>
 #include <glib/gstdio.h>
+#include <sqlite3.h>
 
 #include <tracker-indexer/tracker-module-file.h>
 
@@ -56,6 +57,8 @@
 
         GList *mime_parts;
         GList *current_mime_part;
+	sqlite3 *db;
+	sqlite3_stmt *stmt;
 };
 
 struct TrackerEvolutionImapFileClass {
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]