GNOME.org
 

evolution-data-server r9843 - in branches/gnome-2-24/camel: . providers/local

Author: psankar
Date: Mon Dec 22 06:32:45 2008
New Revision: 9843
URL: http://svn.gnome.org/viewvc/evolution-data-server?rev=9843&view=rev

Log:
2008-12-22  Sankar P  <psankar novell com>

	** Part of Fix for bug #559153

	* camel/camel-db.c:
	* camel/camel-db.h:
	* camel/camel-folder-summary.c:
	* camel/camel-store.c:
	* camel/camel-store.h:
	* camel/providers/local/camel-local-folder.c:
	Migration improvements




Modified:
   branches/gnome-2-24/camel/ChangeLog
   branches/gnome-2-24/camel/camel-db.c
   branches/gnome-2-24/camel/camel-db.h
   branches/gnome-2-24/camel/camel-folder-summary.c
   branches/gnome-2-24/camel/camel-store.c
   branches/gnome-2-24/camel/camel-store.h
   branches/gnome-2-24/camel/providers/local/camel-local-folder.c

Modified: branches/gnome-2-24/camel/camel-db.c
==============================================================================
--- branches/gnome-2-24/camel/camel-db.c	(original)
+++ branches/gnome-2-24/camel/camel-db.c	Mon Dec 22 06:32:45 2008
@@ -47,6 +47,8 @@
 
 static GStaticRecMutex trans_lock = G_STATIC_REC_MUTEX_INIT;	
 
+static int write_mir (CamelDB *cdb, const char *folder_name, CamelMIRecord *record, CamelException *ex, gboolean delete_old_record);
+
 static int 
 cdb_sql_exec (sqlite3 *db, const char* stmt, CamelException *ex) 
 {
@@ -811,13 +813,29 @@
 	return ret;
 }
 
+
+int 
+camel_db_write_fresh_message_info_record (CamelDB *cdb, const char *folder_name, CamelMIRecord *record, CamelException *ex)
+{
+	return write_mir (cdb, folder_name, record, ex, FALSE);
+}
+
+
 int
 camel_db_write_message_info_record (CamelDB *cdb, const char *folder_name, CamelMIRecord *record, CamelException *ex)
 {
+	return write_mir (cdb, folder_name, record, ex, TRUE);
+}
+
+static int 
+write_mir (CamelDB *cdb, const char *folder_name, CamelMIRecord *record, CamelException *ex, gboolean delete_old_record)
+{
 	int ret;
 	char *del_query;
 	char *ins_query;
 
+	/* FIXME: We should migrate from this DELETE followed by INSERT model to an INSERT OR REPLACE model as pointed out by pvanhoof */
+
 	/* NB: UGLIEST Hack. We can't modify the schema now. We are using msg_security (an unsed one to notify of FLAGGED/Dirty infos */
 
 	ins_query = sqlite3_mprintf ("INSERT INTO %Q VALUES (%Q, %d, %d, %d, %d, %d, %d, %d, %d, %d, %d, %ld, %ld, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q, %Q )", 
@@ -831,7 +849,8 @@
 			record->part, record->labels, record->usertags,
 			record->cinfo, record->bdata);
 
-	del_query = sqlite3_mprintf ("DELETE FROM %Q WHERE uid = %Q", folder_name, record->uid);
+	if (delete_old_record)
+			del_query = sqlite3_mprintf ("DELETE FROM %Q WHERE uid = %Q", folder_name, record->uid);
 
 #if 0
 	char *upd_query;
@@ -841,12 +860,14 @@
 	g_free (upd_query);
 #else
 
-	ret = camel_db_add_to_transaction (cdb, del_query, ex);
+	if (delete_old_record)
+			ret = camel_db_add_to_transaction (cdb, del_query, ex);
 	ret = camel_db_add_to_transaction (cdb, ins_query, ex);
 
 #endif
 
-	sqlite3_free (del_query);
+	if (delete_old_record)
+			sqlite3_free (del_query);
 	sqlite3_free (ins_query);
 
 	return ret;
@@ -1197,3 +1218,39 @@
 	CAMEL_DB_RELEASE_SQLITE_MEMORY;
 	return ret;		
 }
+
+int camel_db_start_in_memory_transactions (CamelDB *cdb, CamelException *ex)
+{
+	int ret;
+	char *cmd = sqlite3_mprintf ("ATTACH DATABASE ':memory:' AS %s", CAMEL_DB_IN_MEMORY_DB);
+
+	ret = camel_db_command (cdb, cmd, ex);
+	sqlite3_free (cmd);
+
+	cmd = sqlite3_mprintf ("CREATE TEMPORARY TABLE %Q (  uid TEXT PRIMARY KEY , flags INTEGER , msg_type INTEGER , read INTEGER , deleted INTEGER , replied INTEGER , important INTEGER , junk INTEGER , attachment INTEGER , msg_security INTEGER , size INTEGER , dsent NUMERIC , dreceived NUMERIC , subject TEXT , mail_from TEXT , mail_to TEXT , mail_cc TEXT , mlist TEXT , followup_flag TEXT , followup_completed_on TEXT , followup_due_by TEXT , part TEXT , labels TEXT , usertags TEXT , cinfo TEXT , bdata TEXT )", CAMEL_DB_IN_MEMORY_TABLE);
+	ret = camel_db_command (cdb, cmd, ex);
+	if (ret != 0 )
+		abort ();
+	sqlite3_free (cmd);
+	
+	return ret;
+}
+
+int camel_db_flush_in_memory_transactions (CamelDB *cdb, const char * folder_name, CamelException *ex)
+{
+	int ret;
+	char *cmd = sqlite3_mprintf ("INSERT INTO %Q SELECT * FROM %Q", folder_name, CAMEL_DB_IN_MEMORY_TABLE);
+
+	ret = camel_db_command (cdb, cmd, ex);
+	sqlite3_free (cmd);
+
+	cmd = sqlite3_mprintf ("DROP TABLE %Q", CAMEL_DB_IN_MEMORY_TABLE);
+	ret = camel_db_command (cdb, cmd, ex);
+	sqlite3_free (cmd);
+
+	cmd = sqlite3_mprintf ("DETACH %Q", CAMEL_DB_IN_MEMORY_DB);
+	ret = camel_db_command (cdb, cmd, ex);
+	sqlite3_free (cmd);
+
+	return ret;
+}

Modified: branches/gnome-2-24/camel/camel-db.h
==============================================================================
--- branches/gnome-2-24/camel/camel-db.h	(original)
+++ branches/gnome-2-24/camel/camel-db.h	Mon Dec 22 06:32:45 2008
@@ -4,8 +4,14 @@
 #define __CAMEL_DB_H
 #include <sqlite3.h>
 #include <glib.h>
+
 #define CAMEL_DB_FILE "folders.db"
 
+/* Hopefully no one will create a folder named EVO_IN_meM_hAnDlE */
+#define CAMEL_DB_IN_MEMORY_TABLE "EVO_IN_meM_hAnDlE.temp" 
+#define CAMEL_DB_IN_MEMORY_DB "EVO_IN_meM_hAnDlE" 
+#define CAMEL_DB_IN_MEMORY_TABLE_LIMIT 100000
+
 #include "camel-exception.h"
 
 typedef struct _CamelDBPrivate CamelDBPrivate;
@@ -132,6 +138,7 @@
 int camel_db_prepare_message_info_table (CamelDB *cdb, const char *folder_name, CamelException *ex);
 
 int camel_db_write_message_info_record (CamelDB *cdb, const char *folder_name, CamelMIRecord *record, CamelException *ex);
+int camel_db_write_fresh_message_info_record (CamelDB *cdb, const char *folder_name, CamelMIRecord *record, CamelException *ex);
 int camel_db_read_message_info_records (CamelDB *cdb, const char *folder_name, gpointer p, CamelDBSelectCB read_mir_callback, CamelException *ex);
 int camel_db_read_message_info_record_with_uid (CamelDB *cdb, const char *folder_name, const char *uid, gpointer p, CamelDBSelectCB read_mir_callback, CamelException *ex);
 
@@ -168,5 +175,9 @@
 int camel_db_set_collate (CamelDB *cdb, const char *col, const char *collate, CamelDBCollate func);
 /* Migration APIS */
 int camel_db_migrate_vfolders_to_14(CamelDB *cdb, const char *folder, CamelException *ex);
+
+int camel_db_start_in_memory_transactions (CamelDB *cdb, CamelException *ex);
+int camel_db_flush_in_memory_transactions (CamelDB *cdb, const char * folder_name, CamelException *ex);
+
 #endif
 

Modified: branches/gnome-2-24/camel/camel-folder-summary.c
==============================================================================
--- branches/gnome-2-24/camel/camel-folder-summary.c	(original)
+++ branches/gnome-2-24/camel/camel-folder-summary.c	Mon Dec 22 06:32:45 2008
@@ -128,7 +128,7 @@
 static int		         content_info_save(CamelFolderSummary *, FILE *, CamelMessageContentInfo *);
 static void		         content_info_free(CamelFolderSummary *, CamelMessageContentInfo *);
 
-static int save_message_infos_to_db (CamelFolderSummary *s, CamelException *ex);
+static int save_message_infos_to_db (CamelFolderSummary *s, gboolean fresh_mir, CamelException *ex);
 static int camel_read_mir_callback (void * ref, int ncol, char ** cols, char ** name);
 
 static char *next_uid_string(CamelFolderSummary *s);
@@ -1233,23 +1233,20 @@
 	if (fclose (in) != 0)
 		return -1;
 
-
-	camel_db_begin_transaction (cdb, &ex);
-
-	ret = save_message_infos_to_db (s, &ex);
-
-	if (ret != 0) {
-		camel_db_abort_transaction (cdb, &ex);
+	record = (((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_to_db (s, &ex));
+	if (!record) {
 		return -1;
 	}
-	camel_db_end_transaction (cdb, &ex);
 
-	record = (((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_to_db (s, &ex));
-	if (!record) {
+	ret = save_message_infos_to_db (s, TRUE, &ex);
+
+	if (ret != 0) {
 		return -1;
 	}
 	
+	camel_db_begin_transaction (cdb, &ex);
 	ret = camel_db_write_folder_info_record (cdb, record, &ex);
+	camel_db_end_transaction (cdb, &ex);
 
 	g_free (record->bdata);
 	g_free (record);
@@ -1317,17 +1314,24 @@
 	return 0;
 }
 
+typedef struct {
+	CamelException *ex;
+	gboolean migration;
+	int progress;
+} SaveToDBArgs;
+
 static void
 save_to_db_cb (gpointer key, gpointer value, gpointer data)
 {
-	CamelException *ex = (CamelException *)data;
+	SaveToDBArgs *args = (SaveToDBArgs *) data;
+	CamelException *ex = args->ex;
 	CamelMessageInfoBase *mi = (CamelMessageInfoBase *)value;	
 	CamelFolderSummary *s = (CamelFolderSummary *)mi->summary;
 	char *folder_name = s->folder->full_name;
 	CamelDB *cdb = s->folder->parent_store->cdb_w;
 	CamelMIRecord *mir;
 
-	if (!mi->dirty)
+	if (!args->migration && !mi->dirty)
 		return;
 
 	mir = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->message_info_to_db (s, (CamelMessageInfo *)mi);
@@ -1341,22 +1345,44 @@
 		}
 	}
 
-	if (camel_db_write_message_info_record (cdb, folder_name, mir, ex) != 0) {
-		camel_db_camel_mir_free (mir);
-		return;
+	if (!args->migration) {
+			if (camel_db_write_message_info_record (cdb, folder_name, mir, ex) != 0) {
+					camel_db_camel_mir_free (mir);
+					return;
+			}
+	} else {
+			if (camel_db_write_fresh_message_info_record (cdb, CAMEL_DB_IN_MEMORY_TABLE, mir, ex) != 0) {
+					camel_db_camel_mir_free (mir);
+					return;
+			}
+
+			if (args->progress > CAMEL_DB_IN_MEMORY_TABLE_LIMIT) {
+			    g_print ("BULK INsert limit reached \n");
+				camel_db_flush_in_memory_transactions (cdb, folder_name, ex);
+				camel_db_start_in_memory_transactions (cdb, ex);
+				args->progress = 0;
+			} else {
+				args->progress ++;
+			}
 	}
 
 	/* Reset the flags */
 	mi->dirty = FALSE;
+	mi->flags &= ~CAMEL_MESSAGE_FOLDER_FLAGGED;
 	
 	camel_db_camel_mir_free (mir);	
 }
 
 static int
-save_message_infos_to_db (CamelFolderSummary *s, CamelException *ex)
+save_message_infos_to_db (CamelFolderSummary *s, gboolean fresh_mirs, CamelException *ex)
 {
 	CamelDB *cdb = s->folder->parent_store->cdb_w;
 	char *folder_name;
+	SaveToDBArgs args;
+
+	args.ex = ex;
+	args.migration = fresh_mirs;
+	args.progress = 0;
 
 	folder_name = s->folder->full_name;
 	if (camel_db_prepare_message_info_table (cdb, folder_name, ex) != 0) {
@@ -1364,7 +1390,7 @@
 	}
 	CAMEL_SUMMARY_LOCK(s, summary_lock);
 	/* Push MessageInfo-es */
-	g_hash_table_foreach (s->loaded_infos, save_to_db_cb, ex);
+	g_hash_table_foreach (s->loaded_infos, save_to_db_cb, &args);
 	CAMEL_SUMMARY_UNLOCK(s, summary_lock);
 #warning "make sure we free the message infos that are loaded are freed if not used anymore or should we leave that to the timer? "
 	
@@ -1393,7 +1419,7 @@
 
 	camel_db_begin_transaction (cdb, ex);
 
-	ret = save_message_infos_to_db (s, ex);
+	ret = save_message_infos_to_db (s, FALSE, ex);
 	if (ret != 0) {
 		camel_db_abort_transaction (cdb, ex);
 		/* Failed, so lets reset the flag */

Modified: branches/gnome-2-24/camel/camel-store.c
==============================================================================
--- branches/gnome-2-24/camel/camel-store.c	(original)
+++ branches/gnome-2-24/camel/camel-store.c	Mon Dec 22 06:32:45 2008
@@ -322,6 +322,21 @@
 	}
 
 	if (!folder) {
+
+		if (flags & CAMEL_STORE_IS_MIGRATING) {
+				if ((store->flags & CAMEL_STORE_VTRASH) && strcmp(folder_name, CAMEL_VTRASH_NAME) == 0) {
+						if (store->folders) 
+								camel_object_bag_abort(store->folders, folder_name);
+						return NULL;
+				}
+
+				if ((store->flags & CAMEL_STORE_VJUNK) && strcmp(folder_name, CAMEL_VJUNK_NAME) == 0) {
+						if (store->folders) 
+								camel_object_bag_abort(store->folders, folder_name);
+						return NULL;
+				}
+		}
+
 		if ((store->flags & CAMEL_STORE_VTRASH) && strcmp(folder_name, CAMEL_VTRASH_NAME) == 0) {
 			folder = CS_CLASS(store)->get_trash(store, ex);
 		} else if ((store->flags & CAMEL_STORE_VJUNK) && strcmp(folder_name, CAMEL_VJUNK_NAME) == 0) {

Modified: branches/gnome-2-24/camel/camel-store.h
==============================================================================
--- branches/gnome-2-24/camel/camel-store.h	(original)
+++ branches/gnome-2-24/camel/camel-store.h	Mon Dec 22 06:32:45 2008
@@ -115,6 +115,7 @@
 #define CAMEL_STORE_FILTER_INBOX	(1 << 2)
 #define CAMEL_STORE_VJUNK		(1 << 3)
 #define CAMEL_STORE_PROXY		(1 << 4)
+#define CAMEL_STORE_IS_MIGRATING (1 << 5)
 
 struct _CamelDB;
 

Modified: branches/gnome-2-24/camel/providers/local/camel-local-folder.c
==============================================================================
--- branches/gnome-2-24/camel/providers/local/camel-local-folder.c	(original)
+++ branches/gnome-2-24/camel/providers/local/camel-local-folder.c	Mon Dec 22 06:32:45 2008
@@ -289,7 +289,7 @@
 	}
 
 	folder->summary = (CamelFolderSummary *)CLOCALF_CLASS(lf)->create_summary(lf, lf->summary_path, lf->folder_path, lf->index);
-	if (camel_local_summary_load((CamelLocalSummary *)folder->summary, forceindex, NULL) == -1) {
+	if (!(flags & CAMEL_STORE_IS_MIGRATING) && camel_local_summary_load((CamelLocalSummary *)folder->summary, forceindex, NULL) == -1) {
 		/* ? */
 		if (camel_local_summary_check((CamelLocalSummary *)folder->summary, lf->changes, ex) == 0) {
 			/* we sync here so that any hard work setting up the folder isn't lost */
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]