[evolution-data-server] Bug #545505 - Properly free unused message infos periodically

[Milan Crha <mcrha src gnome org>]

commit b0b52e49e5a0e99bc3bad401409d118dca1b5f1e
Author: Milan Crha <mcrha redhat com>
Date:   Thu Apr 29 16:02:17 2010 +0200

    Bug #545505 - Properly free unused message infos periodically

 camel/camel-disco-folder.c                         |    1 +
 camel/camel-folder-search.c                        |    4 +-
 camel/camel-folder-summary.c                       |  160 ++++++++++----------
 camel/camel-folder-summary.h                       |    6 +-
 camel/camel-folder-thread.c                        |    3 +-
 camel/camel-vee-folder.c                           |    9 +-
 camel/camel-vtrash-folder.c                        |    1 +
 camel/providers/groupwise/camel-groupwise-folder.c |    2 +
 camel/providers/imap/camel-imap-folder.c           |    4 +-
 camel/providers/local/camel-maildir-folder.c       |   13 ++-
 camel/providers/local/camel-maildir-summary.c      |    6 +-
 camel/providers/local/camel-mbox-folder.c          |   11 +-
 camel/providers/local/camel-mbox-summary.c         |    5 +-
 camel/providers/local/camel-mh-summary.c           |    2 +
 camel/providers/local/camel-spool-summary.c        |    1 +
 camel/providers/nntp/camel-nntp-folder.c           |   42 ++++-
 16 files changed, 160 insertions(+), 110 deletions(-)
---
diff --git a/camel/camel-disco-folder.c b/camel/camel-disco-folder.c
index 364cc23..1095312 100644
--- a/camel/camel-disco-folder.c
+++ b/camel/camel-disco-folder.c
@@ -268,6 +268,7 @@ disco_expunge (CamelFolder *folder,
 	gboolean success;
 
 	uids = g_ptr_array_new ();
+	camel_folder_summary_prepare_fetch_all (folder->summary, ex);
 	count = camel_folder_summary_count (folder->summary);
 	for (i = 0; i < count; i++) {
 		info = camel_folder_summary_index (folder->summary, i);
diff --git a/camel/camel-folder-search.c b/camel/camel-folder-search.c
index 7483c70..bfca34b 100644
--- a/camel/camel-folder-search.c
+++ b/camel/camel-folder-search.c
@@ -849,8 +849,8 @@ search_match_all(struct _ESExp *f, gint argc, struct _ESExpTerm **argv, CamelFol
 
 	v = search->summary_set?search->summary_set:search->summary;
 
-	if (v->len > g_hash_table_size (search->folder->summary->loaded_infos) && !CAMEL_IS_VEE_FOLDER (search->folder)) {
-		camel_folder_summary_reload_from_db (search->folder->summary, search->priv->ex);
+	if (!CAMEL_IS_VEE_FOLDER (search->folder)) {
+		camel_folder_summary_prepare_fetch_all (search->folder->summary, search->priv->ex);
 	}
 
 	for (i=0;i<v->len;i++) {
diff --git a/camel/camel-folder-summary.c b/camel/camel-folder-summary.c
index e37583d..550b108 100644
--- a/camel/camel-folder-summary.c
+++ b/camel/camel-folder-summary.c
@@ -63,7 +63,7 @@
 	((obj), CAMEL_TYPE_FOLDER_SUMMARY, CamelFolderSummaryPrivate))
 
 /* Make 5 minutes as default cache drop */
-#define SUMMARY_CACHE_DROP 300 
+#define SUMMARY_CACHE_DROP 300
 #define dd(x) if (camel_debug("sync")) x
 
 struct _CamelFolderSummaryPrivate {
@@ -80,11 +80,11 @@ struct _CamelFolderSummaryPrivate {
 
 	struct _CamelIndex *index;
 
-	GMutex *summary_lock;	/* for the summary hashtable/array */
-	GMutex *io_lock;	/* load/save lock, for access to saved_count, etc */
-	GMutex *filter_lock;	/* for accessing any of the filtering/indexing stuff, since we share them */
-	GMutex *alloc_lock;	/* for setting up and using allocators */
-	GMutex *ref_lock;	/* for reffing/unreffing messageinfo's ALWAYS obtain before summary_lock */
+	GStaticRecMutex summary_lock;	/* for the summary hashtable/array */
+	GStaticRecMutex io_lock;	/* load/save lock, for access to saved_count, etc */
+	GStaticRecMutex filter_lock;	/* for accessing any of the filtering/indexing stuff, since we share them */
+	GStaticRecMutex alloc_lock;	/* for setting up and using allocators */
+	GStaticRecMutex ref_lock;	/* for reffing/unreffing messageinfo's ALWAYS obtain before summary_lock */
 	GHashTable *flag_cache;
 
 	gboolean need_preview;
@@ -122,6 +122,8 @@ struct _node {
 	struct _node *next;
 };
 
+static void cfs_schedule_info_release_timer (CamelFolderSummary *s);
+
 static struct _node *my_list_append(struct _node **list, struct _node *n);
 static gint my_list_size(struct _node **list);
 
@@ -239,11 +241,11 @@ folder_summary_finalize (GObject *object)
 	g_free(summary->meta_summary->path);
 	g_free(summary->meta_summary);
 
-	g_mutex_free(summary->priv->summary_lock);
-	g_mutex_free(summary->priv->io_lock);
-	g_mutex_free(summary->priv->filter_lock);
-	g_mutex_free(summary->priv->alloc_lock);
-	g_mutex_free(summary->priv->ref_lock);
+	g_static_rec_mutex_free (&summary->priv->summary_lock);
+	g_static_rec_mutex_free (&summary->priv->io_lock);
+	g_static_rec_mutex_free (&summary->priv->filter_lock);
+	g_static_rec_mutex_free (&summary->priv->alloc_lock);
+	g_static_rec_mutex_free (&summary->priv->ref_lock);
 
 	/* Chain up to parent's finalize() method. */
 	G_OBJECT_CLASS (camel_folder_summary_parent_class)->finalize (object);
@@ -959,11 +961,11 @@ camel_folder_summary_init (CamelFolderSummary *summary)
 	summary->uids = g_ptr_array_new ();
 	summary->loaded_infos = g_hash_table_new (g_str_hash, g_str_equal);
 
-	summary->priv->summary_lock = g_mutex_new();
-	summary->priv->io_lock = g_mutex_new();
-	summary->priv->filter_lock = g_mutex_new();
-	summary->priv->alloc_lock = g_mutex_new();
-	summary->priv->ref_lock = g_mutex_new();
+	g_static_rec_mutex_init (&summary->priv->summary_lock);
+	g_static_rec_mutex_init (&summary->priv->io_lock);
+	g_static_rec_mutex_init (&summary->priv->filter_lock);
+	g_static_rec_mutex_init (&summary->priv->alloc_lock);
+	g_static_rec_mutex_init (&summary->priv->ref_lock);
 
 	summary->meta_summary = g_malloc0(sizeof(CamelFolderMetaSummary));
 
@@ -1060,6 +1062,8 @@ camel_folder_summary_set_build_content(CamelFolderSummary *s, gboolean state)
  * Get the number of summary items stored in this summary.
  *
  * Returns: the number of items in the summary
+ *
+ * @see camel_folder_summary_prepare_fetch_all()
  **/
 guint
 camel_folder_summary_count(CamelFolderSummary *s)
@@ -1078,6 +1082,8 @@ camel_folder_summary_count(CamelFolderSummary *s)
  * ref'd or free'd as appropriate.
  *
  * Returns: the summary item, or %NULL if @index is out of range
+ *
+ * @see camel_folder_summary_prepare_fetch_all()
  **/
 CamelMessageInfo *
 camel_folder_summary_index (CamelFolderSummary *s, gint i)
@@ -1256,7 +1262,6 @@ camel_folder_summary_peek_info (CamelFolderSummary *s, const gchar *uid)
 
 struct _db_pass_data {
 	CamelFolderSummary *summary;
-	gboolean double_ref;
 	gboolean add; /* or just insert to hashtable */
 };
 
@@ -1283,22 +1288,20 @@ message_info_from_uid (CamelFolderSummary *s, const gchar *uid)
 		folder_name = s->folder->full_name;
 		cdb = s->folder->parent_store->cdb_r;
 
-		camel_folder_summary_unlock (s, CFS_SUMMARY_LOCK);
-
 		data.summary = s;
-		data.double_ref = TRUE;
 		data.add = FALSE;
 
 		ret = camel_db_read_message_info_record_with_uid (
 			cdb, folder_name, uid, &data,
 			camel_read_mir_callback, &ex);
 		if (ret != 0) {
+			camel_folder_summary_unlock (s, CFS_SUMMARY_LOCK);
+			if (camel_exception_is_set (&ex))
+				g_warning ("%s: Failed read '%s' in '%s' from db: %s (0x%x)", G_STRFUNC, uid, folder_name, camel_exception_get_description (&ex), camel_exception_get_id (&ex));
 			camel_exception_clear (&ex);
 			return NULL;
 		}
 
-		camel_folder_summary_lock (s, CFS_SUMMARY_LOCK);
-
 		/* We would have double reffed at camel_read_mir_callback */
 		info = g_hash_table_lookup (s->loaded_infos, uid);
 
@@ -1311,6 +1314,8 @@ message_info_from_uid (CamelFolderSummary *s, const gchar *uid)
 			camel_exception_clear (&ex);
 			g_free (errmsg);
 		}
+
+		cfs_schedule_info_release_timer (s);
 	}
 
 	if (info)
@@ -1332,6 +1337,8 @@ message_info_from_uid (CamelFolderSummary *s, const gchar *uid)
  * ref'd or free'd as appropriate.
  *
  * Returns: the summary item, or %NULL if the uid @uid is not available
+ *
+ * @see camel_folder_summary_prepare_fetch_all()
  **/
 CamelMessageInfo *
 camel_folder_summary_uid (CamelFolderSummary *summary,
@@ -1547,18 +1554,14 @@ remove_cache (CamelSession *session, CamelSessionThreadMsg *msg)
 {
 	struct _folder_summary_free_msg *m = (struct _folder_summary_free_msg *)msg;
 	CamelFolderSummary *s = m->summary;
-	CamelException ex;
 	GSList *to_free_list = NULL, *l;
 
 	CAMEL_DB_RELEASE_SQLITE_MEMORY;
-	camel_exception_init (&ex);
-	camel_folder_sync (s->folder, FALSE, &ex);
-	camel_exception_clear (&ex);
 
 	if (time(NULL) - s->cache_load_time < SUMMARY_CACHE_DROP)
 		return;
 
-	dd(printf("removing cache for  %s %d %p\n", s->folder ? s->folder->full_name : s->summary_path, g_hash_table_size (s->loaded_infos), (gpointer) s->loaded_infos));
+	dd(printf("removing cache for  %s %d %p ", s->folder ? s->folder->full_name : s->summary_path, g_hash_table_size (s->loaded_infos), (gpointer) s->loaded_infos));
 	/* FIXME[disk-summary] hack. fix it */
 	camel_folder_summary_lock (s, CFS_SUMMARY_LOCK);
 
@@ -1572,8 +1575,8 @@ remove_cache (CamelSession *session, CamelSessionThreadMsg *msg)
 		camel_message_info_free (l->data);
 	g_slist_free (to_free_list);
 
+	dd(printf("   done .. now %d\n", g_hash_table_size (s->loaded_infos)));
 	camel_folder_summary_unlock (s, CFS_SUMMARY_LOCK);
-	dd(printf("done .. now %d\n",g_hash_table_size (s->loaded_infos)));
 
 	s->cache_load_time = time(NULL);
 }
@@ -1614,13 +1617,30 @@ cfs_try_release_memory (CamelFolderSummary *s)
 	return TRUE;
 }
 
-/**
- * camel_folder_summary_cache_size:
- *
- * Since: 2.24
- **/
-gint
-camel_folder_summary_cache_size (CamelFolderSummary *s)
+static void
+cfs_schedule_info_release_timer (CamelFolderSummary *s)
+{
+	g_return_if_fail (CAMEL_IS_FOLDER_SUMMARY (s));
+
+	if (!s->timeout_handle) {
+		static gboolean know_can_do = FALSE, can_do = TRUE;
+
+		if (!know_can_do) {
+			can_do = !g_getenv ("CAMEL_FREE_INFOS");
+			know_can_do = TRUE;
+		}
+
+		/* FIXME[disk-summary] LRU please and not timeouts */
+		if (can_do)
+			s->timeout_handle = g_timeout_add_seconds (SUMMARY_CACHE_DROP, (GSourceFunc) cfs_try_release_memory, s);
+	}
+
+	/* update also cache load time to the actual, to not release something just loaded */
+	s->cache_load_time = time (NULL);
+}
+
+static gint
+cfs_cache_size (CamelFolderSummary *s)
 {
 	/* FIXME[disk-summary] this is a timely hack. fix it well */
 	if (!CAMEL_IS_VEE_FOLDER(s->folder))
@@ -1725,14 +1745,8 @@ static CamelSessionThreadOps preview_update_ops = {
 
 /* end */
 
-/**
- * camel_folder_summary_reload_from_db:
- *
- * Since: 2.24
- **/
-gint
-camel_folder_summary_reload_from_db (CamelFolderSummary *s,
-                                     CamelException *ex)
+static gint
+cfs_reload_from_db (CamelFolderSummary *s, CamelException *ex)
 {
 	CamelDB *cdb;
 	gchar *folder_name;
@@ -1748,14 +1762,10 @@ camel_folder_summary_reload_from_db (CamelFolderSummary *s,
 
 	/* FIXME FOR SANKAR: No need to pass the address of summary here. */
 	data.summary = s;
-	data.double_ref = FALSE;
 	data.add = FALSE;
 	ret = camel_db_read_message_info_records (cdb, folder_name, (gpointer)&data, camel_read_mir_callback, NULL);
 
-	s->cache_load_time = time (NULL);
-        /* FIXME[disk-summary] LRU please and not timeouts */
-	if (!g_getenv("CAMEL_FREE_INFOS") && !s->timeout_handle)
-		s->timeout_handle = g_timeout_add_seconds (SUMMARY_CACHE_DROP, (GSourceFunc) cfs_try_release_memory, s);
+	cfs_schedule_info_release_timer (s);
 
 	if (CAMEL_FOLDER_SUMMARY_GET_PRIVATE(s)->need_preview) {
 		struct _preview_update_msg *m;
@@ -1783,31 +1793,32 @@ camel_folder_summary_add_preview (CamelFolderSummary *s, CamelMessageInfo *info)
 }
 
 /**
- * camel_folder_summary_ensure_infos_loaded:
+ * camel_folder_summary_prepare_fetch_all:
  * @s: #CamelFolderSummary object
- * @at_least: How many infos already loaded are considered fine to not reload all of them.
- *    Use -1 to force reload of all of them if not in memory yet.
  * @ex: #CamelException object.
  *
- * Loads all infos into memory, if they are not yet.
+ * Loads all infos into memory, if they are not yet and ensures
+ * they will not be freed in next couple minutes. Call this function
+ * before any mass operation or when all message infos will be needed,
+ * for better performance.
  *
- * Since: 2.28
+ * Since: 2.31
  **/
 void
-camel_folder_summary_ensure_infos_loaded (CamelFolderSummary *s,
-                                          gint at_least,
-                                          CamelException *ex)
+camel_folder_summary_prepare_fetch_all (CamelFolderSummary *s, CamelException *ex)
 {
 	guint loaded, known;
 
 	g_return_if_fail (s != NULL);
 
-	loaded = camel_folder_summary_cache_size (s);
+	loaded = cfs_cache_size (s);
 	known = camel_folder_summary_count (s);
 
-	if ((at_least == -1 && known != loaded) || at_least > loaded) {
-		camel_folder_summary_reload_from_db (s, ex);
-	}
+	if (known - loaded > 50)
+		cfs_reload_from_db (s, ex);
+
+	/* update also cache load time, even when not loaded anything */
+	s->cache_load_time = time (NULL);
 }
 
 #if 0
@@ -1884,7 +1895,6 @@ camel_folder_summary_load_from_db (CamelFolderSummary *s,
 #if 0
 	data.summary = s;
 	data.add = TRUE;
-	data.double_ref = FALSE;
 	ret = camel_db_read_message_info_records (cdb, folder_name, (gpointer) &data, camel_read_mir_callback, ex);
 #endif
 
@@ -1995,10 +2005,6 @@ camel_read_mir_callback (gpointer  ref, gint ncol, gchar ** cols, gchar ** name)
 			}
 		}
 
-		if (data->double_ref)
-			/* double reffing, because, at times frees before, I could read it. so we dont ref and give it again, just use it */
-			camel_message_info_ref(info);
-
 		/* Just now we are reading from the DB, it can't be dirty. */
 		((CamelMessageInfoBase *)info)->dirty = FALSE;
 		if (data->add)
@@ -3219,7 +3225,7 @@ camel_folder_summary_remove_uid(CamelFolderSummary *s, const gchar *uid)
 		camel_folder_summary_lock (s, CFS_REF_LOCK);
 		if (g_hash_table_lookup_extended(s->loaded_infos, uid, (gpointer)&olduid, (gpointer)&oldinfo)) {
 				/* make sure it doesn't vanish while we're removing it */
-				oldinfo->refcount++;
+				camel_message_info_ref (oldinfo);
 				camel_folder_summary_unlock (s, CFS_REF_LOCK);
 				camel_folder_summary_unlock (s, CFS_SUMMARY_LOCK);
 				camel_folder_summary_remove(s, oldinfo);
@@ -3257,7 +3263,7 @@ camel_folder_summary_remove_uid_fast (CamelFolderSummary *s, const gchar *uid)
 		camel_folder_summary_lock (s, CFS_REF_LOCK);
 		if (g_hash_table_lookup_extended(s->loaded_infos, uid, (gpointer)&olduid, (gpointer)&oldinfo)) {
 				/* make sure it doesn't vanish while we're removing it */
-				oldinfo->refcount++;
+				camel_message_info_ref (oldinfo);
 				camel_folder_summary_unlock (s, CFS_REF_LOCK);
 				g_hash_table_remove (s->loaded_infos, olduid);
 				summary_remove_uid (s, olduid);
@@ -5283,19 +5289,19 @@ camel_folder_summary_lock (CamelFolderSummary *summary,
 
 	switch (lock) {
 		case CFS_SUMMARY_LOCK:
-			g_mutex_lock (summary->priv->summary_lock);
+			g_static_rec_mutex_lock (&summary->priv->summary_lock);
 			break;
 		case CFS_IO_LOCK:
-			g_mutex_lock (summary->priv->io_lock);
+			g_static_rec_mutex_lock (&summary->priv->io_lock);
 			break;
 		case CFS_FILTER_LOCK:
-			g_mutex_lock (summary->priv->filter_lock);
+			g_static_rec_mutex_lock (&summary->priv->filter_lock);
 			break;
 		case CFS_ALLOC_LOCK:
-			g_mutex_lock (summary->priv->alloc_lock);
+			g_static_rec_mutex_lock (&summary->priv->alloc_lock);
 			break;
 		case CFS_REF_LOCK:
-			g_mutex_lock (summary->priv->ref_lock);
+			g_static_rec_mutex_lock (&summary->priv->ref_lock);
 			break;
 		default:
 			g_return_if_reached ();
@@ -5319,19 +5325,19 @@ camel_folder_summary_unlock (CamelFolderSummary *summary,
 
 	switch (lock) {
 		case CFS_SUMMARY_LOCK:
-			g_mutex_unlock (summary->priv->summary_lock);
+			g_static_rec_mutex_unlock (&summary->priv->summary_lock);
 			break;
 		case CFS_IO_LOCK:
-			g_mutex_unlock (summary->priv->io_lock);
+			g_static_rec_mutex_unlock (&summary->priv->io_lock);
 			break;
 		case CFS_FILTER_LOCK:
-			g_mutex_unlock (summary->priv->filter_lock);
+			g_static_rec_mutex_unlock (&summary->priv->filter_lock);
 			break;
 		case CFS_ALLOC_LOCK:
-			g_mutex_unlock (summary->priv->alloc_lock);
+			g_static_rec_mutex_unlock (&summary->priv->alloc_lock);
 			break;
 		case CFS_REF_LOCK:
-			g_mutex_unlock (summary->priv->ref_lock);
+			g_static_rec_mutex_unlock (&summary->priv->ref_lock);
 			break;
 		default:
 			g_return_if_reached ();
diff --git a/camel/camel-folder-summary.h b/camel/camel-folder-summary.h
index 3441b3a..48bfe53 100644
--- a/camel/camel-folder-summary.h
+++ b/camel/camel-folder-summary.h
@@ -386,12 +386,8 @@ CamelMessageInfo * camel_folder_summary_peek_info (CamelFolderSummary *s, const
 
 /* Get only the uids of dirty/changed things to sync to server/db */
 GPtrArray * camel_folder_summary_get_changed (CamelFolderSummary *s);
-/* Gets the size of loaded mi's */
-gint camel_folder_summary_cache_size (CamelFolderSummary *s);
 /* reload the summary at any required point if required */
-gint camel_folder_summary_reload_from_db (CamelFolderSummary *s, CamelException *ex);
-/* ensures all CamelMessagesInfos loaded in memory */
-void camel_folder_summary_ensure_infos_loaded (CamelFolderSummary *s, gint at_least, CamelException *ex);
+void camel_folder_summary_prepare_fetch_all (CamelFolderSummary *s, CamelException *ex);
 /* insert mi to summary */
 void camel_folder_summary_insert (CamelFolderSummary *s, CamelMessageInfo *info, gboolean load);
 
diff --git a/camel/camel-folder-thread.c b/camel/camel-folder-thread.c
index 2e7e2aa..9b6dd80 100644
--- a/camel/camel-folder-thread.c
+++ b/camel/camel-folder-thread.c
@@ -607,10 +607,9 @@ camel_folder_thread_messages_new (CamelFolder *folder, GPtrArray *uids, gboolean
 	thread->node_chunks = e_memchunk_new(32, sizeof(CamelFolderThreadNode));
 	thread->folder = g_object_ref (folder);
 
+	camel_folder_summary_prepare_fetch_all (folder->summary, NULL);
 	fsummary = camel_folder_summary_array (folder->summary);
 	thread->summary = summary = g_ptr_array_new();
-	if (fsummary->len - camel_folder_summary_cache_size (folder->summary) > 50)
-		camel_folder_summary_reload_from_db (folder->summary, NULL);
 
 	/* prefer given order from the summary order */
 	if (!uids)
diff --git a/camel/camel-vee-folder.c b/camel/camel-vee-folder.c
index 533e4f5..937d34c 100644
--- a/camel/camel-vee-folder.c
+++ b/camel/camel-vee-folder.c
@@ -564,6 +564,8 @@ subfolder_renamed_update (CamelVeeFolder *vf, CamelFolder *sub, gchar hash[8])
 
 	camel_vee_folder_lock (vf, CVF_SUMMARY_LOCK);
 
+	camel_folder_summary_prepare_fetch_all (((CamelFolder *)vf)->summary, NULL);
+
 	count = camel_folder_summary_count (((CamelFolder *)vf)->summary);
 	for (i=0;i<count;i++) {
 		CamelVeeMessageInfo *mi = (CamelVeeMessageInfo *)camel_folder_summary_index (((CamelFolder *)vf)->summary, i);
@@ -966,6 +968,7 @@ vee_folder_getv (CamelObject *object,
 				CamelVeeMessageInfo *vinfo;
 
 				unread = deleted = visible = junked = junked_not_deleted = 0;
+				camel_folder_summary_prepare_fetch_all (folder->summary, ex);
 				count = camel_folder_summary_count (folder->summary);
 				for (j=0; j<count; j++) {
 					if ((info = (CamelMessageInfoBase *) camel_folder_summary_index (folder->summary, j))) {
@@ -1122,8 +1125,9 @@ vee_folder_sync (CamelFolder *folder,
 		/* Cleanup Junk/Trash uids */
 		GSList *del = NULL;
 		gint i, count;
-		count = folder->summary->uids->len;
 
+		camel_folder_summary_prepare_fetch_all (folder->summary, ex);
+		count = camel_folder_summary_count (folder->summary);
 		for (i=0; i < count; i++) {
 			CamelVeeMessageInfo *mi = (CamelVeeMessageInfo *)camel_folder_summary_index (folder->summary, i);
 			if (mi->old_flags & CAMEL_MESSAGE_DELETED) {
@@ -1538,6 +1542,7 @@ vee_folder_remove_folder_helper (CamelVeeFolder *vf, CamelFolder *source)
 		if (killun) {
 			start = -1;
 			last = -1;
+			camel_folder_summary_prepare_fetch_all (((CamelFolder *)folder_unmatched)->summary, NULL);
 			count = camel_folder_summary_count (((CamelFolder *)folder_unmatched)->summary);
 			for (i=0;i<count;i++) {
 				CamelVeeMessageInfo *mi = (CamelVeeMessageInfo *)camel_folder_summary_index (((CamelFolder *)folder_unmatched)->summary, i);
@@ -1566,6 +1571,7 @@ vee_folder_remove_folder_helper (CamelVeeFolder *vf, CamelFolder *source)
 	/*FIXME: This can be optimized a lot like, searching for UID in the summary uids */
 	start = -1;
 	last = -1;
+	camel_folder_summary_prepare_fetch_all (folder->summary, NULL);
 	count = camel_folder_summary_count (folder->summary);
 	for (i=0;i<count;i++) {
 		CamelVeeMessageInfo *mi = (CamelVeeMessageInfo *)camel_folder_summary_index (folder->summary, i);
@@ -1766,6 +1772,7 @@ vee_folder_rebuild_folder (CamelVeeFolder *vee_folder,
 	   current source) */
 	start = -1;
 	last = -1;
+	camel_folder_summary_prepare_fetch_all (folder->summary, ex);
 	count = camel_folder_summary_count (folder->summary);
 	for (i=0;i<count;i++) {
 		CamelVeeMessageInfo *mi = (CamelVeeMessageInfo *)camel_folder_summary_index (folder->summary, i);
diff --git a/camel/camel-vtrash-folder.c b/camel/camel-vtrash-folder.c
index b013308..0337603 100644
--- a/camel/camel-vtrash-folder.c
+++ b/camel/camel-vtrash-folder.c
@@ -111,6 +111,7 @@ vtrash_folder_getv (CamelObject *object,
 				CamelVeeMessageInfo *vinfo;
 
 				unread = deleted = visible = junked = junked_not_deleted = 0;
+				camel_folder_summary_prepare_fetch_all (folder->summary, ex);
 				count = camel_folder_summary_count(folder->summary);
 				for (j=0; j<count; j++) {
 					if ((info = (CamelMessageInfoBase *) camel_folder_summary_index(folder->summary, j))) {
diff --git a/camel/providers/groupwise/camel-groupwise-folder.c b/camel/providers/groupwise/camel-groupwise-folder.c
index 5d47efe..0b642d0 100644
--- a/camel/providers/groupwise/camel-groupwise-folder.c
+++ b/camel/providers/groupwise/camel-groupwise-folder.c
@@ -782,6 +782,7 @@ groupwise_sync (CamelFolder *folder, gboolean expunge, CamelMessageInfo *update_
 		return TRUE;
 
 	changes = camel_folder_change_info_new ();
+	camel_folder_summary_prepare_fetch_all (folder->summary, ex);
 	count = camel_folder_summary_count (folder->summary);
 	CAMEL_GROUPWISE_FOLDER_REC_LOCK (folder, cache_lock);
 	for (i=0; i < count; i++) {
@@ -2724,6 +2725,7 @@ groupwise_expunge (CamelFolder *folder, CamelException *ex)
 
 	container_id =  g_strdup (camel_groupwise_store_container_id_lookup (groupwise_store, folder->full_name));
 
+	camel_folder_summary_prepare_fetch_all (folder->summary, ex);
 	max = camel_folder_summary_count (folder->summary);
 	for (i = 0; i < max; i++) {
 		info = camel_folder_summary_index (folder->summary, i);
diff --git a/camel/providers/imap/camel-imap-folder.c b/camel/providers/imap/camel-imap-folder.c
index 3b4004c..fce9a39 100644
--- a/camel/providers/imap/camel-imap-folder.c
+++ b/camel/providers/imap/camel-imap-folder.c
@@ -986,8 +986,7 @@ imap_rescan (CamelFolder *folder, gint exists, CamelException *ex)
 	 */
 	removed = g_array_new (FALSE, FALSE, sizeof (gint));
 
-	if (summary_len - camel_folder_summary_cache_size (folder->summary) > 50)
-		camel_folder_summary_reload_from_db (folder->summary, ex);
+	camel_folder_summary_prepare_fetch_all (folder->summary, ex);
 
 	for (i = 0, j = 0; i < summary_len && new[j].uid; i++) {
 		gboolean changed = FALSE;
@@ -1450,6 +1449,7 @@ imap_sync (CamelFolder *folder,
 		/* Make sure we're connected before issuing commands */
 		if (!camel_imap_store_connected(store, ex)) {
 			g_free(set);
+			camel_message_info_free(info);
 			break;
 		}
 
diff --git a/camel/providers/local/camel-maildir-folder.c b/camel/providers/local/camel-maildir-folder.c
index ce6efe0..ee070d7 100644
--- a/camel/providers/local/camel-maildir-folder.c
+++ b/camel/providers/local/camel-maildir-folder.c
@@ -249,6 +249,7 @@ maildir_get_filename (CamelFolder *folder,
 	CamelLocalFolder *lf = (CamelLocalFolder *)folder;
 	CamelMaildirMessageInfo *mdi;
 	CamelMessageInfo *info;
+	gchar *res;
 
 	/* get the message summary info */
 	if ((info = camel_folder_summary_uid(folder->summary, uid)) == NULL) {
@@ -261,7 +262,11 @@ maildir_get_filename (CamelFolder *folder,
 	mdi = (CamelMaildirMessageInfo *)info;
 
 	/* what do we do if the message flags (and :info data) changes?  filename mismatch - need to recheck I guess */
-	return g_strdup_printf("%s/cur/%s", lf->folder_path, camel_maildir_info_filename(mdi));
+	res = g_strdup_printf("%s/cur/%s", lf->folder_path, camel_maildir_info_filename (mdi));
+
+	camel_message_info_free (info);
+
+	return res;
 }
 
 static CamelMimeMessage *
@@ -347,6 +352,9 @@ maildir_cmp_uids (CamelFolder *folder,
 	tma = camel_message_info_date_received (a);
 	tmb = camel_message_info_date_received (b);
 
+	camel_message_info_free (a);
+	camel_message_info_free (b);
+
 	return tma < tmb ? -1 : tma == tmb ? 0 : 1;
 }
 
@@ -362,8 +370,7 @@ maildir_sort_uids (CamelFolder *folder,
 
 		camel_exception_init (&ex);
 
-		camel_folder_summary_ensure_infos_loaded (
-			folder->summary, uids->len, &ex);
+		camel_folder_summary_prepare_fetch_all (folder->summary, &ex);
 
 		if (camel_exception_is_set (&ex))
 			g_warning ("%s: %s", G_STRFUNC, camel_exception_get_description (&ex));
diff --git a/camel/providers/local/camel-maildir-summary.c b/camel/providers/local/camel-maildir-summary.c
index de0b659..faefcac 100644
--- a/camel/providers/local/camel-maildir-summary.c
+++ b/camel/providers/local/camel-maildir-summary.c
@@ -560,11 +560,8 @@ maildir_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changes, Ca
 
 	/* keeps track of all uid's that have not been processed */
 	left = g_hash_table_new(g_str_hash, g_str_equal);
+	camel_folder_summary_prepare_fetch_all (s, ex);
 	count = camel_folder_summary_count (s);
-	if (count != camel_folder_summary_cache_size (s)) {
-		camel_folder_summary_reload_from_db (s, ex);
-		count = camel_folder_summary_count (s);
-	}
 	forceindex = count == 0;
 	for (i=0;i<count;i++) {
 		info = camel_folder_summary_index((CamelFolderSummary *)cls, i);
@@ -728,6 +725,7 @@ maildir_summary_sync (CamelLocalSummary *cls,
 
 	camel_operation_start(NULL, _("Storing folder"));
 
+	camel_folder_summary_prepare_fetch_all ((CamelFolderSummary *)cls, ex);
 	count = camel_folder_summary_count((CamelFolderSummary *)cls);
 	for (i=count-1;i>=0;i--) {
 		camel_operation_progress(NULL, (count-i)*100/count);
diff --git a/camel/providers/local/camel-mbox-folder.c b/camel/providers/local/camel-mbox-folder.c
index dfa299b..6c9096e 100644
--- a/camel/providers/local/camel-mbox-folder.c
+++ b/camel/providers/local/camel-mbox-folder.c
@@ -467,6 +467,7 @@ static gint
 mbox_cmp_uids (CamelFolder *folder, const gchar *uid1, const gchar *uid2)
 {
 	CamelMboxMessageInfo *a, *b;
+	gint res;
 
 	g_return_val_if_fail (folder != NULL, 0);
 	g_return_val_if_fail (folder->summary != NULL, 0);
@@ -477,7 +478,12 @@ mbox_cmp_uids (CamelFolder *folder, const gchar *uid1, const gchar *uid2)
 	g_return_val_if_fail (a != NULL, 0);
 	g_return_val_if_fail (b != NULL, 0);
 
-	return a->frompos < b->frompos ? -1 : a->frompos == b->frompos ? 0 : 1;
+	res = a->frompos < b->frompos ? -1 : a->frompos == b->frompos ? 0 : 1;
+
+	camel_message_info_free ((CamelMessageInfo *)a);
+	camel_message_info_free ((CamelMessageInfo *)b);
+
+	return res;
 }
 
 static void
@@ -491,8 +497,7 @@ mbox_sort_uids (CamelFolder *folder, GPtrArray *uids)
 
 		camel_exception_init (&ex);
 
-		camel_folder_summary_ensure_infos_loaded (
-			folder->summary, uids->len, &ex);
+		camel_folder_summary_prepare_fetch_all (folder->summary, &ex);
 
 		if (camel_exception_is_set (&ex))
 			g_warning ("%s: %s", G_STRFUNC, camel_exception_get_description (&ex));
diff --git a/camel/providers/local/camel-mbox-summary.c b/camel/providers/local/camel-mbox-summary.c
index 9776241..72ebb32 100644
--- a/camel/providers/local/camel-mbox-summary.c
+++ b/camel/providers/local/camel-mbox-summary.c
@@ -522,9 +522,8 @@ summary_update(CamelLocalSummary *cls, off_t offset, CamelFolderChangeInfo *chan
 	/* we mark messages as to whether we've seen them or not.
 	   If we're not starting from the start, we must be starting
 	   from the old end, so everything must be treated as new */
+	camel_folder_summary_prepare_fetch_all (s, ex);
 	count = camel_folder_summary_count(s);
-	if (count != camel_folder_summary_cache_size(s)) /* It makes sense to load summary, if it isn't there. */
-		camel_folder_summary_reload_from_db (s, ex);
 	for (i=0;i<count;i++) {
 		mi = (CamelMboxMessageInfo *)camel_folder_summary_index(s, i);
 		if (offset == 0)
@@ -635,6 +634,7 @@ mbox_summary_check(CamelLocalSummary *cls, CamelFolderChangeInfo *changes, Camel
 	if (st.st_size == 0) {
 		/* empty?  No need to scan at all */
 		d(printf("Empty mbox, clearing summary\n"));
+		camel_folder_summary_prepare_fetch_all (s, ex);
 		count= camel_folder_summary_count(s);
 		for (i=0;i<count;i++) {
 			CamelMessageInfo *info = camel_folder_summary_index(s, i);
@@ -1070,6 +1070,7 @@ camel_mbox_summary_sync_mbox(CamelMboxSummary *cls, guint32 flags, CamelFolderCh
 	camel_mime_parser_scan_pre_from(mp, TRUE);
 	camel_mime_parser_init_with_fd(mp, fd);
 
+	camel_folder_summary_prepare_fetch_all (s, ex);
 	count = camel_folder_summary_count(s);
 	for (i = 0; i < count; i++) {
 		gint pc = (i + 1) * 100 / count;
diff --git a/camel/providers/local/camel-mh-summary.c b/camel/providers/local/camel-mh-summary.c
index 9a48078..f118007 100644
--- a/camel/providers/local/camel-mh-summary.c
+++ b/camel/providers/local/camel-mh-summary.c
@@ -228,6 +228,7 @@ mh_summary_check (CamelLocalSummary *cls,
 
 	/* keeps track of all uid's that have not been processed */
 	left = g_hash_table_new(g_str_hash, g_str_equal);
+	camel_folder_summary_prepare_fetch_all ((CamelFolderSummary *)cls, ex);
 	count = camel_folder_summary_count((CamelFolderSummary *)cls);
 	forceindex = count == 0;
 	for (i=0;i<count;i++) {
@@ -299,6 +300,7 @@ mh_summary_sync (CamelLocalSummary *cls,
 
 	/* FIXME: need to update/honour .mh_sequences or whatever it is */
 
+	camel_folder_summary_prepare_fetch_all ((CamelFolderSummary *)cls, ex);
 	count = camel_folder_summary_count((CamelFolderSummary *)cls);
 	for (i=count-1;i>=0;i--) {
 		info = (CamelLocalMessageInfo *)camel_folder_summary_index((CamelFolderSummary *)cls, i);
diff --git a/camel/providers/local/camel-spool-summary.c b/camel/providers/local/camel-spool-summary.c
index 258cda8..098f76c 100644
--- a/camel/providers/local/camel-spool-summary.c
+++ b/camel/providers/local/camel-spool-summary.c
@@ -302,6 +302,7 @@ spool_summary_check (CamelLocalSummary *cls,
 
 	/* check to see if we need to copy/update the file; missing xev headers prompt this */
 	work = FALSE;
+	camel_folder_summary_prepare_fetch_all (s, ex);
 	count = camel_folder_summary_count(s);
 	for (i=0;!work && i<count; i++) {
 		CamelMboxMessageInfo *info = (CamelMboxMessageInfo *)camel_folder_summary_index(s, i);
diff --git a/camel/providers/nntp/camel-nntp-folder.c b/camel/providers/nntp/camel-nntp-folder.c
index 7cb9493..8965048 100644
--- a/camel/providers/nntp/camel-nntp-folder.c
+++ b/camel/providers/nntp/camel-nntp-folder.c
@@ -113,27 +113,39 @@ nntp_folder_refresh_info_online (CamelFolder *folder,
 	return success;
 }
 
-static gboolean
-nntp_folder_sync_online (CamelFolder *folder, CamelException *ex)
+static void
+unset_flagged_flag (const gchar *uid, CamelFolderSummary *summary)
 {
-	gboolean success;
-
-	camel_service_lock (CAMEL_SERVICE (folder->parent_store), CS_REC_CONNECT_LOCK);
+	CamelMessageInfo *info;
 
-	success = camel_folder_summary_save_to_db (folder->summary, ex);
+	info = camel_folder_summary_uid (summary, uid);
+	if (info) {
+		CamelMessageInfoBase *base = (CamelMessageInfoBase *) info;
 
-	camel_service_unlock (CAMEL_SERVICE (folder->parent_store), CS_REC_CONNECT_LOCK);
+		if ((base->flags & CAMEL_MESSAGE_FOLDER_FLAGGED) != 0) {
+			base->flags &= ~CAMEL_MESSAGE_FOLDER_FLAGGED;
+			base->dirty = TRUE;
+		}
 
-	return success;
+		camel_message_info_free (info);
+	}
 }
 
 static gboolean
-nntp_folder_sync_offline (CamelFolder *folder, CamelException *ex)
+nntp_folder_sync (CamelFolder *folder, CamelException *ex)
 {
+	GPtrArray *changed;
 	gboolean success;
 
 	camel_service_lock (CAMEL_SERVICE (folder->parent_store), CS_REC_CONNECT_LOCK);
 
+	changed = camel_folder_summary_get_changed (folder->summary);
+	if (changed) {
+		g_ptr_array_foreach (changed, (GFunc) unset_flagged_flag, folder->summary);
+		g_ptr_array_foreach (changed, (GFunc) camel_pstring_free, NULL);
+		g_ptr_array_free (changed, TRUE);
+		camel_folder_summary_touch (folder->summary);
+	}
 	success = camel_folder_summary_save_to_db (folder->summary, ex);
 
 	camel_service_unlock (CAMEL_SERVICE (folder->parent_store), CS_REC_CONNECT_LOCK);
@@ -141,6 +153,18 @@ nntp_folder_sync_offline (CamelFolder *folder, CamelException *ex)
 	return success;
 }
 
+static gboolean
+nntp_folder_sync_online (CamelFolder *folder, CamelException *ex)
+{
+	return nntp_folder_sync (folder, ex);
+}
+
+static gboolean
+nntp_folder_sync_offline (CamelFolder *folder, CamelException *ex)
+{
+	return nntp_folder_sync (folder, ex);
+}
+
 static gchar *
 nntp_get_filename (CamelFolder *folder, const gchar *uid, CamelException *ex)
 {