GNOME.org
 

Re: [evolution-patches] Initial download of a folder

On Mon, 2006-10-23 at 11:09 +0200, Philip Van Hoof wrote:

> Sidenote for, for example, Brutus and other 3th party developers who
> would like to get support for their service in tinymail (in case of
> Brutus, I also want support for Exchange, so chances are high that I
> will integrate it at some point anyway).
> 
> Each nth (in my experiment that's each 1000th) header, the method that

Because I noticed that 1:1001 is slower than 1:1 on some IMAP servers,
this version of the patch determines the next n using the serie 1, 2, 4,
8, ... with a limit of 1000 (then it stays 1000).

Maybe I will use a 1, 3, 7, 15, 31 or an equivalent as a test. I don't
think it's going to make a drastic difference in speed.

For small folders, it seems to do make a difference in speed (1:1 in
stead of 1:1001).

Of course I'm on a *! *#*&# Courier IMAPd, but that one should also work
well of course.


-- 
Philip Van Hoof, software developer
home: me at pvanhoof dot be
gnome: pvanhoof at gnome dot org
work: vanhoof at x-tend dot be
blog: http://pvanhoof.be/blog

Index: camel/providers/imap/camel-imap-folder.c
===================================================================
--- camel/providers/imap/camel-imap-folder.c	(revision 1046)
+++ camel/providers/imap/camel-imap-folder.c	(working copy)
@@ -2348,7 +2348,9 @@
 	CamelStream *stream;
 	char *uid, *resp;
 	GData *data;
-	
+	gboolean more = TRUE;
+	unsigned int nextn = 1;
+
 	if (store->server_level >= IMAP_LEVEL_IMAP4REV1)
 		header_spec = "HEADER.FIELDS (" CAMEL_MESSAGE_INFO_HEADERS MAILING_LIST_HEADERS ")";
 	else
@@ -2360,7 +2362,12 @@
 	
 	if( g_getenv ("EVO_IMAP_FETCH_ALL_HEADERS") )
 		header_spec = "HEADER";
-	
+
+while (more)
+{
+	gint count = 0;
+	more = FALSE;
+
 	/* Figure out if any of the new messages are already cached (which
 	 * may be the case if we're re-syncing after disconnected operation).
 	 * If so, get their UIDs, FLAGS, and SIZEs. If not, get all that
@@ -2374,25 +2381,38 @@
 		camel_message_info_free(&mi->info);
 	} else
 		uidval = 0;
-	
+
 	size = (exists - seq) * (IMAP_PRETEND_SIZEOF_FLAGS + IMAP_PRETEND_SIZEOF_SIZE + IMAP_PRETEND_SIZEOF_HEADERS);
 	got = 0;
 	if (!camel_imap_command_start (store, folder, ex,
-				       "UID FETCH %d:* (FLAGS RFC822.SIZE INTERNALDATE BODY.PEEK[%s])",
-				       uidval + 1, header_spec))
+				       "UID FETCH %d:%d (FLAGS RFC822.SIZE INTERNALDATE BODY.PEEK[%s])",
+				       uidval + 1, uidval + 1 + nextn, header_spec))
 		return;
+
+	/* 1, 2, 4, 8, .. 1000 etc etc */
+
+	if (nextn < 1000)
+		nextn += nextn;
+	else
+		nextn = 1000;
+
 	camel_operation_start (NULL, _("Fetching summary information for new messages in %s"), folder->name);
 	
 	/* Parse the responses. We can't add a message to the summary
 	 * until we've gotten its headers, and there's no guarantee
 	 * the server will send the responses in a useful order...
 	 */
+
 	fetch_data = g_ptr_array_new ();
 	messages = g_ptr_array_new ();
+
 	while ((type = camel_imap_command_response (store, &resp, ex)) ==
-	       CAMEL_IMAP_RESPONSE_UNTAGGED) {
+	       CAMEL_IMAP_RESPONSE_UNTAGGED) 
+	{
+		more = TRUE;
 		data = parse_fetch_response (imap_folder, resp);
 		g_free (resp);
+
 		if (!data)
 			continue;
 		
@@ -2401,7 +2421,7 @@
 			g_datalist_clear (&data);
 			continue;
 		}
-		
+
 		if (g_datalist_get_data (&data, "FLAGS"))
 			got += IMAP_PRETEND_SIZEOF_FLAGS;
 		if (g_datalist_get_data (&data, "RFC822.SIZE"))
@@ -2416,43 +2436,46 @@
 			add_message_from_data (folder, messages, first, data);
 			g_datalist_set_data (&data, "BODY_PART_STREAM", NULL);
 		}
-		
-		camel_operation_progress (NULL, got * 100 / size);
+
+		if (size > 0)
+			camel_operation_progress (NULL, got * 100 / size);
 		g_ptr_array_add (fetch_data, data);
 	}
 	camel_operation_end (NULL);
-	
+
 	if (type == CAMEL_IMAP_RESPONSE_ERROR)
 		goto lose;
-	
+
 	/* Free the final tagged response */
 	g_free (resp);
-	
+
 	/* Figure out which headers we still need to fetch. */
 	needheaders = g_ptr_array_new ();
 	size = got = 0;
-	for (i = 0; i < fetch_data->len; i++) {
+
+	for (i = 0; i < fetch_data->len; i++) 
+	{
 		data = fetch_data->pdata[i];
 		if (g_datalist_get_data (&data, "BODY_PART_LEN"))
 			continue;
-		
+
 		uid = g_datalist_get_data (&data, "UID");
 		if (uid) {
 			g_ptr_array_add (needheaders, uid);
 			size += IMAP_PRETEND_SIZEOF_HEADERS;
 		}
 	}
-	
+
 	/* And fetch them */
-	if (needheaders->len) {
+	if (needheaders->len) 
+	{
 		char *uidset;
 		int uid = 0;
-		
+
 		qsort (needheaders->pdata, needheaders->len,
 		       sizeof (void *), uid_compar);
-		
+
 		camel_operation_start (NULL, _("Fetching summary information for new messages in %s"), folder->name);
-		
 		while (uid < needheaders->len) {
 			uidset = imap_uid_array_to_set (folder->summary, needheaders, uid, UID_SET_LIMIT, &uid);
 			if (!camel_imap_command_start (store, folder, ex,
@@ -2464,7 +2487,7 @@
 				goto lose;
 			}
 			g_free (uidset);
-			
+
 			while ((type = camel_imap_command_response (store, &resp, ex))
 			       == CAMEL_IMAP_RESPONSE_UNTAGGED) {
 				data = parse_fetch_response (imap_folder, resp);
@@ -2476,7 +2499,8 @@
 				if (stream) {
 					add_message_from_data (folder, messages, first, data);
 					got += IMAP_PRETEND_SIZEOF_HEADERS;
-					camel_operation_progress (NULL, got * 100 / size);
+					if (size > 0)
+						camel_operation_progress (NULL, got * 100 / size);
 				}
 				g_datalist_clear (&data);
 			}
@@ -2491,7 +2515,7 @@
 		g_ptr_array_free (needheaders, TRUE);
 		camel_operation_end (NULL);
 	}
-	
+
 	/* Now finish up summary entries (fix UIDs, set flags and size) */
 	for (i = 0; i < fetch_data->len; i++) {
 		data = fetch_data->pdata[i];
@@ -2501,7 +2525,7 @@
 			g_datalist_clear (&data);
 			continue;
 		}
-		
+
 		mi = messages->pdata[seq - first];
 		if (mi == NULL) {
 			CamelMessageInfo *pmi = NULL;
@@ -2535,9 +2559,10 @@
 		
 		uid = g_datalist_get_data (&data, "UID");
 
-		/* This strdup() might not get freed since the mmap() patch! */
-		if (uid)
+		if (uid) {
 			mi->info.uid = g_strdup (uid);
+			mi->info.uid_needs_free = TRUE;
+		}
 
 		flags = GPOINTER_TO_INT (g_datalist_get_data (&data, "FLAGS"));
 		if (flags) {
@@ -2548,26 +2573,29 @@
 			mi->info.flags |= flags;
 			flags_to_label(folder, mi);
 		}
-		
+
 #ifdef NON_TINYMAIL_FEATURES
 		size = GPOINTER_TO_INT (g_datalist_get_data (&data, "RFC822.SIZE"));
 		if (size)
 			mi->info.size = size;
 #endif
+
 		g_datalist_clear (&data);
 	}
 	g_ptr_array_free (fetch_data, TRUE);
-	
+
 	/* And add the entries to the summary, etc. */
 	for (i = 0; i < messages->len; i++) {
 		mi = messages->pdata[i];
+
 		if (!mi) {
-			g_warning ("No information for message %d", i + first);
+			/* g_warning ("No information for message %d", i + first);
 			camel_exception_setv (ex, CAMEL_EXCEPTION_SYSTEM,
 					      _("Incomplete server response: no information provided for message %d"),
-					      i + first);
-			break;
+					      i + first); */
+			continue;
 		}
+
 		uid = (char *)camel_message_info_uid(mi);
 		if (uid[0] == 0) {
 			g_warning("Server provided no uid: message %d", i + first);
@@ -2599,15 +2627,17 @@
 			camel_folder_change_info_recent_uid(changes, camel_message_info_uid (mi));
 	}
 
+	camel_folder_summary_dump_mmap (folder->summary);
+
 	for ( ; i < messages->len; i++) {
 		if ((mi = messages->pdata[i]))
 			camel_message_info_free(&mi->info);
 	}
-	
 	g_ptr_array_free (messages, TRUE);
-	
-	return;
-	
+
+
+	goto endbmore;
+
  lose:
 	if (fetch_data) {
 		for (i = 0; i < fetch_data->len; i++) {
@@ -2623,6 +2653,12 @@
 		}
 		g_ptr_array_free (messages, TRUE);
 	}
+
+ endbmore:
+ count = 0;
+
+ } /* more */
+
 }
 
 /* Called with the store's connect_lock locked */
Index: camel/providers/Makefile.am
===================================================================
--- camel/providers/Makefile.am	(revision 1046)
+++ camel/providers/Makefile.am	(working copy)
@@ -1,18 +1,20 @@
 ## Process this file with automake to produce Makefile.in
 
+
+SUBDIRS = pop3 smtp imap local  
+
 if ENABLE_NNTP
-NNTP_DIR=nntp
+SUBDIRS += nntp 
 endif
 
 if ENABLE_IMAPP
-IMAPP_DIR=imapp
+SUBDIRS += imapp 
 endif
 
+
 if OS_WIN32
 else
-SENDMAIL_DIR=sendmail
+SUBDIRS += sendmail 
 endif
 
-SUBDIRS = pop3 $(SENDMAIL_DIR) smtp imap $(NNTP_DIR) local $(IMAPP_DIR) 
 
-
Index: camel/camel-folder.c
===================================================================
--- camel/camel-folder.c	(revision 1046)
+++ camel/camel-folder.c	(working copy)
@@ -2116,7 +2116,7 @@
 	struct _CamelFolderChangeInfoPrivate *p;
 	GPtrArray *olduids;
 	char *olduid;
-	
+
 	g_assert(info != NULL);
 	
 	p = info->priv;
Index: camel/camel-folder-summary.c
===================================================================
--- camel/camel-folder-summary.c	(revision 1046)
+++ camel/camel-folder-summary.c	(working copy)
@@ -113,6 +113,10 @@
 static void camel_folder_summary_init       (CamelFolderSummary *obj);
 static void camel_folder_summary_finalize   (CamelObject *obj);
 
+
+static void camel_folder_summary_mmap_add(CamelFolderSummary *s, CamelMessageInfo *info);
+static void camel_folder_summary_unload_mmap (CamelFolderSummary *s);
+
 static CamelObjectClass *camel_folder_summary_parent;
 
 static void
@@ -161,6 +165,26 @@
 	g_slice_free1 ((gint)user_data, data);
 }
 
+static gboolean always_true (gpointer key, gpointer value, gpointer gp)
+{
+	return TRUE;
+}
+
+static void 
+camel_folder_summary_unload_mmap (CamelFolderSummary *s)
+{
+	camel_folder_summary_clear(s);
+	if (s->file)
+		g_mapped_file_free (s->file);
+	s->file = NULL;
+
+	g_ptr_array_foreach (s->messages, foreach_msginfo, (gpointer)s->message_info_size);
+	if (s->messages->len > 0)
+		g_ptr_array_remove_range (s->messages, 0, s->messages->len-1);
+	g_hash_table_foreach_remove (s->messages_uid, always_true, NULL);
+
+}
+
 static void
 camel_folder_summary_finalize (CamelObject *obj)
 {
@@ -177,6 +201,7 @@
 	g_ptr_array_foreach (s->messages, foreach_msginfo, (gpointer)s->message_info_size);
 	g_ptr_array_free(s->messages, TRUE);
 	g_hash_table_destroy(s->messages_uid);
+	s->messages_uid = NULL;
 
 	g_hash_table_foreach(p->filter_charset, free_o_name, 0);
 	g_hash_table_destroy(p->filter_charset);
@@ -591,7 +616,7 @@
 			}
 		}
 
-		camel_folder_summary_add(s, mi);
+		camel_folder_summary_mmap_add(s, mi);
 	}
 
 
@@ -807,6 +832,17 @@
 }
 
 
+
+void 
+camel_folder_summary_dump_mmap (CamelFolderSummary *s)
+{
+	camel_folder_summary_save (s);
+	camel_folder_summary_unload_mmap (s);
+	camel_folder_summary_load(s);
+
+	return;
+}
+
 /**
  * camel_folder_summary_add:
  * @summary: a #CamelFolderSummary object
@@ -845,7 +881,25 @@
 	CAMEL_SUMMARY_UNLOCK(s, summary_lock);
 }
 
+static void
+camel_folder_summary_mmap_add(CamelFolderSummary *s, CamelMessageInfo *info)
+{
+	CAMEL_SUMMARY_LOCK(s, summary_lock);
 
+/* unnecessary for pooled vectors */
+#ifdef DOESTRV
+	/* this is vitally important, and also if this is ever modified, then
+	   the hash table needs to be resynced */
+	info->strings = e_strv_pack(info->strings);
+#endif
+
+	g_ptr_array_add(s->messages, info);
+	g_hash_table_insert(s->messages_uid, (char *)camel_message_info_uid(info), info);
+	s->flags |= CAMEL_SUMMARY_DIRTY;
+
+	CAMEL_SUMMARY_UNLOCK(s, summary_lock);
+}
+
 /**
  * camel_folder_summary_add_from_header:
  * @summary: a #CamelFolderSummary object
@@ -1090,9 +1144,9 @@
 void
 camel_folder_summary_touch(CamelFolderSummary *s)
 {
-	CAMEL_SUMMARY_LOCK(s, summary_lock);
+	//CAMEL_SUMMARY_LOCK(s, summary_lock);
 	s->flags |= CAMEL_SUMMARY_DIRTY;
-	CAMEL_SUMMARY_UNLOCK(s, summary_lock);
+	//CAMEL_SUMMARY_UNLOCK(s, summary_lock);
 }
 
 
Index: camel/camel-folder-summary.h
===================================================================
--- camel/camel-folder-summary.h	(revision 1046)
+++ camel/camel-folder-summary.h	(working copy)
@@ -308,6 +308,8 @@
 CamelType			 camel_folder_summary_get_type	(void);
 CamelFolderSummary      *camel_folder_summary_new	(struct _CamelFolder *folder);
 
+void camel_folder_summary_dump_mmap (CamelFolderSummary *s);
+
 unsigned char*
 decode_uint32 (unsigned char *start, guint32 *dest, gboolean is_string);
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]