Re: [Evolution-hackers] camel-folder-summary.c with mmap

[Philip Van Hoof <spam pvanhoof be>]

On Sun, 2006-07-09 at 18:08 +0200, Philip Van Hoof wrote:
> On Sun, 2006-07-09 at 14:59 +0200, Philip Van Hoof wrote:
> 
> > Note about this E-mail. I'm assuming people who will read this, have a
> > technical in-depth knowledge of Camel and camel-folder-summary.c

o. This small addition (fixes03) replaces the refresh_info_online of the
CamelDiscoFolderClass with refresh_info of the CamelFolderClass. (But in
general I agree that part of the patch (saving in a load method) is ugly
and needs refactoring and/or rethinking.

> o. This new version of the patch cleans up all compilation warnings.
> 
> o. It detects when the summary file doesn't exist (and will make camel
> fallback into scanning the folder and creating a new one if already
> possible)
> 
> o. It removes some bugs and little issues.
> 
> o. It implements support for the mmap() summary in the MBOX and LOCAL
> provider. Corrects the support for the mmap() summary in the IMAP
> provider (I made a small mistake that I've corrected).
> 
> o. Corrects a tiny locking issue
> 
> o. Is the first patch that actually made it possible to launch
> Evolution. My Evolution crashed (after a few minutes, while it was
> parsing the mmap()ed summary info of my Bugzilla folder on two bytes
> that come right after the content-info of a specific E-mail in that
> summary file). This definitely means that there's somewhere a very small
> glitch that needs to be corrected.
> 
> o. Removes all the memory leaks which I blogged about and illustrated in
> those first graphs.
> 
> o. Removes the strdup() that is being done on the "tokens" table (you
> guys should definitely try to remove it in the non-mmap() implementation
> too. It's needlessly duplicating > ~600 kb on a 14,558 messages folder).
> 
> o. Removes the printf()'s and other debugging information
> 
> o. Fixes reading some integers like the dates and other timestamp
> information.
> 
> Attached are the patch (fixes02) and two massif valgrind reports. One
> shows loading and unloading a folder with ~ 650 headers. The other shows
> loading and unloading a folder with ~ 14,550 headers. Both using the
> mmap(). Note that without the mmap() patch, the memory for the summary 
> file of the 14,550 one would consume something like ~13MB or even ~15MB.

-- 
Philip Van Hoof, software developer at x-tend 
home: me at pvanhoof dot be 
gnome: pvanhoof at gnome dot org 
work: vanhoof at x-tend dot be 
http://www.pvanhoof.be - http://www.x-tend.be

? camel-mime-tables.c
Index: camel-file-utils.c
===================================================================
RCS file: /cvs/gnome/evolution-data-server/camel/camel-file-utils.c,v
retrieving revision 1.17
diff -u -r1.17 camel-file-utils.c
--- camel-file-utils.c	2 Jun 2006 00:52:29 -0000	1.17
+++ camel-file-utils.c	10 Jul 2006 07:54:41 -0000
@@ -269,11 +269,21 @@
 	
 	if ((len = strlen (str)) > 65536)
 		len = 65536;
+
+	if (len==0) len=-1;
 	
 	if (camel_file_util_encode_uint32 (out, len+1) == -1)
 		return -1;
-	if (len == 0 || fwrite (str, len, 1, out) == 1)
-		return 0;
+
+	if (len != 0)
+	{
+		if (fwrite (str, len, 1, out) == 1)
+		{
+			fputc ('\0', out);
+			return 0;
+		}
+	}
+
 	return -1;
 }
 
Index: camel-folder-summary.c
===================================================================
RCS file: /cvs/gnome/evolution-data-server/camel/camel-folder-summary.c,v
retrieving revision 1.149
diff -u -r1.149 camel-folder-summary.c
--- camel-folder-summary.c	6 Jul 2006 19:43:46 -0000	1.149
+++ camel-folder-summary.c	10 Jul 2006 07:54:42 -0000
@@ -50,6 +50,7 @@
 #include "camel-stream-null.h"
 #include "camel-stream-filter.h"
 #include "camel-string-utils.h"
+#include "camel-disco-folder.h"
 
 #include "libedataserver/md5-utils.h"
 #include "libedataserver/e-memory.h"
@@ -66,15 +67,15 @@
 /* this should probably be conditional on it existing */
 #define USE_BSEARCH
 
-#define d(x)
+#define d(x)	
 #define io(x)			/* io debug */
-#define w(x)
+#define w(x)	
 
 #if 0
 extern int strdup_count, malloc_count, free_count;
 #endif
 
-#define CAMEL_FOLDER_SUMMARY_VERSION (13)
+#define CAMEL_FOLDER_SUMMARY_VERSION (14)
 
 #define _PRIVATE(o) (((CamelFolderSummary *)(o))->priv)
 
@@ -83,16 +84,17 @@
 	struct _node *next;
 };
 
+
 static struct _node *my_list_append(struct _node **list, struct _node *n);
 static int my_list_size(struct _node **list);
 
-static int summary_header_load(CamelFolderSummary *, FILE *);
-static int summary_header_save(CamelFolderSummary *, FILE *);
+static int summary_header_load(CamelFolderSummary *, FILE *in);
+static int summary_header_save(CamelFolderSummary *, FILE *out);
 
 static CamelMessageInfo * message_info_new_from_header(CamelFolderSummary *, struct _camel_header_raw *);
 static CamelMessageInfo * message_info_new_from_parser(CamelFolderSummary *, CamelMimeParser *);
 static CamelMessageInfo * message_info_new_from_message(CamelFolderSummary *s, CamelMimeMessage *msg);
-static CamelMessageInfo * message_info_load(CamelFolderSummary *, FILE *);
+static CamelMessageInfo * message_info_load(CamelFolderSummary *, FILE *in);
 static int		  message_info_save(CamelFolderSummary *, FILE *, CamelMessageInfo *);
 static void		  message_info_free(CamelFolderSummary *, CamelMessageInfo *);
 
@@ -160,6 +162,11 @@
 	struct _CamelFolderSummaryPrivate *p;
 	CamelFolderSummary *s = (CamelFolderSummary *)obj;
 
+
+	if (s->file)
+		g_mapped_file_free (s->file);
+	s->file = NULL;
+
 	p = _PRIVATE(obj);
 
 	camel_folder_summary_clear(s);
@@ -236,7 +243,7 @@
 	CamelFolderSummary *new = CAMEL_FOLDER_SUMMARY ( camel_object_new (camel_folder_summary_get_type ()));
 
 	new->folder = folder;
-
+	
 	return new;
 }
 
@@ -504,17 +511,24 @@
 	int i;
 	guint32 count;
 	CamelMessageContentInfo *ci, *part;
+	unsigned char *ptrchr = s->filepos;
 
 	ci = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->content_info_load(s, in);
+
 	if (ci == NULL)
 		return NULL;
 
-	if (camel_file_util_decode_uint32(in, &count) == -1 || count > 500) {
+	ptrchr = s->filepos;
+	ptrchr = decode_uint32 ((unsigned char*)ptrchr, &count, FALSE);
+	s->filepos = ptrchr;
+
+	if (count > 500) {
 		camel_folder_summary_content_info_free(s, ci);
 		return NULL;
 	}
 
 	for (i=0;i<count;i++) {
+
 		part = perform_content_info_load(s, in);
 		if (part) {
 			my_list_append((struct _node **)&ci->childs, (struct _node *)part);
@@ -540,31 +554,34 @@
 int
 camel_folder_summary_load(CamelFolderSummary *s)
 {
-	FILE *in;
 	int i;
 	CamelMessageInfo *mi;
 
-	if (s->summary_path == NULL)
-		return 0;
 
-	in = g_fopen(s->summary_path, "rb");
-	if (in == NULL)
-		return -1;
+	if (s->summary_path == NULL || !g_file_test (s->summary_path, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_REGULAR))
+		return 0;
 
 	CAMEL_SUMMARY_LOCK(s, io_lock);
-	if ( ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_load(s, in) == -1)
+
+	if (!s->file)
+		s->file = g_mapped_file_new (s->summary_path, FALSE, NULL);
+
+	s->filepos = g_mapped_file_get_contents (s->file);
+
+	if ( ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_load(s, NULL) == -1)
 		goto error;
 
 	/* now read in each message ... */
 	for (i=0;i<s->saved_count;i++) {
-		mi = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->message_info_load(s, in);
+		mi = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->message_info_load(s, NULL);
 
 		if (mi == NULL)
 			goto error;
 
 		/* FIXME: this should be done differently, how i don't know */
+
 		if (s->build_content) {
-			((CamelMessageInfoBase *)mi)->content = perform_content_info_load(s, in);
+			((CamelMessageInfoBase *)mi)->content = perform_content_info_load(s, NULL);
 			if (((CamelMessageInfoBase *)mi)->content == NULL) {
 				camel_message_info_free(mi);
 				goto error;
@@ -574,13 +591,11 @@
 		camel_folder_summary_add(s, mi);
 	}
 
-	CAMEL_SUMMARY_UNLOCK(s, io_lock);
-	
-	if (fclose (in) != 0)
-		return -1;
 
 	s->flags &= ~CAMEL_SUMMARY_DIRTY;
 
+	CAMEL_SUMMARY_UNLOCK(s, io_lock);
+
 	return 0;
 
 error:
@@ -588,7 +603,6 @@
 		g_warning ("Cannot load summary file: `%s': %s", s->summary_path, g_strerror (errno));
 	
 	CAMEL_SUMMARY_UNLOCK(s, io_lock);
-	fclose (in);
 	s->flags |= ~CAMEL_SUMMARY_DIRTY;
 
 	return -1;
@@ -724,21 +738,20 @@
 int
 camel_folder_summary_header_load(CamelFolderSummary *s)
 {
-	FILE *in;
 	int ret;
 
-	if (s->summary_path == NULL)
+	if (s->summary_path == NULL || !g_file_test (s->summary_path, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_REGULAR))
 		return 0;
 
-	in = g_fopen(s->summary_path, "rb");
-	if (in == NULL)
-		return -1;
+	if (!s->file)
+		s->file = g_mapped_file_new (s->summary_path, FALSE, NULL);
+
+	s->filepos = g_mapped_file_get_contents (s->file);
 
 	CAMEL_SUMMARY_LOCK(s, io_lock);
-	ret = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_load(s, in);
+	ret = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_load(s, NULL);
 	CAMEL_SUMMARY_UNLOCK(s, io_lock);
 	
-	fclose(in);
 	s->flags &= ~CAMEL_SUMMARY_DIRTY;
 	return ret;
 }
@@ -762,7 +775,7 @@
 		if (mi == info)
 			return 0;
 		d(printf ("Trying to insert message with clashing uid (%s).  new uid re-assigned", camel_message_info_uid(info)));
-		g_free(info->uid);
+		// g_free(info->uid);
 		uid = info->uid = camel_folder_summary_next_uid_string(s);
 		camel_message_info_set_flags(info, CAMEL_MESSAGE_FOLDER_FLAGGED, CAMEL_MESSAGE_FOLDER_FLAGGED);
 		CAMEL_SUMMARY_LOCK(s, summary_lock);
@@ -1302,12 +1315,14 @@
 		if (token != -1) {
 			return camel_file_util_encode_uint32(out, token+1);
 		} else {
-			if (camel_file_util_encode_uint32(out, len+32) == -1)
+			if (camel_file_util_encode_uint32(out, len+32 + 1) == -1)
 				return -1;
 			if (fwrite(str, len, 1, out) != 1)
 				return -1;
+			else fputc ('\0', out);
 		}
 	}
+
 	return 0;
 }
 
@@ -1322,24 +1337,23 @@
  * Returns %0 on success or %-1 on fail
  **/
 int
-camel_folder_summary_decode_token(FILE *in, char **str)
+camel_folder_summary_decode_token(CamelFolderSummary *s, FILE *in, char **str)
 {
 	char *ret;
 	guint32 len;
+	unsigned char *ptrchr = s->filepos;
 
 	io(printf("Decode token ...\n"));
-	
-	if (camel_file_util_decode_uint32(in, &len) == -1) {
-		io(printf ("Could not decode token from file"));
-		*str = NULL;
-		return -1;
-	}
+
+	ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, TRUE);
+
 
 	if (len<32) {
+
 		if (len <= 0) {
 			ret = NULL;
 		} else if (len<= tokens_len) {
-			ret = g_strdup(tokens[len-1]);
+			ret = tokens[len-1];
 		} else {
 			io(printf ("Invalid token encountered: %d", len));
 			*str = NULL;
@@ -1350,17 +1364,14 @@
 		*str = NULL;
 		return -1;
 	} else {
+
 		len -= 32;
-		ret = g_malloc(len+1);
-		if (len > 0 && fread(ret, len, 1, in) != 1) {
-			g_free(ret);
-			*str = NULL;
-			return -1;
-		}
-		ret[len]=0;
+		ret = (char*)ptrchr;
+		ptrchr += len;
+
 	}
 
-	io(printf("Token = '%s'\n", ret));
+	s->filepos = ptrchr;
 
 	*str = ret;
 	return 0;
@@ -1392,40 +1403,81 @@
 static int
 summary_header_load(CamelFolderSummary *s, FILE *in)
 {
-	fseek(in, 0, SEEK_SET);
+	time_t *ptrt;
+	gint32 *ptr32 = s->filepos;
 
-	io(printf("Loading header\n"));
+	s->version = g_ntohl (*ptr32); ptr32++; 
 
-	if (camel_file_util_decode_fixed_int32(in, &s->version) == -1)
-		return -1;
+	io(printf("Loading header %d", (s->version&0xff)));
 
-	/* Legacy version check, before version 12 we have no upgrade knowledge */
+	/* Legacy version check, before version 13 we have no upgrade knowledge */
 	if ((s->version > 0xff) && (s->version & 0xff) < 12) {
 		io(printf ("Summary header version mismatch"));
 		errno = EINVAL;
 		return -1;
 	}
 
-	if (!(s->version < 0x100 && s->version >= 13))
-		io(printf("Loading legacy summary\n"));
-	else
-		io(printf("loading new-format summary\n"));
+	/* Check for mmap version */
+	if (s->folder && s->version != CAMEL_FOLDER_SUMMARY_VERSION) {
+		gboolean old;
+
+		io(printf ("Summary header version needs upgrade"));
+
+		g_mapped_file_free (s->file);
+		s->file = NULL;
+
+		s->flags |= CAMEL_SUMMARY_DIRTY;
+		s->folder->summary = s;
+	
+		if (CAMEL_IS_FOLDER (s->folder))
+		{
+			CamelFolderClass *folder_klass = (CamelFolderClass*)CAMEL_OBJECT_GET_CLASS(s->folder);
+			folder_klass->refresh_info (s->folder, NULL);
+		} else {
+
+			/* Local store? */
+
+			errno = EINVAL;
+			return -1;
+		}
+
+		old = s->build_content;
+		camel_folder_summary_set_build_content (s, TRUE);
+		CAMEL_SUMMARY_UNLOCK(s, io_lock);
+		camel_folder_summary_save (s);
+		CAMEL_SUMMARY_LOCK(s, io_lock);
+		camel_folder_summary_set_build_content (s, old);
+
+		if (s->summary_path == NULL || !g_file_test (s->summary_path, G_FILE_TEST_EXISTS | G_FILE_TEST_IS_REGULAR))
+		{
+			errno = EINVAL;
+			return -1;
+		}
+
+		s->file = g_mapped_file_new (s->summary_path, FALSE, NULL);
+		ptr32 = (gint32*)g_mapped_file_get_contents (s->file);
+		s->filepos = ptr32;
+		s->version = g_ntohl (*ptr32); ptr32++; 
 
-	/* legacy version */
-	if (camel_file_util_decode_fixed_int32(in, &s->flags) == -1
-	    || camel_file_util_decode_fixed_int32(in, &s->nextuid) == -1
-	    || camel_file_util_decode_time_t(in, &s->time) == -1
-	    || camel_file_util_decode_fixed_int32(in, &s->saved_count) == -1) {
-		return -1;
 	}
 
-	/* version 13 */
-	if (s->version < 0x100 && s->version >= 13
-	    && (camel_file_util_decode_fixed_int32(in, &s->unread_count) == -1
-		|| camel_file_util_decode_fixed_int32(in, &s->deleted_count) == -1
-		|| camel_file_util_decode_fixed_int32(in, &s->junk_count) == -1)) {
-		return -1;
+	s->flags = g_ntohl (*ptr32); ptr32++; 
+	s->nextuid = g_ntohl (*ptr32); ptr32++; 
+
+	ptrt = (time_t*)ptr32;
+	s->time = (time_t)g_ntohl (*ptrt); ptrt++; 
+	ptr32 = (gint32*)ptrt;
+
+	s->saved_count = g_ntohl (*ptr32); ptr32++; 
+
+
+	if (s->version < 0x100 && s->version >= 13) {
+		s->unread_count = g_ntohl (*ptr32); ptr32++; 
+		s->deleted_count = g_ntohl (*ptr32); ptr32++; 
+		s->junk_count = g_ntohl (*ptr32); ptr32++; 
 	}
+     
+	s->filepos = ptr32;
 
 	return 0;
 }
@@ -1681,79 +1733,137 @@
 	return (CamelMessageInfo *)mi;
 }
 
+unsigned char*
+decode_uint32 (unsigned char *start, guint32 *dest, gboolean is_string)
+{
+	guint32 value = 0;
+	unsigned char uv = *start;
+	int v;
+
+	/* until we get the last byte, keep decoding 7 bits at a time */
+
+        while ( ((v = uv) & 0x80) == 0 ) {
+                value |= v;
+                value <<= 7;
+		start++; uv = *start;
+        }
+
+	*dest = value | (v & 0x7f);
+
+	if (is_string && *dest == 1)
+		*dest = 0;
+
+	return ++start;
+}
+
 static CamelMessageInfo *
 message_info_load(CamelFolderSummary *s, FILE *in)
 {
 	CamelMessageInfoBase *mi;
-	guint count;
+	guint count, len;
+	unsigned char *ptrchr = s->filepos;
+	gint32 *ptr32;
+	time_t *ptrt;   
 	int i;
-	char *subject, *from, *to, *cc, *mlist, *uid;
 
 	mi = (CamelMessageInfoBase *)camel_message_info_new(s);
 
 	io(printf("Loading message info\n"));
 
-	camel_file_util_decode_string(in, &uid);
-	camel_file_util_decode_uint32(in, &mi->flags);
-	camel_file_util_decode_uint32(in, &mi->size);
-	camel_file_util_decode_time_t(in, &mi->date_sent);
-	camel_file_util_decode_time_t(in, &mi->date_received);
-	camel_file_util_decode_string(in, &subject);
-	camel_file_util_decode_string(in, &from);
-	camel_file_util_decode_string(in, &to);
-	camel_file_util_decode_string(in, &cc);
-	camel_file_util_decode_string(in, &mlist);
-	
-	mi->uid = uid;
-	mi->subject = camel_pstring_add (subject, TRUE);
-	mi->from = camel_pstring_add (from, TRUE);
-	mi->to = camel_pstring_add (to, TRUE);
-	mi->cc = camel_pstring_add (cc, TRUE);
-	mi->mlist = camel_pstring_add (mlist, TRUE);
-	
+	ptrchr = decode_uint32 (ptrchr, &len, TRUE);
+	mi->uid = (char*)ptrchr;
+	ptrchr += len;
+       
+	ptrchr = decode_uint32 (ptrchr, &mi->flags, FALSE);
+	ptrchr = decode_uint32 (ptrchr, &mi->size, FALSE);
+       
+	ptrt = (time_t*) ptrchr;
+	mi->date_sent = (time_t) g_ntohl (*ptrt); ptrt++;
+	mi->date_received = (time_t) g_ntohl (*ptrt); ptrt++;
+
+	ptrchr = (unsigned char*) ptrt;
+       
+	ptrchr = decode_uint32 (ptrchr, &len, TRUE);
+	mi->subject = (const char*)ptrchr;
+	ptrchr += len;
+
+	ptrchr = decode_uint32 (ptrchr, &len, TRUE);
+	mi->from = (const char*)ptrchr;
+	ptrchr += len;
+
+	ptrchr = decode_uint32 (ptrchr, &len, TRUE);
+	mi->to = (const char*)ptrchr;
+	ptrchr += len;
+
+	ptrchr = decode_uint32 (ptrchr, &len, TRUE);
+	mi->cc = (const char*)ptrchr;
+	ptrchr += len;
+
+	ptrchr = decode_uint32 (ptrchr, &len, TRUE);
+	mi->mlist = (const char*)ptrchr;
+	ptrchr += len;	
+
 	mi->content = NULL;
 
-	camel_file_util_decode_fixed_int32(in, &mi->message_id.id.part.hi);
-	camel_file_util_decode_fixed_int32(in, &mi->message_id.id.part.lo);
+	ptr32 = (gint32*) ptrchr;
 
-	if (camel_file_util_decode_uint32(in, &count) == -1 || count > 500)
-		goto error;
+	mi->message_id.id.part.hi = g_ntohl (*ptr32); ptr32++;
+	mi->message_id.id.part.lo = g_ntohl (*ptr32); ptr32++;
 
-	if (count > 0) {
-		mi->references = g_malloc(sizeof(*mi->references) + ((count-1) * sizeof(mi->references->references[0])));
-		mi->references->size = count;
-		for (i=0;i<count;i++) {
-			camel_file_util_decode_fixed_int32(in, &mi->references->references[i].id.part.hi);
-			camel_file_util_decode_fixed_int32(in, &mi->references->references[i].id.part.lo);
-		}
-	}
+	ptrchr = (unsigned char*) ptr32;
+	ptrchr = decode_uint32 (ptrchr, &count, FALSE);
+	ptr32 = (gint32*) ptrchr;
 
-	if (camel_file_util_decode_uint32(in, &count) == -1 || count > 500)
-		goto error;
+	mi->references = g_malloc(sizeof(*mi->references) + ((count-1) * sizeof(mi->references->references[0])));
+	mi->references->size = count;
+	
+	for (i=0;i<count;i++) {
+		mi->references->references[i].id.part.hi = g_ntohl (*ptr32); ptr32++;
+		mi->references->references[i].id.part.lo = g_ntohl (*ptr32); ptr32++;
+	}
 
+	ptrchr = (unsigned char*) ptr32;
+	ptrchr = decode_uint32 (ptrchr, &count, FALSE);
+	
 	for (i=0;i<count;i++) {
 		char *name;
-		if (camel_file_util_decode_string(in, &name) == -1 || name == NULL)
+
+		ptrchr = decode_uint32 (ptrchr, &len, TRUE);
+		name = (char*)ptrchr;
+		ptrchr += len;
+
+		if (name == NULL)
 			goto error;
+
 		camel_flag_set(&mi->user_flags, name, TRUE);
-		g_free(name);
 	}
 
-	if (camel_file_util_decode_uint32(in, &count) == -1 || count > 500)
+	ptrchr = decode_uint32 (ptrchr, &count, FALSE);
+
+	if (count > 500)
 		goto error;
 
 	for (i=0;i<count;i++) {
 		char *name, *value;
-		if (camel_file_util_decode_string(in, &name) == -1 || name == NULL
-		    || camel_file_util_decode_string(in, &value) == -1)
+
+		ptrchr = decode_uint32 (ptrchr, &len, TRUE);
+		name = (char*)ptrchr;
+		ptrchr += len;
+
+		if (name == NULL)
 			goto error;
+
+		ptrchr = decode_uint32 (ptrchr, &len, TRUE);
+		value =(char*) ptrchr;
+		ptrchr += len;
+
 		camel_tag_set(&mi->user_tags, name, value);
-		g_free(name);
-		g_free(value);
 	}
 
-	if (!ferror(in))
-		return (CamelMessageInfo *)mi;
+
+	s->filepos = (void*) ptrchr;
+
+	return (CamelMessageInfo *)mi;
 
 error:
 	camel_message_info_free((CamelMessageInfo *)mi);
@@ -1821,15 +1931,8 @@
 {
 	CamelMessageInfoBase *mi = (CamelMessageInfoBase *)info;
 
-	g_free(mi->uid);
-	camel_pstring_free(mi->subject);
-	camel_pstring_free(mi->from);
-	camel_pstring_free(mi->to);
-	camel_pstring_free(mi->cc);
-	camel_pstring_free(mi->mlist);
 	g_free(mi->references);
-	camel_flag_list_free(&mi->user_flags);
-	camel_tag_list_free(&mi->user_tags);
+
 	if (s)
 		e_memchunk_free(s->message_info_chunks, mi);
 	else
@@ -1861,46 +1964,43 @@
 	guint32 count, i;
 	CamelContentType *ct;
 
+	unsigned char *ptrchr = s->filepos;
+
 	io(printf("Loading content info\n"));
 
 	ci = camel_folder_summary_content_info_new(s);
 	
-	camel_folder_summary_decode_token(in, &type);
-	camel_folder_summary_decode_token(in, &subtype);
+	camel_folder_summary_decode_token(s, in, &type);
+	camel_folder_summary_decode_token(s, in, &subtype);
+
 	ct = camel_content_type_new(type, subtype);
-	g_free(type);		/* can this be removed? */
-	g_free(subtype);
-	if (camel_file_util_decode_uint32(in, &count) == -1 || count > 500)
-		goto error;
+
+	ptrchr = s->filepos;
+	ptrchr = decode_uint32 ((unsigned char*)ptrchr, &count, FALSE);	
+	s->filepos = ptrchr;
 	
 	for (i=0;i<count;i++) {
 		char *name, *value;
-		camel_folder_summary_decode_token(in, &name);
-		camel_folder_summary_decode_token(in, &value);
-		if (!(name && value))
-			goto error;
+
+		camel_folder_summary_decode_token(s, in, &name);
+		camel_folder_summary_decode_token(s, in, &value);
 		
-		camel_content_type_set_param(ct, name, value);
-		/* TODO: do this so we dont have to double alloc/free */
-		g_free(name);
-		g_free(value);
+		camel_content_type_set_param (ct, name, value);
 	}
-	ci->type = ct;
 
-	camel_folder_summary_decode_token(in, &ci->id);
-	camel_folder_summary_decode_token(in, &ci->description);
-	camel_folder_summary_decode_token(in, &ci->encoding);
+	ci->type = ct;
 
-	camel_file_util_decode_uint32(in, &ci->size);
+	camel_folder_summary_decode_token(s, in, &ci->id);
+	camel_folder_summary_decode_token(s, in, &ci->description);
+	camel_folder_summary_decode_token(s, in, &ci->encoding);
+
+	ptrchr = s->filepos;
+	ptrchr = decode_uint32 ((unsigned char*)ptrchr, &ci->size, FALSE);
+	s->filepos = ptrchr;
 
 	ci->childs = NULL;
 
-	if (!ferror(in))
-		return ci;
-
- error:
-	camel_folder_summary_content_info_free(s, ci);
-	return NULL;
+	return ci;
 }
 
 static int
@@ -1937,9 +2037,7 @@
 content_info_free(CamelFolderSummary *s, CamelMessageContentInfo *ci)
 {
 	camel_content_type_unref(ci->type);
-	g_free(ci->id);
-	g_free(ci->description);
-	g_free(ci->encoding);
+
 	e_memchunk_free(s->content_info_chunks, ci);
 }
 
Index: camel-folder-summary.h
===================================================================
RCS file: /cvs/gnome/evolution-data-server/camel/camel-folder-summary.h,v
retrieving revision 1.86
diff -u -r1.86 camel-folder-summary.h
--- camel-folder-summary.h	31 Aug 2005 04:21:56 -0000	1.86
+++ camel-folder-summary.h	10 Jul 2006 07:54:42 -0000
@@ -224,6 +224,9 @@
 	GHashTable *messages_uid; /* CamelMessageInfo's by uid */
 
 	struct _CamelFolder *folder; /* parent folder, for events */
+
+	GMappedFile *file;
+	void *filepos;
 };
 
 struct _CamelFolderSummaryClass {
@@ -277,6 +280,9 @@
 CamelType			 camel_folder_summary_get_type	(void);
 CamelFolderSummary      *camel_folder_summary_new	(struct _CamelFolder *folder);
 
+unsigned char*
+decode_uint32 (unsigned char *start, guint32 *dest, gboolean is_string);
+
 void camel_folder_summary_set_filename(CamelFolderSummary *summary, const char *filename);
 void camel_folder_summary_set_index(CamelFolderSummary *summary, CamelIndex *index);
 void camel_folder_summary_set_build_content(CamelFolderSummary *summary, gboolean state);
@@ -329,7 +335,7 @@
 
 /* basically like strings, but certain keywords can be compressed and de-cased */
 int camel_folder_summary_encode_token(FILE *out, const char *str);
-int camel_folder_summary_decode_token(FILE *in, char **str);
+int camel_folder_summary_decode_token(CamelFolderSummary *s,FILE *in, char **str);
 
 /* message flag operations */
 gboolean	camel_flag_get(CamelFlag **list, const char *name);
Index: providers/imap/camel-imap-summary.c
===================================================================
RCS file: /cvs/gnome/evolution-data-server/camel/providers/imap/camel-imap-summary.c,v
retrieving revision 1.25
diff -u -r1.25 camel-imap-summary.c
--- providers/imap/camel-imap-summary.c	31 Aug 2005 04:26:02 -0000	1.25
+++ providers/imap/camel-imap-summary.c	10 Jul 2006 07:54:42 -0000
@@ -148,28 +148,34 @@
 summary_header_load (CamelFolderSummary *s, FILE *in)
 {
 	CamelImapSummary *ims = CAMEL_IMAP_SUMMARY (s);
+	gint32 *ptr32;
 	
 	if (camel_imap_summary_parent->summary_header_load (s, in) == -1)
 		return -1;
 
 	/* Legacy version */
 	if (s->version == 0x30c)
-		return camel_file_util_decode_uint32(in, &ims->validity);
+	{
+		unsigned char* ptrchr = s->filepos;
+		ptrchr = decode_uint32 (ptrchr, &ims->validity, FALSE);
+		s->filepos = ptrchr;
+		return 0;
+	}
 
 	/* Version 1 */
-	if (camel_file_util_decode_fixed_int32(in, &ims->version) == -1)
-		return -1;
+	ptr32 = s->filepos;
+	ims->version = g_ntohl (*ptr32); ptr32++;
+	s->filepos = ptr32;
 	
 	if (ims->version == 2) {
 		/* Version 2: for compat with version 2 of the imap4 summary files */
 		int have_mlist;
-		
-		if (camel_file_util_decode_fixed_int32 (in, &have_mlist) == -1)
-			return -1;
+		have_mlist = g_ntohl (*ptr32); ptr32++;
+		s->filepos = ptr32;
 	}
-	
-	if (camel_file_util_decode_fixed_int32(in, &ims->validity) == -1)
-		return -1;
+
+	ims->validity = g_ntohl (*ptr32); ptr32++;
+	s->filepos = ptr32;
 	
 	if (ims->version > CAMEL_IMAP_SUMMARY_VERSION) {
 		g_warning("Unkown summary version\n");
@@ -212,19 +218,15 @@
 	CamelImapMessageInfo *iinfo;
 
 	info = camel_imap_summary_parent->message_info_load (s, in);
-	if (info) {
-		iinfo = (CamelImapMessageInfo *)info;
-
-		if (camel_file_util_decode_uint32 (in, &iinfo->server_flags) == -1)
-			goto error;
 
+	if (info) {
+		unsigned char* ptrchr = s->filepos;
+		ptrchr = decode_uint32 (ptrchr, &iinfo->server_flags, FALSE);
+		s->filepos = ptrchr;
 		label_to_flags(iinfo);
 	}
 
 	return info;
-error:
-	camel_message_info_free(info);
-	return NULL;
 }
 
 static int
@@ -254,10 +256,22 @@
 static CamelMessageContentInfo *
 content_info_load (CamelFolderSummary *s, FILE *in)
 {
-	if (fgetc (in))
-		return camel_imap_summary_parent->content_info_load (s, in);
-	else
-		return camel_folder_summary_content_info_new (s);
+	unsigned char* ptrchr = s->filepos;
+	CamelMessageContentInfo *retval;
+
+	if (*ptrchr == 1) {
+		ptrchr++;
+		s->filepos = ptrchr;
+
+		retval = camel_imap_summary_parent->content_info_load (s, in);
+	} else {
+		ptrchr++; 
+		s->filepos = ptrchr;
+
+		retval = camel_folder_summary_content_info_new (s);
+	}
+
+	return retval;
 }
 
 static int
Index: providers/local/camel-local-summary.c
===================================================================
RCS file: /cvs/gnome/evolution-data-server/camel/providers/local/camel-local-summary.c,v
retrieving revision 1.34
diff -u -r1.34 camel-local-summary.c
--- providers/local/camel-local-summary.c	9 Jun 2006 01:21:53 -0000	1.34
+++ providers/local/camel-local-summary.c	10 Jul 2006 07:54:42 -0000
@@ -593,6 +593,7 @@
 summary_header_load(CamelFolderSummary *s, FILE *in)
 {
 	CamelLocalSummary *cls = (CamelLocalSummary *)s;
+	gint32 *ptr32 = s->filepos;
 
 	/* We dont actually add our own headers, but version that we don't anyway */
 
@@ -604,7 +605,13 @@
 		return 0;
 
 	/* otherwise load the version number */
-	return camel_file_util_decode_fixed_int32(in, &cls->version);
+	
+	ptr32 = s->filepos;
+	cls->version = g_ntohl (*ptr32); ptr32++;
+
+	s->filepos = ptr32;
+
+	return 0;
 }
 
 static int
Index: providers/local/camel-mbox-summary.c
===================================================================
RCS file: /cvs/gnome/evolution-data-server/camel/providers/local/camel-mbox-summary.c,v
retrieving revision 1.60
diff -u -r1.60 camel-mbox-summary.c
--- providers/local/camel-mbox-summary.c	15 Jun 2006 11:16:25 -0000	1.60
+++ providers/local/camel-mbox-summary.c	10 Jul 2006 07:54:43 -0000
@@ -237,18 +237,27 @@
 summary_header_load(CamelFolderSummary *s, FILE *in)
 {
 	CamelMboxSummary *mbs = CAMEL_MBOX_SUMMARY(s);
+	gint32 *ptr32;
 
 	if (((CamelFolderSummaryClass *)camel_mbox_summary_parent)->summary_header_load(s, in) == -1)
 		return -1;
 
 	/* legacy version */
 	if (s->version == 0x120c)
-		return camel_file_util_decode_uint32(in, (guint32 *) &mbs->folder_size);
+	{
+		unsigned char* ptrchr = s->filepos;
+		ptrchr = decode_uint32 (ptrchr, &mbs->folder_size, FALSE);
+		s->filepos = ptrchr;
+	}
 
 	/* version 1 */
-	if (camel_file_util_decode_fixed_int32(in, &mbs->version) == -1
-	    || camel_file_util_decode_size_t(in, &mbs->folder_size) == -1)
-		return -1;
+
+	ptr32 = s->filepos;
+
+	mbs->version = g_ntohl (*ptr32); ptr32++;
+	mbs->folder_size = g_ntohl (*ptr32); ptr32++;
+
+	s->filepos = ptr32;
 
 	return 0;
 }
@@ -361,6 +370,26 @@
 	return mi;
 }
 
+#define MMAP_DECODE_T(type)				\
+static unsigned char*					\
+mmap_decode_##type(unsigned char *start, type *dest)	\
+{							\
+	unsigned char uv = *start;			\
+	type save = 0;					\
+	int i = sizeof(type) - 1;			\
+	int v;						\
+							\
+        while (i >= 0 && (v = uv)) {			\
+		save |= ((type)v) << (i * 8);		\
+		i--;					\
+		start++; uv=*start;			\
+	}						\
+	*dest = save;					\
+	return ++start;					\
+}
+
+MMAP_DECODE_T(off_t)
+
 static CamelMessageInfo *
 message_info_load(CamelFolderSummary *s, FILE *in)
 {
@@ -371,9 +400,12 @@
 	mi = ((CamelFolderSummaryClass *)camel_mbox_summary_parent)->message_info_load(s, in);
 	if (mi) {
 		CamelMboxMessageInfo *mbi = (CamelMboxMessageInfo *)mi;
+
+		unsigned char *ptrchr = s->filepos;
 		
-		if (camel_file_util_decode_off_t(in, &mbi->frompos) == -1)
-			goto error;
+		ptrchr = mmap_decode_off_t(ptrchr, &mbi->frompos);
+
+		s->filepos = ptrchr;
 	}
 	
 	return mi;
@@ -382,6 +414,7 @@
 	return NULL;
 }
 
+
 static int
 message_info_save(CamelFolderSummary *s, FILE *out, CamelMessageInfo *mi)
 {
@@ -390,7 +423,7 @@
 	io(printf("saving mbox message info\n"));
 
 	if (((CamelFolderSummaryClass *)camel_mbox_summary_parent)->message_info_save(s, out, mi) == -1
-	    || camel_file_util_encode_off_t(out, mbi->frompos) == -1)
+	    || camel_file_util_encode_off_t (out, mbi->frompos) == -1)
 		return -1;
 
 	return 0;