On Sun, 2006-07-09 at 12:53 +0200, Philip Van Hoof wrote: > You really want to try it ;-) > > http://pvanhoof.be/blog/index.php/2006/07/09/the-camel-folder-summaryc-with-mmap-now-works Note about this E-mail. I'm assuming people who will read this, have a technical in-depth knowledge of Camel and camel-folder-summary.c The reason why the current (not my mmap() patched) implementation clearly needs more memory than the size of the summary file, is because the content-info tokens are stored compressed in the file. They are, however, NOT stored compressed in the memory cause of an strdup in the "decode_token" method. For example, if you save a content-info that is in the static-table, the length of the pstring is not added with 32. If you do, it's added with 32 and the string itself is not saved to the file. However. The implementation camel_folder_summary_decode_token does an strdup of the entry in the table. This bypassed the very idea of the compression. The file will be small, but the memory needed wont be. This new patch removes that strdup. Note that this patch also includes the mmap() idea. If you want to remove the strdup, you'll need to check whether you can remove it in the finalization (where the free's are happening). You can for example not strdup() it and simply before the free check whether the pointer is located in the table. And if not, free it. That way, you would (without the mmap() idea) save ~100 kb on folders that are > 10,000 headers. The new patch also fixes some incorrect free()'s. These free()'s aren't needed anymore because of the destruction of the mmap() area. They actually where real bugs that caused segmentation errors (free()ing a pointer in a mmap() area is not good). More updates on this patch are probably going to follow. So I'm going to start using a naming scheme that will help you guys tracking this. I'm going to use camel_folder_summary_with_mmap_fixesXX.diff where XX is the version of the patch. We'll start at 01 with this one. The attached massif graph is tinymail opening and closing a few folders. I might not always attach the valgrind report. But if I do, it's always a one on one relation with the patch (unless I explicitly say different). If you try the patch with Evolution, you will PROBABLY get a stack dump like this after running it in gdb. This is OF COURSE because I haven't yet made the same changes in the MBOX provider of Camel, as the one that I had to make in the IMAP provider. I'm of course going to do that soon. Just give me some time to test everything with tinymail first a little bit. After all, tinymail is really why I'm doing this. The reason why it works with tinymail is of course because tinymail will not convert POP boxes to MBOX ones (like what Evolution does). So there's simply no MBOX providers being used by tinymail. Evolution, however, always has some MBOX'es like the Inbox and Thrash. STRONG NOTE. IF you are online and IF you start using my patch, ALL your summary files will be rewritten once this MBOX bug is fixed. This might NOT be what you want. I propose to keep a copy of your ".evolution" directory just in case !!! I -->HAVE<-- warned you. Program received signal SIGSEGV, Segmentation fault. [Switching to Thread -1232492320 (LWP 8932)] 0xb7027640 in fread () from /lib/tls/i686/cmov/libc.so.6 (gdb) bt #0 0xb7027640 in fread () from /lib/tls/i686/cmov/libc.so.6 #1 0xb6b7c749 in camel_file_util_decode_fixed_int32 (in=0x0, dest=0x0) at camel-file-utils.c:144 #2 0xb56ea5a6 in summary_header_load (s=0x8208178, in=0x0) at camel-local-summary.c:607 #3 0xb56edc69 in summary_header_load (s=0x8208178, in=0x0) at camel-mbox-summary.c:241 #4 0xb6b3bfad in camel_folder_summary_load (s=0x8208178) at camel-folder-summary.c:574 #5 0xb56e9bec in local_summary_load (cls=0x0, forceindex=0, ex=0x0) at camel-local-summary.c:143 #6 0xb56e9c5c in camel_local_summary_load (cls=0x8208178, forceindex=0, ex=0x0) at camel-local-summary.c:155 #7 0xb56e7b78 in camel_local_folder_construct (lf=0x8201b58, parent_store=0x81fa200, full_name=0xb59be4c1 "Drafts", flags=1, ex=0xbff45ba4) at camel-local-folder.c:292 #8 0xb56eaf48 in camel_mbox_folder_new (parent_store=0x0, full_name=0x0, flags=0, ex=0x0) at camel-mbox-folder.c:134 #9 0xb56ebfef in get_folder (store=0x81fa200, folder_name=0xb59be4c1 "Drafts", flags=<value optimized out>, ex=0xbff45ba4) at camel-mbox-store.c:210 #10 0xb6b54781 in camel_store_get_folder (store=0x81fa200, folder_name=0xb59be4c1 "Drafts", flags=<value optimized out>, ex=0xbff45ba4) at camel-store.c:260 #11 0xb598cc8e in mail_filter_delete_uri () from /usr/lib/evolution/2.6/components/libevolution-mail.so #12 0xb598f319 in mail_component_load_store_by_uri () from /usr/lib/evolution/2.6/components/libevolution-mail.so #13 0xb7f27306 in _ORBIT_skel_small_GNOME_Evolution_Component_createView () from /usr/lib/evolution/2.6/libeshell.so.0 #14 0xb7a1a665 in ORBit_c_stub_invoke () from /usr/lib/libORBit-2.so.0 #15 0xb7f28bab in GNOME_Evolution_Component_createView () from /usr/lib/evolution/2.6/libeshell.so.0 #16 0x0805a711 in e_shell_window_get_type () #17 0xb73e768b in IA__g_cclosure_marshal_VOID__INT (closure=0x80faaa0, return_value=0x0, n_param_values=2, param_values=0xbff45fcc, invocation_hint=0xbff45ebc, marshal_data=0x81f5c38) at gmarshal.c:216 #18 0xb73db79f in IA__g_closure_invoke (closure=0x81f5c38, return_value=0x0, n_param_values=0, param_values=0x0, invocation_hint=0x0) at gclosure.c:490 #19 0xb73ea2ea in signal_emit_unlocked_R (node=0x81f5c60, detail=0, instance=0x81929b0, emission_return=0x0, instance_and_params=0xbff45fcc) at gsignal.c:2438 #20 0xb73ebb19 in IA__g_signal_emit_valist (instance=0x81929b0, signal_id=197, detail=0, var_args=<value optimized out>) at gsignal.c:2197 #21 0xb73ebe89 in IA__g_signal_emit (instance=0x0, signal_id=0, detail=0) at gsignal.c:2241 #22 0x0805dbc1 in e_sidebar_select_button () #23 0x0805b3a9 in e_shell_window_switch_to_component () #24 0x0805c36a in e_shell_window_new () #25 0x08054513 in e_shell_create_window () #26 0x0805eb34 in es_menu_hook_get_type () #27 0xb7370be2 in g_idle_dispatch (source=0x8189660, callback=0, user_data=0x0) at gmain.c:3796 #28 0xb736e8c6 in IA__g_main_context_dispatch (context=0x80a4b70) at gmain.c:1916 #29 0xb7371986 in g_main_context_iterate (context=0x80a4b70, block=1, dispatch=1, self=0x806da98) at gmain.c:2547 #30 0xb7371ca8 in IA__g_main_loop_run (loop=0x80e7d70) at gmain.c:2751 #31 0xb7a9d477 in bonobo_main () from /usr/lib/libbonobo-2.so.0 #32 0x0805f03c in main -- Philip Van Hoof, software developer at x-tend home: me at pvanhoof dot be gnome: pvanhoof at gnome dot org work: vanhoof at x-tend dot be http://www.pvanhoof.be - http://www.x-tend.be
Attachment:
massif.7970.ps.gz
Description: application/gzpostscript
? camel-mime-tables.c
Index: camel-file-utils.c
===================================================================
RCS file: /cvs/gnome/evolution-data-server/camel/camel-file-utils.c,v
retrieving revision 1.17
diff -u -r1.17 camel-file-utils.c
--- camel-file-utils.c 2 Jun 2006 00:52:29 -0000 1.17
+++ camel-file-utils.c 9 Jul 2006 12:46:58 -0000
@@ -269,11 +269,21 @@
if ((len = strlen (str)) > 65536)
len = 65536;
+
+ if (len==0) len=-1;
if (camel_file_util_encode_uint32 (out, len+1) == -1)
return -1;
- if (len == 0 || fwrite (str, len, 1, out) == 1)
- return 0;
+
+ if (len != 0)
+ {
+ if (fwrite (str, len, 1, out) == 1)
+ {
+ fputc ('\0', out);
+ return 0;
+ }
+ }
+
return -1;
}
Index: camel-folder-summary.c
===================================================================
RCS file: /cvs/gnome/evolution-data-server/camel/camel-folder-summary.c,v
retrieving revision 1.149
diff -u -r1.149 camel-folder-summary.c
--- camel-folder-summary.c 6 Jul 2006 19:43:46 -0000 1.149
+++ camel-folder-summary.c 9 Jul 2006 12:47:02 -0000
@@ -50,6 +50,7 @@
#include "camel-stream-null.h"
#include "camel-stream-filter.h"
#include "camel-string-utils.h"
+#include "camel-disco-folder.h"
#include "libedataserver/md5-utils.h"
#include "libedataserver/e-memory.h"
@@ -66,15 +67,15 @@
/* this should probably be conditional on it existing */
#define USE_BSEARCH
-#define d(x)
+#define d(x)
#define io(x) /* io debug */
-#define w(x)
+#define w(x)
#if 0
extern int strdup_count, malloc_count, free_count;
#endif
-#define CAMEL_FOLDER_SUMMARY_VERSION (13)
+#define CAMEL_FOLDER_SUMMARY_VERSION (14)
#define _PRIVATE(o) (((CamelFolderSummary *)(o))->priv)
@@ -83,16 +84,17 @@
struct _node *next;
};
+
static struct _node *my_list_append(struct _node **list, struct _node *n);
static int my_list_size(struct _node **list);
-static int summary_header_load(CamelFolderSummary *, FILE *);
-static int summary_header_save(CamelFolderSummary *, FILE *);
+static int summary_header_load(CamelFolderSummary *, FILE *in);
+static int summary_header_save(CamelFolderSummary *, FILE *out);
static CamelMessageInfo * message_info_new_from_header(CamelFolderSummary *, struct _camel_header_raw *);
static CamelMessageInfo * message_info_new_from_parser(CamelFolderSummary *, CamelMimeParser *);
static CamelMessageInfo * message_info_new_from_message(CamelFolderSummary *s, CamelMimeMessage *msg);
-static CamelMessageInfo * message_info_load(CamelFolderSummary *, FILE *);
+static CamelMessageInfo * message_info_load(CamelFolderSummary *, FILE *in);
static int message_info_save(CamelFolderSummary *, FILE *, CamelMessageInfo *);
static void message_info_free(CamelFolderSummary *, CamelMessageInfo *);
@@ -160,6 +162,11 @@
struct _CamelFolderSummaryPrivate *p;
CamelFolderSummary *s = (CamelFolderSummary *)obj;
+
+ if (s->file)
+ g_mapped_file_free (s->file);
+ s->file = NULL;
+
p = _PRIVATE(obj);
camel_folder_summary_clear(s);
@@ -236,7 +243,7 @@
CamelFolderSummary *new = CAMEL_FOLDER_SUMMARY ( camel_object_new (camel_folder_summary_get_type ()));
new->folder = folder;
-
+
return new;
}
@@ -504,17 +511,24 @@
int i;
guint32 count;
CamelMessageContentInfo *ci, *part;
+ unsigned char *ptrchr = s->filepos;
ci = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->content_info_load(s, in);
+
if (ci == NULL)
return NULL;
- if (camel_file_util_decode_uint32(in, &count) == -1 || count > 500) {
+ ptrchr = s->filepos;
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &count, FALSE);
+ s->filepos = ptrchr;
+
+ if (count > 500) {
camel_folder_summary_content_info_free(s, ci);
return NULL;
}
for (i=0;i<count;i++) {
+
part = perform_content_info_load(s, in);
if (part) {
my_list_append((struct _node **)&ci->childs, (struct _node *)part);
@@ -547,22 +561,28 @@
if (s->summary_path == NULL)
return 0;
- in = g_fopen(s->summary_path, "rb");
+ if (!s->file)
+ {
+ s->file = g_mapped_file_new (s->summary_path, FALSE, NULL);
+ s->filepos = g_mapped_file_get_contents (s->file);
+ }
+
if (in == NULL)
return -1;
CAMEL_SUMMARY_LOCK(s, io_lock);
- if ( ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_load(s, in) == -1)
+ if ( ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_load(s, NULL) == -1)
goto error;
/* now read in each message ... */
for (i=0;i<s->saved_count;i++) {
- mi = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->message_info_load(s, in);
+ mi = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->message_info_load(s, NULL);
if (mi == NULL)
goto error;
/* FIXME: this should be done differently, how i don't know */
+
if (s->build_content) {
((CamelMessageInfoBase *)mi)->content = perform_content_info_load(s, in);
if (((CamelMessageInfoBase *)mi)->content == NULL) {
@@ -575,9 +595,6 @@
}
CAMEL_SUMMARY_UNLOCK(s, io_lock);
-
- if (fclose (in) != 0)
- return -1;
s->flags &= ~CAMEL_SUMMARY_DIRTY;
@@ -588,7 +605,6 @@
g_warning ("Cannot load summary file: `%s': %s", s->summary_path, g_strerror (errno));
CAMEL_SUMMARY_UNLOCK(s, io_lock);
- fclose (in);
s->flags |= ~CAMEL_SUMMARY_DIRTY;
return -1;
@@ -724,21 +740,21 @@
int
camel_folder_summary_header_load(CamelFolderSummary *s)
{
- FILE *in;
int ret;
if (s->summary_path == NULL)
return 0;
- in = g_fopen(s->summary_path, "rb");
- if (in == NULL)
- return -1;
+ if (!s->file)
+ {
+ s->file = g_mapped_file_new (s->summary_path, FALSE, NULL);
+ s->filepos = g_mapped_file_get_contents (s->file);
+ }
CAMEL_SUMMARY_LOCK(s, io_lock);
- ret = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_load(s, in);
+ ret = ((CamelFolderSummaryClass *)(CAMEL_OBJECT_GET_CLASS(s)))->summary_header_load(s, NULL);
CAMEL_SUMMARY_UNLOCK(s, io_lock);
- fclose(in);
s->flags &= ~CAMEL_SUMMARY_DIRTY;
return ret;
}
@@ -1302,12 +1318,14 @@
if (token != -1) {
return camel_file_util_encode_uint32(out, token+1);
} else {
- if (camel_file_util_encode_uint32(out, len+32) == -1)
+ if (camel_file_util_encode_uint32(out, len+32 + 1) == -1)
return -1;
if (fwrite(str, len, 1, out) != 1)
return -1;
+ else fputc ('\0', out);
}
}
+
return 0;
}
@@ -1322,24 +1340,23 @@
* Returns %0 on success or %-1 on fail
**/
int
-camel_folder_summary_decode_token(FILE *in, char **str)
+camel_folder_summary_decode_token(CamelFolderSummary *s, FILE *in, char **str)
{
char *ret;
guint32 len;
+ unsigned char *ptrchr = s->filepos;
io(printf("Decode token ...\n"));
-
- if (camel_file_util_decode_uint32(in, &len) == -1) {
- io(printf ("Could not decode token from file"));
- *str = NULL;
- return -1;
- }
+
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, TRUE);
+
if (len<32) {
+
if (len <= 0) {
ret = NULL;
} else if (len<= tokens_len) {
- ret = g_strdup(tokens[len-1]);
+ ret = tokens[len-1];
} else {
io(printf ("Invalid token encountered: %d", len));
*str = NULL;
@@ -1350,17 +1367,14 @@
*str = NULL;
return -1;
} else {
+
len -= 32;
- ret = g_malloc(len+1);
- if (len > 0 && fread(ret, len, 1, in) != 1) {
- g_free(ret);
- *str = NULL;
- return -1;
- }
- ret[len]=0;
+ ret = ptrchr;
+ ptrchr += len;
+
}
- io(printf("Token = '%s'\n", ret));
+ s->filepos = ptrchr;
*str = ret;
return 0;
@@ -1392,40 +1406,74 @@
static int
summary_header_load(CamelFolderSummary *s, FILE *in)
{
- fseek(in, 0, SEEK_SET);
+ time_t *ptrt;
+ gint32 *ptr32 = s->filepos;
- io(printf("Loading header\n"));
+ s->version = g_ntohl (*ptr32); ptr32++;
- if (camel_file_util_decode_fixed_int32(in, &s->version) == -1)
- return -1;
+ io(printf("Loading header %d", (s->version&0xff)));
- /* Legacy version check, before version 12 we have no upgrade knowledge */
+ /* Legacy version check, before version 13 we have no upgrade knowledge */
if ((s->version > 0xff) && (s->version & 0xff) < 12) {
io(printf ("Summary header version mismatch"));
errno = EINVAL;
return -1;
}
- if (!(s->version < 0x100 && s->version >= 13))
- io(printf("Loading legacy summary\n"));
- else
+ if (s->version != CAMEL_FOLDER_SUMMARY_VERSION) {
+ io(printf ("Summary header version needs upgrade"));
+
+ g_mapped_file_free (s->file);
+ s->file = NULL;
+ s->flags |= CAMEL_SUMMARY_DIRTY;
+
+ CAMEL_SUMMARY_UNLOCK(s, io_lock);
+
+ s->folder->summary = s;
+
+ /* TODO: Handle offline case */
+
+ /* ((CamelDiscoFolderClass *)(CAMEL_OBJECT_GET_CLASS(s->folder)))->refresh_info_offline (s->folder, NULL); */
+ ((CamelDiscoFolderClass *)(CAMEL_OBJECT_GET_CLASS(s->folder)))->refresh_info_online (s->folder, NULL);
+
+ camel_folder_summary_set_build_content (s, TRUE);
+ camel_folder_summary_save (s);
+ camel_folder_summary_set_build_content (s, FALSE);
+
+ CAMEL_SUMMARY_LOCK(s, io_lock);
+
+ s->file = g_mapped_file_new (s->summary_path, FALSE, NULL);
+ ptr32 = g_mapped_file_get_contents (s->file);
+ s->filepos = ptr32;
+ s->version = g_ntohl (*ptr32); ptr32++;
+
+ io(printf("Header upgraded to %d", (s->version&0xff)));
+
+
+ } else
io(printf("loading new-format summary\n"));
- /* legacy version */
- if (camel_file_util_decode_fixed_int32(in, &s->flags) == -1
- || camel_file_util_decode_fixed_int32(in, &s->nextuid) == -1
- || camel_file_util_decode_time_t(in, &s->time) == -1
- || camel_file_util_decode_fixed_int32(in, &s->saved_count) == -1) {
- return -1;
- }
+ s->flags = g_ntohl (*ptr32); ptr32++;
+ s->nextuid = g_ntohl (*ptr32); ptr32++;
+ ptrt = (time_t*)ptr32;
+ s->time = *ptrt; ptrt++;
+ ptr32 = (gint32*)ptrt;
+ s->saved_count = g_ntohl (*ptr32); ptr32++;
+
+
+ if (s->version < 0x100 && s->version >= 13) {
+ s->unread_count = g_ntohl (*ptr32); ptr32++;
+ s->deleted_count = g_ntohl (*ptr32); ptr32++;
+ s->junk_count = g_ntohl (*ptr32); ptr32++;
+ }
+
+ io(printf ("CAMEL: ur:%d, dc:%d, jc:%d, v:%d\n",
+ s->unread_count, s->deleted_count, s->junk_count, s->version));
+
+ /* The first is the implementation version, the second I don't yet know */
+ ptr32++; ptr32++;
- /* version 13 */
- if (s->version < 0x100 && s->version >= 13
- && (camel_file_util_decode_fixed_int32(in, &s->unread_count) == -1
- || camel_file_util_decode_fixed_int32(in, &s->deleted_count) == -1
- || camel_file_util_decode_fixed_int32(in, &s->junk_count) == -1)) {
- return -1;
- }
+ s->filepos = ptr32;
return 0;
}
@@ -1681,79 +1729,172 @@
return (CamelMessageInfo *)mi;
}
+unsigned char*
+decode_uint32 (unsigned char *start, guint32 *dest, gboolean is_string)
+{
+ guint32 value = 0;
+ unsigned char uv = *start;
+ int v;
+
+ /* until we get the last byte, keep decoding 7 bits at a time */
+
+ while ( ((v = uv) & 0x80) == 0 ) {
+ value |= v;
+ value <<= 7;
+ start++; uv = *start;
+ }
+
+ *dest = value | (v & 0x7f);
+
+ if (is_string && *dest == 1)
+ *dest = 0;
+
+
+ return ++start;
+
+}
+
static CamelMessageInfo *
message_info_load(CamelFolderSummary *s, FILE *in)
{
CamelMessageInfoBase *mi;
- guint count;
+ guint count, len, nlen;
+ unsigned char *ptrchr = s->filepos;
+ gint32 *ptr32;
+ time_t *ptrt;
int i;
- char *subject, *from, *to, *cc, *mlist, *uid;
mi = (CamelMessageInfoBase *)camel_message_info_new(s);
io(printf("Loading message info\n"));
- camel_file_util_decode_string(in, &uid);
- camel_file_util_decode_uint32(in, &mi->flags);
- camel_file_util_decode_uint32(in, &mi->size);
- camel_file_util_decode_time_t(in, &mi->date_sent);
- camel_file_util_decode_time_t(in, &mi->date_received);
- camel_file_util_decode_string(in, &subject);
- camel_file_util_decode_string(in, &from);
- camel_file_util_decode_string(in, &to);
- camel_file_util_decode_string(in, &cc);
- camel_file_util_decode_string(in, &mlist);
-
- mi->uid = uid;
- mi->subject = camel_pstring_add (subject, TRUE);
- mi->from = camel_pstring_add (from, TRUE);
- mi->to = camel_pstring_add (to, TRUE);
- mi->cc = camel_pstring_add (cc, TRUE);
- mi->mlist = camel_pstring_add (mlist, TRUE);
-
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, TRUE);
+ io(printf ("uid len = %d, str=%s\n", len, ptrchr));
+ mi->uid = ptrchr;
+ ptrchr += len;
+
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, FALSE);
+ mi->flags = ptrchr;
+
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, FALSE);
+ mi->size = ptrchr;
+
+ ptrt = ptrchr;
+ mi->date_sent = ptrt; ptrt++;
+ mi->date_received = ptrt; ptrt++;
+
+ ptrchr = ptrt;
+
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, TRUE);
+ mi->subject = ptrchr;
+ ptrchr += len;
+
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, TRUE);
+ mi->from = ptrchr;
+ ptrchr += len;
+
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, TRUE);
+ mi->to = ptrchr;
+ ptrchr += len;
+
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, TRUE);
+ mi->cc = ptrchr;
+ ptrchr += len;
+
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, TRUE);
+ mi->mlist = ptrchr;
+ ptrchr += len;
+
+
+
mi->content = NULL;
- camel_file_util_decode_fixed_int32(in, &mi->message_id.id.part.hi);
- camel_file_util_decode_fixed_int32(in, &mi->message_id.id.part.lo);
+ ptr32 = ptrchr;
+
+
+ mi->message_id.id.part.hi = g_ntohl (*ptr32); ptr32++;
+ mi->message_id.id.part.lo = g_ntohl (*ptr32); ptr32++;
- if (camel_file_util_decode_uint32(in, &count) == -1 || count > 500)
+io(printf ("hi:%d lo:%d \n", mi->message_id.id.part.hi, mi->message_id.id.part.lo));
+
+ ptrchr = ptr32;
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &count, FALSE);
+ ptr32 = ptrchr;
+
+ if (count > 500)
goto error;
- if (count > 0) {
- mi->references = g_malloc(sizeof(*mi->references) + ((count-1) * sizeof(mi->references->references[0])));
- mi->references->size = count;
- for (i=0;i<count;i++) {
- camel_file_util_decode_fixed_int32(in, &mi->references->references[i].id.part.hi);
- camel_file_util_decode_fixed_int32(in, &mi->references->references[i].id.part.lo);
- }
+io(printf ("%d refs\n", count));
+
+ mi->references = g_malloc(sizeof(*mi->references) + ((count-1) * sizeof(mi->references->references[0])));
+ mi->references->size = count;
+
+ for (i=0;i<count;i++) {
+ mi->references->references[i].id.part.hi = g_ntohl (*ptr32); ptr32++;
+ mi->references->references[i].id.part.lo = g_ntohl (*ptr32); ptr32++;
+
+io(printf ("refhi=%d reflo=%d at %d of %d flags\n",
+ mi->references->references[i].id.part.hi,
+ mi->references->references[i].id.part.lo,
+ i, count));
+
}
- if (camel_file_util_decode_uint32(in, &count) == -1 || count > 500)
+ ptrchr = ptr32;
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &count, FALSE);
+
+ if (count > 500)
goto error;
+io(printf ("%d flags\n", count));
+
for (i=0;i<count;i++) {
char *name;
- if (camel_file_util_decode_string(in, &name) == -1 || name == NULL)
+
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, TRUE);
+ name = ptrchr;
+ ptrchr += len;
+
+ if (name == NULL)
goto error;
+
+io(printf ("tag %s at %d of %d\n", name, i, count));
+
camel_flag_set(&mi->user_flags, name, TRUE);
- g_free(name);
}
- if (camel_file_util_decode_uint32(in, &count) == -1 || count > 500)
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &count, FALSE);
+
+io(printf ("%d tag sets\n", count));
+
+ if (count > 500)
goto error;
for (i=0;i<count;i++) {
char *name, *value;
- if (camel_file_util_decode_string(in, &name) == -1 || name == NULL
- || camel_file_util_decode_string(in, &value) == -1)
+
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, TRUE);
+ name = ptrchr;
+ ptrchr += len;
+
+ if (name == NULL)
goto error;
+
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &len, TRUE);
+ value = ptrchr;
+ ptrchr += len;
+
+io(printf ("tag set %s v %s at %d of %d\n", name, value, i, count));
+
camel_tag_set(&mi->user_tags, name, value);
- g_free(name);
- g_free(value);
}
- if (!ferror(in))
- return (CamelMessageInfo *)mi;
+
+ s->filepos = ptrchr;
+
+io(printf ("mi ends at %02x\n", (unsigned char*)*ptrchr));
+
+ return (CamelMessageInfo *)mi;
error:
camel_message_info_free((CamelMessageInfo *)mi);
@@ -1821,15 +1962,11 @@
{
CamelMessageInfoBase *mi = (CamelMessageInfoBase *)info;
- g_free(mi->uid);
- camel_pstring_free(mi->subject);
- camel_pstring_free(mi->from);
- camel_pstring_free(mi->to);
- camel_pstring_free(mi->cc);
- camel_pstring_free(mi->mlist);
g_free(mi->references);
- camel_flag_list_free(&mi->user_flags);
- camel_tag_list_free(&mi->user_tags);
+
+/* camel_flag_list_free(&mi->user_flags);
+ camel_tag_list_free(&mi->user_tags); */
+
if (s)
e_memchunk_free(s->message_info_chunks, mi);
else
@@ -1861,42 +1998,51 @@
guint32 count, i;
CamelContentType *ct;
+ unsigned char *ptrchr = s->filepos;
+ gint32 *ptr32 = s->filepos;
+
io(printf("Loading content info\n"));
ci = camel_folder_summary_content_info_new(s);
- camel_folder_summary_decode_token(in, &type);
- camel_folder_summary_decode_token(in, &subtype);
+ camel_folder_summary_decode_token(s, in, &type);
+
+ camel_folder_summary_decode_token(s, in, &subtype);
+
ct = camel_content_type_new(type, subtype);
- g_free(type); /* can this be removed? */
- g_free(subtype);
- if (camel_file_util_decode_uint32(in, &count) == -1 || count > 500)
+
+ ptrchr = s->filepos;
+
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &count, FALSE);
+
+ s->filepos = ptrchr;
+
+ if (count > 500)
goto error;
for (i=0;i<count;i++) {
char *name, *value;
- camel_folder_summary_decode_token(in, &name);
- camel_folder_summary_decode_token(in, &value);
- if (!(name && value))
- goto error;
+
+ camel_folder_summary_decode_token(s, in, &name);
+ camel_folder_summary_decode_token(s, in, &value);
- camel_content_type_set_param(ct, name, value);
- /* TODO: do this so we dont have to double alloc/free */
- g_free(name);
- g_free(value);
+ camel_content_type_set_param (ct, name, value);
+
}
ci->type = ct;
- camel_folder_summary_decode_token(in, &ci->id);
- camel_folder_summary_decode_token(in, &ci->description);
- camel_folder_summary_decode_token(in, &ci->encoding);
+ camel_folder_summary_decode_token(s, in, &ci->id);
- camel_file_util_decode_uint32(in, &ci->size);
+ camel_folder_summary_decode_token(s, in, &ci->description);
+ camel_folder_summary_decode_token(s, in, &ci->encoding);
+
+ ptrchr = s->filepos;
+ ptrchr = decode_uint32 ((unsigned char*)ptrchr, &ci->size, FALSE);
+ s->filepos = ptrchr;
ci->childs = NULL;
- if (!ferror(in))
- return ci;
+ return ci;
error:
camel_folder_summary_content_info_free(s, ci);
@@ -1937,9 +2083,7 @@
content_info_free(CamelFolderSummary *s, CamelMessageContentInfo *ci)
{
camel_content_type_unref(ci->type);
- g_free(ci->id);
- g_free(ci->description);
- g_free(ci->encoding);
+
e_memchunk_free(s->content_info_chunks, ci);
}
Index: camel-folder-summary.h
===================================================================
RCS file: /cvs/gnome/evolution-data-server/camel/camel-folder-summary.h,v
retrieving revision 1.86
diff -u -r1.86 camel-folder-summary.h
--- camel-folder-summary.h 31 Aug 2005 04:21:56 -0000 1.86
+++ camel-folder-summary.h 9 Jul 2006 12:47:03 -0000
@@ -224,6 +224,9 @@
GHashTable *messages_uid; /* CamelMessageInfo's by uid */
struct _CamelFolder *folder; /* parent folder, for events */
+
+ GMappedFile *file;
+ void *filepos;
};
struct _CamelFolderSummaryClass {
@@ -277,6 +280,9 @@
CamelType camel_folder_summary_get_type (void);
CamelFolderSummary *camel_folder_summary_new (struct _CamelFolder *folder);
+unsigned char*
+decode_uint32 (unsigned char *start, guint32 *dest, gboolean is_string);
+
void camel_folder_summary_set_filename(CamelFolderSummary *summary, const char *filename);
void camel_folder_summary_set_index(CamelFolderSummary *summary, CamelIndex *index);
void camel_folder_summary_set_build_content(CamelFolderSummary *summary, gboolean state);
@@ -329,7 +335,7 @@
/* basically like strings, but certain keywords can be compressed and de-cased */
int camel_folder_summary_encode_token(FILE *out, const char *str);
-int camel_folder_summary_decode_token(FILE *in, char **str);
+int camel_folder_summary_decode_token(CamelFolderSummary *s,FILE *in, char **str);
/* message flag operations */
gboolean camel_flag_get(CamelFlag **list, const char *name);
Index: camel-string-utils.c
===================================================================
RCS file: /cvs/gnome/evolution-data-server/camel/camel-string-utils.c,v
retrieving revision 1.6
diff -u -r1.6 camel-string-utils.c
--- camel-string-utils.c 6 Jul 2006 19:43:47 -0000 1.6
+++ camel-string-utils.c 9 Jul 2006 12:47:05 -0000
@@ -164,10 +164,10 @@
char *pstr;
int count;
- if (s == NULL)
+ if (str == NULL)
return NULL;
- if (s[0] == '\0') {
+ if (str[0] == '\0') {
if (own)
g_free (str);
return "";
@@ -180,6 +180,7 @@
if (g_hash_table_lookup_extended (pstring_table, str, (void **) &pstr, &pcount)) {
count = GPOINTER_TO_INT (pcount) + 1;
g_hash_table_insert (pstring_table, pstr, GINT_TO_POINTER (count));
+ g_free (str);
} else {
pstr = own ? str : g_strdup (str);
g_hash_table_insert (pstring_table, pstr, GINT_TO_POINTER (1));
Index: providers/imap/camel-imap-summary.c
===================================================================
RCS file: /cvs/gnome/evolution-data-server/camel/providers/imap/camel-imap-summary.c,v
retrieving revision 1.25
diff -u -r1.25 camel-imap-summary.c
--- providers/imap/camel-imap-summary.c 31 Aug 2005 04:26:02 -0000 1.25
+++ providers/imap/camel-imap-summary.c 9 Jul 2006 12:47:06 -0000
@@ -152,16 +152,18 @@
if (camel_imap_summary_parent->summary_header_load (s, in) == -1)
return -1;
- /* Legacy version */
+ ims->validity = CAMEL_IMAP_SUMMARY_VERSION;
+
+ /* Legacy version
if (s->version == 0x30c)
return camel_file_util_decode_uint32(in, &ims->validity);
- /* Version 1 */
+ * Version 1 *
if (camel_file_util_decode_fixed_int32(in, &ims->version) == -1)
return -1;
if (ims->version == 2) {
- /* Version 2: for compat with version 2 of the imap4 summary files */
+ * Version 2: for compat with version 2 of the imap4 summary files *
int have_mlist;
if (camel_file_util_decode_fixed_int32 (in, &have_mlist) == -1)
@@ -176,7 +178,7 @@
errno = EINVAL;
return -1;
}
-
+*/
return 0;
}
@@ -212,19 +214,15 @@
CamelImapMessageInfo *iinfo;
info = camel_imap_summary_parent->message_info_load (s, in);
- if (info) {
- iinfo = (CamelImapMessageInfo *)info;
-
- if (camel_file_util_decode_uint32 (in, &iinfo->server_flags) == -1)
- goto error;
+ if (info) {
+ unsigned char* ptrchr = s->filepos;
+ ptrchr = decode_uint32 (ptrchr, &iinfo->server_flags, TRUE);
+ s->filepos = ptrchr;
label_to_flags(iinfo);
}
return info;
-error:
- camel_message_info_free(info);
- return NULL;
}
static int
@@ -254,10 +252,22 @@
static CamelMessageContentInfo *
content_info_load (CamelFolderSummary *s, FILE *in)
{
- if (fgetc (in))
- return camel_imap_summary_parent->content_info_load (s, in);
- else
- return camel_folder_summary_content_info_new (s);
+ unsigned char* ptrchr = s->filepos;
+ CamelMessageContentInfo *retval;
+
+ if (*ptrchr == 1) {
+ ptrchr++;
+ s->filepos = ptrchr;
+
+ retval = camel_imap_summary_parent->content_info_load (s, in);
+ } else {
+ ptrchr++;
+ s->filepos = ptrchr;
+
+ retval = camel_folder_summary_content_info_new (s);
+ }
+
+ return retval;
}
static int