Storing Mail Snippets in TextCache

["Kevin Kubasik" <kevin kubasik net>]

Just to try and start tackling the issue of snippeting in the
Evolution Backend, I set SnippetMode = true in my FilterMail and let
my machine go at it. Snippeting actually works fine.. if not splendid.
What is our major concern with storing mail snippets in the textcache?
(or at least making it an option).

If its space on the drive (That textcache can grow ;) ) With my
compressed textcache patch (attached, and in the bugzilla [1] ) its
about 6.7 megs for the textcache with over 10,000 mails (totaling
about 120 megs of space on my imap server) .

I dunno, its just a thought. Since actually snippeting the mail as
selfcached would be extremely difficult with gmime, and at this point,
my textcache is significantly smaller than my indexies (indexs folder
is about 27.4 megs) and performance isn't seriously impacted, it might
be worth a shot.

1- http://bugzilla.gnome.org/show_bug.cgi?id=339470
--
Cheers,
Kevin Kubasik
http://kubasik.net/blog

Index: ./beagled/TextCache.cs
===================================================================
RCS file: /cvs/gnome/beagle/beagled/TextCache.cs,v
retrieving revision 1.31
diff -u -1 -2 -r1.31 TextCache.cs
--- ./beagled/TextCache.cs	28 Feb 2006 20:09:38 -0000	1.31
+++ ./beagled/TextCache.cs	6 May 2006 04:05:27 -0000
@@ -23,24 +23,26 @@
 // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 // DEALINGS IN THE SOFTWARE.
 //
 
 
 using System;
 using System.Collections;
 using System.IO;
 using System.Threading;
 
 using Mono.Data.SqliteClient;
 
+using ICSharpCode.SharpZipLib.GZip;
+
 using Beagle.Util;
 
 namespace Beagle.Daemon {
 
 	public class TextCache {
 
 		static public bool Debug = false;
 
 		public const string SELF_CACHE_TAG = "*self*";
 
 		private string text_cache_dir;
 		private SqliteConnection connection;
@@ -255,25 +257,25 @@
 
 		private string LookupPath (Uri uri, bool create_if_not_found)
 		{
 			lock (connection) {
 				string path = LookupPathRawUnlocked (uri, create_if_not_found);
 				if (path == SELF_CACHE_TAG) {
 					// FIXME: How do we handle URI remapping for self-cached items?
 #if false
 					if (uri_remapper != null)
 						uri = uri_remapper (uri);
 #endif
 					if (! uri.IsFile) {
-						string msg = String.Format ("Non-file uri {0} flagged as self-cached", uri);
+						string msg = String.Format ("Non-file uri {0} flagged as self-cached, Path: {1}", uri, uri.LocalPath );
 						throw new Exception (msg);
 					}
 					return uri.LocalPath;
 				}
 				return path;
 			}
 		}
 		
 		public void MarkAsSelfCached (Uri uri)
 		{
 			lock (connection)
 				Insert (uri, SELF_CACHE_TAG);
@@ -281,30 +283,31 @@
 
 		private bool world_readable = false;
 		public bool WorldReadable {
 			get { return this.world_readable; }
 			set { this.world_readable = value; }
 		}
 
 		public TextWriter GetWriter (Uri uri)
 		{
 			// FIXME: Uri remapping?
 			string path = LookupPath (uri, true);
 
-			FileStream stream;
-			stream = new FileStream (path, FileMode.Create, FileAccess.Write, FileShare.ReadWrite);
-
+			GZipOutputStream stream;
+			stream = new GZipOutputStream( 
+				new FileStream (path, FileMode.Create, FileAccess.Write, FileShare.ReadWrite));
+			//Console.WriteLine("Getting GZipWriter for " + path);
 			// We don't expect to need this again in the near future.
-			FileAdvise.FlushCache (stream);
-			
+			//stream.Flush();
+
 			if (! world_readable) {
 				// Make files only readable by the owner.
 				Mono.Unix.Native.Syscall.chmod (path, (Mono.Unix.Native.FilePermissions) 384);
 			}
 
 			StreamWriter writer;
 			writer = new StreamWriter (stream);
 			return writer;
 		}
 
 		public void WriteFromReader (Uri uri, TextReader reader)
 		{
@@ -323,37 +326,53 @@
 				return;
 			}
 
 			TextWriter writer;
 			writer = GetWriter (uri);
 			writer.WriteLine (str);
 			writer.Close ();
 		}
 
 		// FIXME: Uri remapping?
 		public TextReader GetReader (Uri uri)
 		{
+			//Console.WriteLine("Reader Requested for " + uri.ToString());
 			string path = LookupPath (uri, false);
 			if (path == null)
 				return null;
-
-			FileStream stream;
-			try {
-				stream = new FileStream (path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
-			} catch (FileNotFoundException ex) {
-				return null;
+			StreamReader reader;
+			
+			if (XdgMime.GetMimeType (path) == "application/x-gzip" && path.StartsWith (text_cache_dir)) {
+				GZipInputStream stream;
+				try {
+					stream = new GZipInputStream (
+						new FileStream (path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite));
+				} catch (FileNotFoundException ex) {
+					return null;
+				}
+				//Console.WriteLine("Getting GZipReader for " + path);
+				reader = new StreamReader (stream);
+			}
+			else {
+				FileStream stream;
+				try {
+					stream = new FileStream (path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
+				} catch (FileNotFoundException ex) {
+					return null;
+				}
+				//Console.WriteLine("Getting FileReader for " + path);
+				reader = new StreamReader (stream);
 			}
 			
-			StreamReader reader;
-			reader = new StreamReader (stream);
+			
 			return reader;
 		}
 
 		public void Delete (Uri uri)
 		{
 			lock (connection) {
 				string path = LookupPathRawUnlocked (uri, false);
 				if (path != null) {
 					MaybeStartTransaction_Unlocked ();
 					DoNonQuery ("DELETE FROM uri_index WHERE uri='{0}' AND filename='{1}'", 
 					            UriToString (uri), path);
 					if (path != SELF_CACHE_TAG)
Index: ./beagled/FileSystemQueryable/FileSystemQueryable.cs
===================================================================
RCS file: /cvs/gnome/beagle/beagled/FileSystemQueryable/FileSystemQueryable.cs,v
retrieving revision 1.106
diff -u -1 -2 -r1.106 FileSystemQueryable.cs
--- ./beagled/FileSystemQueryable/FileSystemQueryable.cs	29 Apr 2006 15:44:25 -0000	1.106
+++ ./beagled/FileSystemQueryable/FileSystemQueryable.cs	6 May 2006 04:05:27 -0000
@@ -1374,36 +1374,36 @@
 			if (filter.Ignore (parent, Path.GetFileName (path), is_directory))
 				return false;
 
 			return true;
 		}
 
 		override public string GetSnippet (string [] query_terms, Hit hit)
 		{
 			// Uri remapping from a hit is easy: the internal uri
 			// is stored in a property.
 			Uri uri;
 			uri = UriFu.UriStringToUri (hit ["beagle:InternalUri"]);
-
 			string path;
 			path = TextCache.UserCache.LookupPathRaw (uri);
 
 			if (path == null)
 				return null;
 
 			// If this is self-cached, use the remapped Uri
 			if (path == TextCache.SELF_CACHE_TAG)
-				path = hit.Uri.LocalPath;
+				return SnippetFu.GetSnippetFromFile (query_terms, hit.Uri.LocalPath);
 
-			return SnippetFu.GetSnippetFromFile (query_terms, path);
+			
+			return SnippetFu.GetSnippet (query_terms, TextCache.UserCache.GetReader(uri));
 		}
 
 		override public void Start ()
 		{
 			base.Start ();
 			
 			event_backend.Start (this);
 
 			LoadConfiguration ();
 
 			Logger.Log.Debug ("Done starting FileSystemQueryable");
 		}