beagle r4608 - in trunk/beagle/beagled: . FileSystemQueryable

[dbera svn gnome org]

Author: dbera
Date: Thu Mar 13 02:56:55 2008
New Revision: 4608
URL: http://svn.gnome.org/viewvc/beagle?rev=4608&view=rev

Log:
* Use FieldSelector in LuceneBitArray and LuceneNameResolver. If
nothing, at least a bunch of useless allocations will be saved. Also FSQ
startup should be faster.
* Add a debugging hook in beagled with SIGUSR2 (similar to the one in
IndexHelper). Sending a SIGUSR2 to beagled should print some internal
state information. Currently FSQ is the only one using it. The state
information should point out if we are leaking anything (e.g. the
directory counts should match when you add/remove directories).
* FSQ was not dropping the fileattributes for directories when they were
removed. Would not have caused any trouble but still its good to delete
the fastore information once the directory is removed.


Modified:
   trunk/beagle/beagled/BeagleDaemon.cs
   trunk/beagle/beagled/FileSystemQueryable/FileCrawlTask.cs
   trunk/beagle/beagled/FileSystemQueryable/FileSystemQueryable.cs
   trunk/beagle/beagled/FileSystemQueryable/InternalUriManager.cs
   trunk/beagle/beagled/FileSystemQueryable/TreeCrawlTask.cs
   trunk/beagle/beagled/LuceneBitArray.cs
   trunk/beagle/beagled/LuceneQueryable.cs
   trunk/beagle/beagled/QueryDriver.cs

Modified: trunk/beagle/beagled/BeagleDaemon.cs
==============================================================================
--- trunk/beagle/beagled/BeagleDaemon.cs	(original)
+++ trunk/beagle/beagled/BeagleDaemon.cs	Thu Mar 13 02:56:55 2008
@@ -583,7 +583,8 @@
 				return;
 
 			// Set shutdown flag to true so that other threads can stop initializing
-			if ((Mono.Unix.Native.Signum) signal != Mono.Unix.Native.Signum.SIGUSR1)
+			if ((Mono.Unix.Native.Signum) signal != Mono.Unix.Native.Signum.SIGUSR1 &&
+			    (Mono.Unix.Native.Signum) signal != Mono.Unix.Native.Signum.SIGUSR2)
 				Shutdown.ShutdownRequested = true;
 
 			// Do all signal handling work in the main loop and not in the signal handler.
@@ -594,13 +595,6 @@
 		{
 			Logger.Log.Debug ("Handling signal {0} ({1})", signal, (Mono.Unix.Native.Signum) signal);
 
-			// If we get SIGUSR1, turn the debugging level up.
-			if ((Mono.Unix.Native.Signum) signal == Mono.Unix.Native.Signum.SIGUSR1) {
-				LogLevel old_level = Log.Level;
-				Log.Level = LogLevel.Debug;
-				Log.Debug ("Moving from log level {0} to Debug", old_level);
-			}
-
 			// Pass the signals to the helper too.
 			GLib.Idle.Add (new GLib.IdleHandler (delegate ()
 							    {
@@ -608,8 +602,15 @@
 								    return false; 
 							    }));
 
-			if ((Mono.Unix.Native.Signum) signal == Mono.Unix.Native.Signum.SIGUSR1 ||
-			    (Mono.Unix.Native.Signum) signal == Mono.Unix.Native.Signum.SIGUSR2) {
+			// If we get SIGUSR1, turn the debugging level up.
+			if ((Mono.Unix.Native.Signum) signal == Mono.Unix.Native.Signum.SIGUSR1) {
+				LogLevel old_level = Log.Level;
+				Log.Level = LogLevel.Debug;
+				Log.Debug ("Moving from log level {0} to Debug", old_level);
+				return;
+			} else if ((Mono.Unix.Native.Signum) signal == Mono.Unix.Native.Signum.SIGUSR2) {
+				// Debugging hook for beagled
+				QueryDriver.DebugHook ();
 				return;
 			}
 

Modified: trunk/beagle/beagled/FileSystemQueryable/FileCrawlTask.cs
==============================================================================
--- trunk/beagle/beagled/FileSystemQueryable/FileCrawlTask.cs	(original)
+++ trunk/beagle/beagled/FileSystemQueryable/FileCrawlTask.cs	Thu Mar 13 02:56:55 2008
@@ -135,5 +135,14 @@
 
 			Reschedule = true;
 		}
+
+		internal void DebugHook ()
+		{
+			lock (big_lock)
+				Log.Debug ("FSQ:FileCrawlTask Debughook: Crawling {0} (generator is {1}) and is {2}active",
+					(current_dir != null ? current_dir.FullName : "nothing"),
+					(current_generator != null ? "busy" : "empty"),
+					(is_active ? String.Empty : "in"));
+		}
 	}
 }

Modified: trunk/beagle/beagled/FileSystemQueryable/FileSystemQueryable.cs
==============================================================================
--- trunk/beagle/beagled/FileSystemQueryable/FileSystemQueryable.cs	(original)
+++ trunk/beagle/beagled/FileSystemQueryable/FileSystemQueryable.cs	Thu Mar 13 02:56:55 2008
@@ -464,6 +464,8 @@
 
 			lock (dir_models_by_path)
 				dir_models_by_path.Remove (expired_path);
+
+			FileAttributesStore.Drop (expired_path);
 		}
 
 		public void AddDirectory (DirectoryModel parent, string name)
@@ -1027,6 +1029,14 @@
 						self_task = null;
 				}
 			}
+
+			internal void DebugHook ()
+			{
+				lock (queue_lock)
+					Log.Debug ("FSQ:FileSystemEventsGenerator Debughook: {0} more events to process, generator is {1}active",
+						event_queue.Count,
+						(self_task != null ? String.Empty : "in"));
+			}
 		}
 
 		//////////////////////////////////////////////////////////////////////////
@@ -2014,6 +2024,45 @@
 
 			return internal_uri;
 		}
+
+		//////////////////////////
+
+		internal override void DebugHook ()
+		{
+			Log.Debug ("FSQ DebugHook: {0} roots, {1} roots by path, {2} dir_models_by_id, {3} name_info_by_id, {4} dir_models_by_path",
+				(roots != null ? roots.Count : -1),
+				(roots_by_path != null ? roots_by_path.Count : -1),
+				(dir_models_by_id != null ? dir_models_by_id.Count : -1),
+				(name_info_by_id != null ? name_info_by_id.Count : -1),
+				(dir_models_by_path != null ? dir_models_by_path.Count : -1));
+
+			Log.Debug ("FSQ Debughook: {0} external_pending_indexables",
+				(external_pending_indexables != null ? external_pending_indexables.Count : -1));
+
+			lock (big_lock) {
+				foreach (DirectoryModel root in roots) {
+					int count = 1;
+					ArrayList queue = new ArrayList ();
+					queue.Add (root);
+					DirectoryModel child;
+					while (queue.Count > 0) {
+						child = (DirectoryModel) queue [0];
+						queue.RemoveAt (0);
+
+						if (child.ChildCount == 0)
+							continue;
+						count += child.ChildCount;
+						queue.AddRange (child.Children);
+					}
+					Console.WriteLine ("FSQ Debughook: {0} has {1} subdirs", root.FullName, count);
+				}
+			}
+
+			tree_crawl_task.DebugHook ();
+			file_crawl_task.DebugHook ();
+			fs_event_generator.DebugHook ();
+			uid_manager.DebugHook ();
+		}
 	}
 }
 	

Modified: trunk/beagle/beagled/FileSystemQueryable/InternalUriManager.cs
==============================================================================
--- trunk/beagle/beagled/FileSystemQueryable/InternalUriManager.cs	(original)
+++ trunk/beagle/beagled/FileSystemQueryable/InternalUriManager.cs	Thu Mar 13 02:56:55 2008
@@ -61,6 +61,12 @@
 
 		////////////////////////////////////////////////////////////////
 
+		// Ouch! Hardcoding prefix is not good
+		static FieldSelector fields_nameinfo = new MapFieldSelector (new string[] {"Uri",
+											   "prop:k:" + Property.ExactFilenamePropKey,
+											   "prop:k:" + Property.ParentDirUriPropKey,
+											   "prop:k:" + Property.IsDirectoryPropKey
+										});
 
 		private NameInfo DocumentToNameInfo (Document doc)
 		{
@@ -132,7 +138,7 @@
 
 			if (match_id != -1) {
 				Document doc;
-				doc = reader.Document (match_id);
+				doc = reader.Document (match_id, fields_nameinfo);
 				info = DocumentToNameInfo (doc);
 			}
 
@@ -269,7 +275,7 @@
 					break;
 
 				Document doc;
-				doc = searcher.Doc (i);
+				doc = searcher.Doc (i, fields_nameinfo);
 
 				NameInfo info;
 				info = DocumentToNameInfo (doc);
@@ -384,6 +390,10 @@
 			return GetAllDirectoryNameInfo (null);
 		}
 
+		internal void DebugHook ()
+		{
+			Log.Debug ("FSQ:InternalUriManager Debughook: {0} cached_uid_by_path", cached_uid_by_path.Count);
+		}
 	}
 }
 

Modified: trunk/beagle/beagled/FileSystemQueryable/TreeCrawlTask.cs
==============================================================================
--- trunk/beagle/beagled/FileSystemQueryable/TreeCrawlTask.cs	(original)
+++ trunk/beagle/beagled/FileSystemQueryable/TreeCrawlTask.cs	Thu Mar 13 02:56:55 2008
@@ -129,5 +129,10 @@
 			SetIsActive (false, null);
 		}
 
+		internal void DebugHook ()
+		{
+			lock (big_lock)
+				Log.Debug ("FSQ:TreeCrawlTask Debughook: {0} directories left to crawl, current {1}active", to_be_crawled.Count, is_active ? String.Empty : "in");
+		}
 	}
 }

Modified: trunk/beagle/beagled/LuceneBitArray.cs
==============================================================================
--- trunk/beagle/beagled/LuceneBitArray.cs	(original)
+++ trunk/beagle/beagled/LuceneBitArray.cs	Thu Mar 13 02:56:55 2008
@@ -198,6 +198,8 @@
 
 		////////////////////////////////////////////////////////////
 
+		static FieldSelector fields_uri = new MapFieldSelector (new string[] {"Uri"});
+
 		public void ProjectOnto (LuceneBitArray other)
 		{
 			int j = 0;
@@ -209,7 +211,7 @@
 				j = i+1;
 
 				Document doc;
-				doc = searcher.Doc (i);
+				doc = searcher.Doc (i, fields_uri);
 
 				other.AddUri (doc.Get ("Uri"));
 			}

Modified: trunk/beagle/beagled/LuceneQueryable.cs
==============================================================================
--- trunk/beagle/beagled/LuceneQueryable.cs	(original)
+++ trunk/beagle/beagled/LuceneQueryable.cs	Thu Mar 13 02:56:55 2008
@@ -1054,5 +1054,11 @@
 		{
 			return new LuceneQueryingDriver (index_name, minor_version, read_only_mode);
 		}
+
+		//////////////////////////////////////////////////////////////////////////////////
+
+		virtual internal void DebugHook ()
+		{
+		}
 	}
 }

Modified: trunk/beagle/beagled/QueryDriver.cs
==============================================================================
--- trunk/beagle/beagled/QueryDriver.cs	(original)
+++ trunk/beagle/beagled/QueryDriver.cs	Thu Mar 13 02:56:55 2008
@@ -36,7 +36,7 @@
 
 namespace Beagle.Daemon {
 	
-	public class QueryDriver {
+	public static class QueryDriver {
 
 		// Contains list of queryables explicitly asked by --allow-backend or --backend name
 		// --allow-backend/--backend name : dont read config information and only start backend 'name'
@@ -677,5 +677,25 @@
 				return false;
 			}
 		}
+
+		/////////////////////////////////////////////////////////
+
+		// Various debug hooks; especially persistant state information and memory issues
+
+		public static void DebugHook ()
+		{
+			Log.Debug ("Debughook called:");
+			LuceneQueryable l;
+			foreach (Queryable q in Queryables) {
+				l = q.IQueryable as LuceneQueryable;
+				if (l == null)
+					continue;
+				try {
+					l.DebugHook ();
+				} catch (Exception e) {
+					Log.Debug (e, "Exception during Debughook for {0}", q.Name);
+				}
+			}
+		}
 	}
 }