beagle r4487 - in trunk/beagle: BeagleClient beagled beagled/NetworkServicesQueryable

[dbera svn gnome org]

Author: dbera
Date: Fri Feb 15 00:24:30 2008
New Revision: 4487
URL: http://svn.gnome.org/viewvc/beagle?rev=4487&view=rev

Log:
Add API to quickly get the number of matches for a query. Can be used to provide google-suggest like suggestions, or generate interesting statistics (e.g. histograms).


Modified:
   trunk/beagle/BeagleClient/AssemblyInfo.cs
   trunk/beagle/BeagleClient/Query.cs
   trunk/beagle/BeagleClient/QueryResponses.cs
   trunk/beagle/beagled/AssemblyInfo.cs
   trunk/beagle/beagled/ExternalMetadataQueryable.cs
   trunk/beagle/beagled/IQueryable.cs
   trunk/beagle/beagled/LuceneQueryable.cs
   trunk/beagle/beagled/LuceneQueryingDriver.cs
   trunk/beagle/beagled/NetworkServicesQueryable/NetworkServicesQueryable.cs
   trunk/beagle/beagled/QueryDriver.cs
   trunk/beagle/beagled/QueryExecutor.cs
   trunk/beagle/beagled/Queryable.cs

Modified: trunk/beagle/BeagleClient/AssemblyInfo.cs
==============================================================================
--- trunk/beagle/BeagleClient/AssemblyInfo.cs	(original)
+++ trunk/beagle/BeagleClient/AssemblyInfo.cs	Fri Feb 15 00:24:30 2008
@@ -30,13 +30,14 @@
 
 // Any request message types in the Beagle.dll file must be registered here.
 [assembly: RequestMessageTypes (
+	 typeof (CountMatchQuery),
+	 typeof (DaemonInformationRequest),
 	 typeof (IndexingServiceRequest),
 	 typeof (InformationalMessagesRequest),
+	 typeof (OptimizeIndexesRequest),
 	 typeof (Query),
-	 typeof (DaemonInformationRequest),
-	 typeof (ShutdownRequest),
 	 typeof (ReloadConfigRequest),
-	 typeof (OptimizeIndexesRequest),
+	 typeof (ShutdownRequest),
 	 typeof (SnippetRequest)
 )]
 	 
@@ -49,5 +50,6 @@
 	 typeof (IndexingStatusResponse),
 	 typeof (SearchTermResponse),
 	 typeof (DaemonInformationResponse),
-	 typeof (SnippetResponse)
+	 typeof (SnippetResponse),
+	 typeof (CountMatchQueryResponse)
 )]

Modified: trunk/beagle/BeagleClient/Query.cs
==============================================================================
--- trunk/beagle/BeagleClient/Query.cs	(original)
+++ trunk/beagle/BeagleClient/Query.cs	Fri Feb 15 00:24:30 2008
@@ -244,4 +244,25 @@
 			return sb.ToString ();
 		}
 	}
+
+	// Synchronous query to return the number of matches
+	public class CountMatchQuery : Query {
+
+		public CountMatchQuery (string str) : this ()
+		{
+			AddText (str);
+		}
+
+		public CountMatchQuery ()
+		{
+			// RDFQuery is a sync message
+			this.UnregisterAsyncResponseHandler (typeof (HitsAddedResponse));
+			this.UnregisterAsyncResponseHandler (typeof (HitsSubtractedResponse));
+			this.UnregisterAsyncResponseHandler (typeof (FinishedResponse));
+			this.UnregisterAsyncResponseHandler (typeof (ErrorResponse));
+			this.UnregisterAsyncResponseHandler (typeof (SearchTermResponse));
+
+			Keepalive = false;
+		}
+	}
 }

Modified: trunk/beagle/BeagleClient/QueryResponses.cs
==============================================================================
--- trunk/beagle/BeagleClient/QueryResponses.cs	(original)
+++ trunk/beagle/BeagleClient/QueryResponses.cs	Fri Feb 15 00:24:30 2008
@@ -114,4 +114,8 @@
 			StemmedText = new ArrayList ();
 		}
 	}
+
+	public class CountMatchQueryResponse : ResponseMessage {
+		public int NumMatches = 0;
+	}
 }

Modified: trunk/beagle/beagled/AssemblyInfo.cs
==============================================================================
--- trunk/beagle/beagled/AssemblyInfo.cs	(original)
+++ trunk/beagle/beagled/AssemblyInfo.cs	Fri Feb 15 00:24:30 2008
@@ -51,7 +51,8 @@
 	typeof (QueryExecutor),
 	typeof (ReloadConfigExecutor),
 	typeof (ShutdownExecutor),
-	typeof (SnippetExecutor)
+	typeof (SnippetExecutor),
+	typeof (CountMatchQueryExecutor)
 )]
 
 // All backends in this assembly must be registered here.
@@ -76,5 +77,5 @@
 	typeof (Beagle.Daemon.PidginQueryable.PidginQueryable),
 	typeof (Beagle.Daemon.StaticQueryable),
 	typeof (Beagle.Daemon.TomboyQueryable.TomboyQueryable),
-    typeof (Beagle.Daemon.EmpathyQueryable.EmpathyQueryable)                   
+	typeof (Beagle.Daemon.EmpathyQueryable.EmpathyQueryable)                   
 )]

Modified: trunk/beagle/beagled/ExternalMetadataQueryable.cs
==============================================================================
--- trunk/beagle/beagled/ExternalMetadataQueryable.cs	(original)
+++ trunk/beagle/beagled/ExternalMetadataQueryable.cs	Fri Feb 15 00:24:30 2008
@@ -64,6 +64,11 @@
 		{
 		}
 
+		public int DoCountMatchQuery (Query query)
+		{
+			return 0;
+		}
+
 		public ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text)
 		{
 			return null;

Modified: trunk/beagle/beagled/IQueryable.cs
==============================================================================
--- trunk/beagle/beagled/IQueryable.cs	(original)
+++ trunk/beagle/beagled/IQueryable.cs	Fri Feb 15 00:24:30 2008
@@ -43,6 +43,9 @@
 			      IQueryResult result,
 			      IQueryableChangeData data);
 
+		// Just return the number of matches
+		int DoCountMatchQuery (Query query);
+
 		ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text);
 
 		QueryableStatus GetQueryableStatus ();

Modified: trunk/beagle/beagled/LuceneQueryable.cs
==============================================================================
--- trunk/beagle/beagled/LuceneQueryable.cs	(original)
+++ trunk/beagle/beagled/LuceneQueryable.cs	Fri Feb 15 00:24:30 2008
@@ -355,6 +355,12 @@
 					our_hit_filter);
 		}
 
+		public int DoCountMatchQuery (Query query)
+		{
+			return Driver.DoCountMatchQuery (query, backend_query_part_hook);
+		}
+
+
 		/////////////////////////////////////////
 
 		protected SnippetReader GetSnippetFromTextCache (string [] query_terms, Uri uri, bool full_text)

Modified: trunk/beagle/beagled/LuceneQueryingDriver.cs
==============================================================================
--- trunk/beagle/beagled/LuceneQueryingDriver.cs	(original)
+++ trunk/beagle/beagled/LuceneQueryingDriver.cs	Fri Feb 15 00:24:30 2008
@@ -315,6 +315,88 @@
 
 		////////////////////////////////////////////////////////////////
 
+		public int DoCountMatchQuery (Query query, QueryPartHook query_part_hook)
+		{
+			if (Debug)
+				Logger.Log.Debug ("###### {0}: Starting low-level queries", IndexName);
+
+			Stopwatch total;
+			total = new Stopwatch ();
+			total.Start ();
+
+			ArrayList primary_required_part_queries;
+			ArrayList secondary_required_part_queries;
+
+			LNS.BooleanQuery primary_prohibited_part_query;
+			LNS.BooleanQuery secondary_prohibited_part_query;
+
+			AndHitFilter all_hit_filters;
+
+			ArrayList term_list;
+			term_list = AssembleQuery ( query,
+						    query_part_hook,
+						    null,
+						    out primary_required_part_queries,
+						    out secondary_required_part_queries,
+						    out primary_prohibited_part_query,
+						    out secondary_prohibited_part_query,
+						    out all_hit_filters);
+
+			// If we have no required parts, give up.
+			if (primary_required_part_queries == null)
+				return 0;
+
+			IndexReader primary_reader;
+			LNS.IndexSearcher primary_searcher;
+			IndexReader secondary_reader;
+			LNS.IndexSearcher secondary_searcher;
+
+			BuildSearchers (out primary_reader, out primary_searcher, out secondary_reader, out secondary_searcher);
+
+			// Build whitelists and blacklists for search subsets.
+			LuceneBitArray primary_whitelist, secondary_whitelist;
+			CreateQueryWhitelists (null,
+				primary_searcher,
+				secondary_searcher,
+				primary_prohibited_part_query,
+				secondary_prohibited_part_query,
+				out primary_whitelist,
+				out secondary_whitelist);
+
+			// Now run the low level queries against our indexes.
+			BetterBitArray primary_matches = null;
+			if (primary_required_part_queries != null) {
+
+				if (secondary_searcher != null)
+					primary_matches = DoRequiredQueries_TwoIndex (primary_searcher,
+										      secondary_searcher,
+										      primary_required_part_queries,
+										      secondary_required_part_queries,
+										      primary_whitelist,
+										      secondary_whitelist);
+				else
+					primary_matches = DoRequiredQueries (primary_searcher,
+									     primary_required_part_queries,
+									     primary_whitelist);
+
+			} 
+
+			int result = 0;
+			// FIXME: Pass the count through uri-filter and other validation checks
+			if (primary_matches != null)
+				result = primary_matches.TrueCount;
+
+			CloseSearchers (primary_reader, primary_searcher, secondary_reader, secondary_searcher);
+
+			total.Stop ();
+			if (Debug)
+				Logger.Log.Debug ("###### {0}: Total query run in {1}", IndexName, total);
+
+			return result;
+		}
+
+		////////////////////////////////////////////////////////////////
+
 		public void DoQuery (Query               query,
 				     IQueryResult        result,
 				     ICollection         search_subset_uris, // should be internal uris

Modified: trunk/beagle/beagled/NetworkServicesQueryable/NetworkServicesQueryable.cs
==============================================================================
--- trunk/beagle/beagled/NetworkServicesQueryable/NetworkServicesQueryable.cs	(original)
+++ trunk/beagle/beagled/NetworkServicesQueryable/NetworkServicesQueryable.cs	Fri Feb 15 00:24:30 2008
@@ -125,6 +125,11 @@
 			return;
 		}
 
+		public int DoCountMatchQuery (Query query)
+		{
+			return 0;
+		}
+
 		public ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text)
 		{
 			return null;

Modified: trunk/beagle/beagled/QueryDriver.cs
==============================================================================
--- trunk/beagle/beagled/QueryDriver.cs	(original)
+++ trunk/beagle/beagled/QueryDriver.cs	Fri Feb 15 00:24:30 2008
@@ -630,6 +630,24 @@
 
 		////////////////////////////////////////////////////////
 
+		static public int DoCountMatchQuery (CountMatchQuery query)
+		{
+			DehumanizeQuery (query);
+
+			int num_matches = 0;
+
+			foreach (Queryable q in Queryables) {
+				if (! q.AcceptQuery (query))
+					continue;
+
+				num_matches += q.DoCountMatchQuery (query);
+			}
+
+			return num_matches;
+		}
+
+		////////////////////////////////////////////////////////
+
 		static public IEnumerable GetIndexInformation ()
 		{
 			foreach (Queryable q in Queryables)

Modified: trunk/beagle/beagled/QueryExecutor.cs
==============================================================================
--- trunk/beagle/beagled/QueryExecutor.cs	(original)
+++ trunk/beagle/beagled/QueryExecutor.cs	Fri Feb 15 00:24:30 2008
@@ -109,4 +109,19 @@
 			this.query = null;
 		}
 	}
+
+	[RequestMessage (typeof (CountMatchQuery))]
+	public class CountMatchQueryExecutor : RequestMessageExecutor {
+
+		public override ResponseMessage Execute (RequestMessage request)
+		{
+			CountMatchQueryResponse result = new CountMatchQueryResponse ();
+			CountMatchQuery query = request as CountMatchQuery;
+			if (query == null)
+				return new ErrorResponse ("Only CountMatch query please!");
+
+			result.NumMatches = QueryDriver.DoCountMatchQuery (query);
+			return result;
+		}
+	}
 }

Modified: trunk/beagle/beagled/Queryable.cs
==============================================================================
--- trunk/beagle/beagled/Queryable.cs	(original)
+++ trunk/beagle/beagled/Queryable.cs	Fri Feb 15 00:24:30 2008
@@ -83,6 +83,16 @@
 			}
 		}
 
+		public int DoCountMatchQuery (Query query)
+		{
+			try {
+				return iqueryable.DoCountMatchQuery (query);
+			} catch (Exception ex) {
+				Logger.Log.Warn (ex, "Caught exception calling DoCountMatchQuery on '{0}'", Name);
+				return 0;
+			}
+		}
+
 		public ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text)
 		{
 			if (hit == null)