beagle r4775 - in trunk/beagle: BeagleClient beagled beagled/NetworkServicesQueryable

[dbera svn gnome org]

Author: dbera
Date: Mon Jun  2 23:15:10 2008
New Revision: 4775
URL: http://svn.gnome.org/viewvc/beagle?rev=4775&view=rev

Log:
Flush some pending changes with respect to snippets from remote beagled. But nevertheless, this whole HTTP as a substitute of UnixSocket is the gateway to disaster. The remote server sometimes just stops responding to incoming HTTP requests and even though the sender has a timeout set, the timeout never happens. Basically, the whole querying over HTTP is not gonna work. Still, retrieving snippets was the last missing piece, so the network backend is now done.


Modified:
   trunk/beagle/BeagleClient/Snippet.cs
   trunk/beagle/beagled/NetworkServicesQueryable/HttpTransport.cs
   trunk/beagle/beagled/NetworkServicesQueryable/NetworkServicesQueryable.cs
   trunk/beagle/beagled/Server.cs

Modified: trunk/beagle/BeagleClient/Snippet.cs
==============================================================================
--- trunk/beagle/BeagleClient/Snippet.cs	(original)
+++ trunk/beagle/BeagleClient/Snippet.cs	Mon Jun  2 23:15:10 2008
@@ -274,20 +274,7 @@
 			if (snippet_reader == null)
 				return;
 
-			// If fulltext is false, read lines from snippet reader
-			if (! FullText) {
- 				foreach (SnippetLine snippet_line in snippet_reader.GetSnippet ()) {
-					writer.WriteStartElement ("SnippetLine");
-					writer.WriteAttributeString ("Line", Convert.ToString (snippet_line.Line));
-					foreach (Fragment fragment in snippet_line.Fragments) {
-						writer.WriteStartElement ("Fragment");
-						writer.WriteAttributeString ("QueryTermIndex", Convert.ToString (fragment.QueryTermIndex));
-						writer.WriteString (fragment.Text);
-						writer.WriteEndElement ();
-					}
-					writer.WriteEndElement ();
-				}
-			} else {
+			if (FullText) {
 				writer.WriteStartElement ("SnippetLine");
 				writer.WriteAttributeString ("Line", "1");
 				writer.WriteStartElement ("Fragment");
@@ -302,6 +289,19 @@
 
 				writer.WriteEndElement ();
 				writer.WriteEndElement ();
+			} else if (snippet_reader.GetSnippet() != null) {
+			// If fulltext is false, read lines from snippet reader
+ 				foreach (SnippetLine snippet_line in snippet_reader.GetSnippet ()) {
+					writer.WriteStartElement ("SnippetLine");
+					writer.WriteAttributeString ("Line", Convert.ToString (snippet_line.Line));
+					foreach (Fragment fragment in snippet_line.Fragments) {
+						writer.WriteStartElement ("Fragment");
+						writer.WriteAttributeString ("QueryTermIndex", Convert.ToString (fragment.QueryTermIndex));
+						writer.WriteString (fragment.Text);
+						writer.WriteEndElement ();
+					}
+					writer.WriteEndElement ();
+				}
 			}
 			snippet_reader.Close ();
 		}

Modified: trunk/beagle/beagled/NetworkServicesQueryable/HttpTransport.cs
==============================================================================
--- trunk/beagle/beagled/NetworkServicesQueryable/HttpTransport.cs	(original)
+++ trunk/beagle/beagled/NetworkServicesQueryable/HttpTransport.cs	Mon Jun  2 23:15:10 2008
@@ -61,16 +61,18 @@
 			http_request.KeepAlive = true;
 			http_request.AllowWriteStreamBuffering = false;
 			http_request.SendChunked = true;
+			http_request.Timeout = 5*1000; // 5 sec
+			http_request.ReadWriteTimeout = 5*1000; // 5 sec
 			
 			Stream stream = http_request.GetRequestStream ();			
 			base.SendRequest (request, stream);
-			
-			stream.Flush ();
+
+			// FIXME: Should we close the stream ?
 			stream.Close ();
 				
 			Logger.Log.Debug ("HttpClient: Sent request");
 		}
-		
+
 		public override ResponseMessage Send (RequestMessage request)
 		{
 			if (request.Keepalive)

Modified: trunk/beagle/beagled/NetworkServicesQueryable/NetworkServicesQueryable.cs
==============================================================================
--- trunk/beagle/beagled/NetworkServicesQueryable/NetworkServicesQueryable.cs	(original)
+++ trunk/beagle/beagled/NetworkServicesQueryable/NetworkServicesQueryable.cs	Mon Jun  2 23:15:10 2008
@@ -88,6 +88,7 @@
 			query.HitsAddedEvent -= hits_added_handler;
 			query.HitsSubtractedEvent -= hits_subtracted_handler;
 			query.FinishedEvent -= finished_handler;
+			query.Transports.Clear ();
 
 			if (throw_me != null)
 				throw throw_me;
@@ -102,7 +103,54 @@
 
 		public ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text, int ctx_length, int snp_length)
 		{
-			return null;
+			string source = hit ["beagle:Source"];
+			hit ["beagle:Source"] = hit ["beagle:OrigSource"];
+
+			string network_node = hit ["beagle:NetworkNode"];
+			SnippetReader snippet_reader = null;
+
+			// FIXME: Creating a snippet request, registering transports, all of this
+			// doing everytime for hundreds of hits may become quite expensive.
+			// In that case, pre generate one snippetrequest and use it over and over.
+
+			// Form a correct snippet request
+			SnippetRequest sreq = new SnippetRequest ();
+			sreq.Hit = hit;
+			sreq.QueryTerms = query_terms;
+			sreq.FullText = full_text;
+			sreq.ContextLength = ctx_length;
+			sreq.SnippetLength = snp_length;
+
+			// fake a blocking snippet retrieval
+			sreq.RegisterAsyncResponseHandler (typeof (SnippetResponse),
+							   delegate (ResponseMessage response) {
+				if (response is ErrorResponse) {
+					Log.Error ("Error retrieval snippet for {0} from network node {1}", hit.Uri, network_node);
+					return;
+				}
+
+				snippet_reader = new SnippetReader ((SnippetResponse) response);
+			});
+
+			List<string[]> network_services = Conf.Networking.GetListOptionValues (Conf.Names.NetworkServices);
+			foreach (string[] service in network_services) {
+				if (network_node != service [0])
+					continue;
+
+				sreq.Transports.Clear ();
+				sreq.RegisterTransport (new HttpTransport (service [1]));
+
+				// fake a blocking snippet retrieval
+				try {
+					sreq.SendAsyncBlocking ();
+				} catch (Exception e) {
+					Log.Debug (e, "Error while requesting snippet from {0} for {1}", service [1], hit.Uri);
+				}
+				break;
+			}
+
+			hit ["beagle:Source"] = source; // reset source
+			return snippet_reader;
 		}
 
 		public QueryableStatus GetQueryableStatus ()
@@ -110,5 +158,49 @@
 			QueryableStatus status = new QueryableStatus ();
 			return status;
 		}
+
+		private class SnippetReader : ISnippetReader {
+			private SnippetResponse response;
+			public SnippetReader (SnippetResponse response)
+			{
+				this.response = response;
+			}
+
+			public IEnumerable GetSnippet ()
+			{
+				return response.SnippetList.Snippets;
+			}
+
+			private System.IO.StringReader fulltext_reader = null;
+
+			private bool ReadLineInit ()
+			{
+				if (response.SnippetList.Snippets == null)
+					return false;
+
+				foreach (SnippetLine line in response.SnippetList.Snippets) {
+					if (line.Fragments == null || line.Fragments.Count == 0)
+						return false;
+
+					Fragment fragment = (Fragment) line.Fragments [0];
+					fulltext_reader = new System.IO.StringReader (fragment.Text);
+					return true;
+				}
+
+				return false; // make gmcs happy
+			}
+
+			public string ReadLine ()
+			{
+				if (fulltext_reader == null && ! ReadLineInit ())
+					return null;
+
+				return fulltext_reader.ReadLine ();
+			}
+
+			public void Close ()
+			{
+			}
+		}
 	}
 }

Modified: trunk/beagle/beagled/Server.cs
==============================================================================
--- trunk/beagle/beagled/Server.cs	(original)
+++ trunk/beagle/beagled/Server.cs	Mon Jun  2 23:15:10 2008
@@ -61,12 +61,15 @@
 		{
 			Hit requested = (Hit) items [path];
 
-			if (requested == null)
+			if (requested == null || ! requested.Uri.IsFile) {
+				context.Response.StatusCode = 404;
+				context.Response.Close ();
 				return;
+			}
 
 			//Logger.Log.Debug ("httpitemhandler: requested: {0}", path);
 			context.Response.ContentType = requested.MimeType;
-			
+
 			// FIXME: We can only handle files for now
 			/*if (requested ["Type"] != "File") {
 			  Logger.Log.Debug ("httpitemhandler: can only serve files");
@@ -97,12 +100,16 @@
 		private HttpItemHandler item_handler = null;
 		private System.Timers.Timer keepalive_timer = null;
 		private System.Guid id = Guid.Empty;
+		private string network_node = null;
 
 		public HttpConnectionHandler (System.Guid guid, HttpListenerContext context, HttpItemHandler item_handler)
 		{
 			this.id = guid;
 			this.context = context;
 			this.item_handler = item_handler;
+
+			Config config = Conf.Get (Conf.Names.NetworkingConfig);
+			network_node = config.GetOption ("ServiceName", String.Empty);
 		}
 
 		internal System.Guid Guid {
@@ -174,8 +181,13 @@
 			
 			//Logger.Log.Debug ("HTTP Server: Handling received request message");
 			
-			buffer_stream.Seek (0, SeekOrigin.Begin);
-			base.HandleConnection (buffer_stream);
+                        // The 0xff bytes from a remote beagled comes in a separate http request
+                        // and causes havoc by creating empty messages. HTTP streams do not behave
+                        // like a continous stream like UnixStream
+                        if (total_bytes > 0) {
+				buffer_stream.Seek (0, SeekOrigin.Begin);
+				base.HandleConnection (buffer_stream);
+			}
 			
 			Server.MarkHandlerAsKilled (this);
 			Shutdown.WorkerFinished (context.Request.InputStream);
@@ -253,17 +265,30 @@
 			if (response == null)
 				return;
 
+			// Change the Hit source and add a property with the name of this remote node
 			foreach (Hit hit in response.Hits) {
-				hit.Uri = new System.Uri (context.Request.Url.ToString () + id.ToString ());
-				hit ["beagle:Source"] = "Network";
+				//hit.Uri = new System.Uri (context.Request.Url.ToString () + id.ToString ());
+				IList new_props = new Property [2];
+				new_props [0] = Property.NewKeyword ("beagle:OrigSource", hit ["beagle:Source"]);
+				new_props [1] = Property.NewKeyword ("beagle:NetworkNode", network_node);
+				hit.AddProperty (new_props);
+
+                                // Need to replace the existing beagle:Source property
+                                foreach (Property prop in hit.Properties) {
+                                        if (prop.Key == "beagle:Source") {
+                                                prop.Value = "NetworkServices";
+                                                break;
+                                        }
+                                }
 
-				item_handler.RegisterHit (hit);
-			}	
+				//FIXME Retrieving the files for a remote hit is disabled for now
+				//item_handler.RegisterHit (hit);
+			}
 		}
 
 		public override bool SendResponse (ResponseMessage response)
-		{	
-			//TransformResponse (ref response); Disable for now
+		{
+			TransformResponse (ref response);
 
 			bool r = false;
 
@@ -870,7 +895,7 @@
 					Guid guid = Guid.NewGuid ();
 					HttpItemHandler item_handler = new HttpItemHandler ();
 					item_handlers [guid] = item_handler;
-					
+
 					ConnectionHandler handler = new HttpConnectionHandler (guid, context, item_handler);
 
 					lock (live_handlers)