beagle r4642 - in branches/beagle-rdf: . BeagleClient Filters Util beagled beagled/FileSystemQueryable beagled/IndexHelper beagled/KonversationQueryable beagled/KopeteQueryable beagled/NetworkServicesQueryable beagled/PidginQueryable bludgeon conf-data conf-data/crawl-rules po search search/Beagle.Search tools
- From: dbera svn gnome org
- To: svn-commits-list gnome org
- Subject: beagle r4642 - in branches/beagle-rdf: . BeagleClient Filters Util beagled beagled/FileSystemQueryable beagled/IndexHelper beagled/KonversationQueryable beagled/KopeteQueryable beagled/NetworkServicesQueryable beagled/PidginQueryable bludgeon conf-data conf-data/crawl-rules po search search/Beagle.Search tools
- Date: Mon, 24 Mar 2008 21:09:26 +0000 (GMT)
Author: dbera
Date: Mon Mar 24 21:09:25 2008
New Revision: 4642
URL: http://svn.gnome.org/viewvc/beagle?rev=4642&view=rev
Log:
Merge 4598:4641 from trunk.
Added:
branches/beagle-rdf/Filters/FilterInfo.cs
- copied unchanged from r4641, /trunk/beagle/Filters/FilterInfo.cs
Removed:
branches/beagle-rdf/bludgeon/
Modified:
branches/beagle-rdf/BeagleClient/Snippet.cs
branches/beagle-rdf/BeagleClient/Transport.cs
branches/beagle-rdf/BeagleClient/UnixTransport.cs
branches/beagle-rdf/ChangeLog
branches/beagle-rdf/Filters/AssemblyInfo.cs
branches/beagle-rdf/Filters/FilterTexi.cs
branches/beagle-rdf/Filters/Makefile.am
branches/beagle-rdf/Makefile.am
branches/beagle-rdf/NEWS
branches/beagle-rdf/Util/ExceptionHandlingThread.cs
branches/beagle-rdf/beagled/BeagleDaemon.cs
branches/beagle-rdf/beagled/BuildIndex.cs
branches/beagle-rdf/beagled/ExternalMetadataQueryable.cs
branches/beagle-rdf/beagled/FileSystemQueryable/FileCrawlTask.cs
branches/beagle-rdf/beagled/FileSystemQueryable/FileSystemQueryable.cs
branches/beagle-rdf/beagled/FileSystemQueryable/InternalUriManager.cs
branches/beagle-rdf/beagled/FileSystemQueryable/TreeCrawlTask.cs
branches/beagle-rdf/beagled/FilterFactory.cs
branches/beagle-rdf/beagled/IQueryable.cs
branches/beagle-rdf/beagled/IndexHelper/IndexHelper.cs
branches/beagle-rdf/beagled/KonversationQueryable/KonversationQueryable.cs
branches/beagle-rdf/beagled/KopeteQueryable/KopeteQueryable.cs
branches/beagle-rdf/beagled/LuceneBitArray.cs
branches/beagle-rdf/beagled/LuceneCommon.cs
branches/beagle-rdf/beagled/LuceneIndexingDriver.cs
branches/beagle-rdf/beagled/LuceneQueryable.cs
branches/beagle-rdf/beagled/NetworkServicesQueryable/NetworkServicesQueryable.cs
branches/beagle-rdf/beagled/PidginQueryable/PidginQueryable.cs
branches/beagle-rdf/beagled/QueryDriver.cs
branches/beagle-rdf/beagled/Queryable.cs
branches/beagle-rdf/beagled/Server.cs
branches/beagle-rdf/beagled/Shutdown.cs
branches/beagle-rdf/beagled/SnippetExecutor.cs
branches/beagle-rdf/beagled/SnippetFu.cs
branches/beagle-rdf/beagled/SnippetTest.cs
branches/beagle-rdf/beagled/StaticQueryable.cs
branches/beagle-rdf/conf-data/crawl-rules/crawl-manpages
branches/beagle-rdf/conf-data/query-mapping.xml
branches/beagle-rdf/configure.in
branches/beagle-rdf/po/ChangeLog
branches/beagle-rdf/po/hu.po
branches/beagle-rdf/search/Beagle.Search/Driver.cs
branches/beagle-rdf/search/Makefile.am
branches/beagle-rdf/tools/Query.cs
branches/beagle-rdf/tools/StaticQuery.cs
Modified: branches/beagle-rdf/BeagleClient/Snippet.cs
==============================================================================
--- branches/beagle-rdf/BeagleClient/Snippet.cs (original)
+++ branches/beagle-rdf/BeagleClient/Snippet.cs Mon Mar 24 21:09:25 2008
@@ -47,6 +47,10 @@
// should know that.
public bool FullText = false; // default, not fulltext
+ public int ContextLength = -1; // Use system default = 6 if not specified
+
+ public int SnippetLength = -1; // 200, if not specified
+
public SnippetRequest () : base (false) { }
public SnippetRequest (Query query, Hit hit) : base (false)
@@ -145,12 +149,15 @@
public void AddMatchFragment (int query_term_index, string text)
{
+ text = StringFu.CleanupInvalidXmlCharacters (text);
+ if (String.IsNullOrEmpty (text))
+ return;
+
if (Fragments == null)
Fragments = new ArrayList (3); // mostly will be 3 fragments
// Before we send a snippet over the wire, clean up any
// characters that would be invalid in XML.
- text = StringFu.CleanupInvalidXmlCharacters (text);
Fragments.Add (new Fragment (query_term_index, text));
}
Modified: branches/beagle-rdf/BeagleClient/Transport.cs
==============================================================================
--- branches/beagle-rdf/BeagleClient/Transport.cs (original)
+++ branches/beagle-rdf/BeagleClient/Transport.cs Mon Mar 24 21:09:25 2008
@@ -106,7 +106,6 @@
Logger.Log.Debug ("Received response:\n{0}\n", r.ReadToEnd ());
deserialize_stream.Seek (0, SeekOrigin.Begin);
#endif
-
ResponseWrapper wrapper = (ResponseWrapper)resp_serializer.Deserialize (deserialize_stream);
ResponseMessage response = wrapper.Message;
deserialize_stream.Close ();
Modified: branches/beagle-rdf/BeagleClient/UnixTransport.cs
==============================================================================
--- branches/beagle-rdf/BeagleClient/UnixTransport.cs (original)
+++ branches/beagle-rdf/BeagleClient/UnixTransport.cs Mon Mar 24 21:09:25 2008
@@ -87,6 +87,8 @@
protected override void SendRequest (RequestMessage request)
{
client = new UnixClient (this.socket_name);
+ client.SendBufferSize = 4096;
+ client.ReceiveBufferSize = 4096;
NetworkStream stream = client.GetStream ();
base.SendRequest (request, stream);
@@ -121,7 +123,7 @@
do {
// 0xff signifies end of message
- end_index = ArrayFu.IndexOfByte (this.network_data, (byte) 0xff, prev_index);
+ end_index = Array.IndexOf<byte> (this.network_data, (byte) 0xff, prev_index);
int bytes_count = (end_index == -1 ? bytes_read : end_index) - prev_index;
this.BufferStream.Write (this.network_data, prev_index, bytes_count);
Modified: branches/beagle-rdf/Filters/AssemblyInfo.cs
==============================================================================
--- branches/beagle-rdf/Filters/AssemblyInfo.cs (original)
+++ branches/beagle-rdf/Filters/AssemblyInfo.cs Mon Mar 24 21:09:25 2008
@@ -28,7 +28,7 @@
using Beagle.Filters;
-// All filter types have to be listed here to be loaded.
+// All (non-abstract) filter types have to be listed here to be loaded.
[assembly: Beagle.Daemon.FilterTypes (
typeof(FilterAbiWord),
typeof(FilterApplication),
@@ -55,6 +55,7 @@
typeof(FilterGif),
typeof(FilterHtml),
typeof(FilterIgnore),
+ typeof(FilterInfo),
typeof(FilterJava),
typeof(FilterJavascript),
typeof(FilterJpeg),
@@ -92,6 +93,7 @@
typeof(FilterSpreadsheet),
typeof(FilterSvg),
typeof(FilterTeX),
+ typeof(FilterTexi),
typeof(FilterText),
typeof(FilterTiff),
typeof(FilterTotem),
Modified: branches/beagle-rdf/Filters/FilterTexi.cs
==============================================================================
--- branches/beagle-rdf/Filters/FilterTexi.cs (original)
+++ branches/beagle-rdf/Filters/FilterTexi.cs Mon Mar 24 21:09:25 2008
@@ -31,6 +31,8 @@
using System.Text;
using System.Text.RegularExpressions;
+using Beagle.Daemon;
+
namespace Beagle.Filters {
public class FilterTexi : Beagle.Daemon.Filter {
@@ -44,13 +46,13 @@
public FilterTexi ()
{
// FIXME: Should this be documentation ?
- SetFileType ("document");
+ SetFileType ("documentation");
}
protected override void RegisterSupportedTypes ()
{
// Make this a general texi filter.
- AddSupportedMimeType ("text/x-texinfo");
+ AddSupportedFlavor (FilterFlavor.NewFromMimeType ("text/x-texinfo"));
}
/*
@@ -66,7 +68,6 @@
line = line.Replace (keyword, String.Empty);
AppendLine (line);
- AppendWhiteSpace ();
}
}
}
Modified: branches/beagle-rdf/Filters/Makefile.am
==============================================================================
--- branches/beagle-rdf/Filters/Makefile.am (original)
+++ branches/beagle-rdf/Filters/Makefile.am Mon Mar 24 21:09:25 2008
@@ -70,6 +70,7 @@
$(srcdir)/FilterHtml.cs \
$(srcdir)/FilterIgnore.cs \
$(srcdir)/FilterImage.cs \
+ $(srcdir)/FilterInfo.cs \
$(srcdir)/FilterJava.cs \
$(srcdir)/FilterJpeg.cs \
$(srcdir)/FilterJs.cs \
@@ -107,6 +108,7 @@
$(srcdir)/FilterSpreadsheet.cs \
$(srcdir)/FilterSvg.cs \
$(srcdir)/FilterTeX.cs \
+ $(srcdir)/FilterTexi.cs \
$(srcdir)/FilterText.cs \
$(srcdir)/FilterTiff.cs \
$(srcdir)/FilterTotem.cs \
Modified: branches/beagle-rdf/Makefile.am
==============================================================================
--- branches/beagle-rdf/Makefile.am (original)
+++ branches/beagle-rdf/Makefile.am Mon Mar 24 21:09:25 2008
@@ -39,10 +39,6 @@
SUBDIRS += epiphany-extension
endif
-if ENABLE_BLUDGEON
-SUBDIRS += bludgeon
-endif
-
if ENABLE_THUNDERBIRD
SUBDIRS += thunderbird-extension
endif
Modified: branches/beagle-rdf/NEWS
==============================================================================
--- branches/beagle-rdf/NEWS (original)
+++ branches/beagle-rdf/NEWS Mon Mar 24 21:09:25 2008
@@ -1,3 +1,89 @@
+Version 0.3.4
+23 March 2008
+
+* Need Mono at least 1.2.6; recommended at 1.9
+
+* GMime-sharp recommended at least 2.2.16
+
+* Add API to set the number of words and the total length of the snippets.
+
+* Move to Lucene.Net 2.1 (backported a faster analyzer and other improvements).
+
+* Move query mapping (e.g. mappings for ext:, genre:) to a separate user editable /etc/beagle/query-mapping.xml (global) and ~/.beagle/query-mapping.xml (local). beagled needs to be restarted to read the new mappings.
+
+* Expose a Hide/Show DBus interface for beagle-search.
+
+* Firefox search bar plugin to query using beagle's web interface.
+
+* Add a tool beagle-static-query to query beagle without starting beagled. This is mainly meant for static indexes (like locate with the system index, like mod for a monodoc index).
+
+* Lots of beagle-search beautification.
+
+* Add a query mapping "filename:" to query for filenames and words in it (e.g. match My in MyFile.txt and My_File.txt).
+
+* Enable the TeXinfo filter.
+
+* Add a filter for .info files (including gz/bz2/lzma compressed info files). manpages crawl rule now contains /usr/share/info.
+
+* SIGUSR2 will now also print runtime debugging information of beagled.
+
+* Use the new Mono.Unix.UnixSignal if built with mono 1.9. This should remove the last of the beagled/index-helper hangs during shutdown.
+
+* Add the missing files for the firefox extension sidebar.
+
+* Show a better error message if ~/.beagle is not found when the firefox extension is started (should help debian #463922).
+
+* Fixes #514252 (evo backends now explicitly reference glib-sharp)
+
+* Fixes #509487 (use LANG=C when external programs are started from any filter)
+
+* Fix build with mono 1.9 and gnome-sharp 2.20 (#514252, #518220)
+
+* Fix property change indexing (renames, moves etc.) which broke in 0.3.0
+
+* Fix our filter matching algorithm to correctly choose filters in the right order and move to the next one if one filter fails.
+
+* Fix file leak in ebuild filter.
+
+* Clean up some tmp files which could have left behind during a corner case.
+
+* Fix a startup crash in the Pidgin backend if the Pidgin log directory is absent during beagled startup.
+
+* Check for msgfmt in ./configure before building.
+
+* Fix glib-critical warnings in libbeagle (and python bindings) when snippets with empty fragments were sent by beagled.
+
+* Fix some bugs in snippet due to which valid matches were not included in the snippets.
+
+* Timeout in libbeagle if connection to beagled takes more than 5 secs. This will prevent hangs in konqueror/gtkfilechooser if beagled is hanged for some reason.
+
+* Do not throw error if scheduler information and index information is empty since they can actually be empty.
+
+* Translations:
+ - Spanish (Jorge Gonzalez)
+ - Polish (GNOME PL Team.)
+ - Basque (Inaki Larranaga Murgoitio)
+ - Italian (Gianvito Cavasoli)
+ - Japanese (Takeshi AIHANA)
+ - Norwegian bokmÃl (Kjartan Maraas)
+ - Finnish (Ilkka Tuohela)
+ - Catalan (Jordi Mas, Gil Forcada)
+ - Occitan (Yannig Marchegay)
+ - Swedish (Daniel Nylander)
+ - Brazilian Portuguese (Igor Soares)
+ - French (StÃphane Raimbault)
+ - British English (Philip Withnall)
+ - Hungarian (Gabor Kelemen)
+
+Contributors to this release: Debajyoti Bera, Lukas Lipka, Kevin
+Kubasik, Nirbheek Chauhan and Jaikumar Singh.
+
+Full set of changes:
+http://svn.gnome.org/viewvc/beagle/tags/BEAGLE_0_3_4/beagle/ChangeLog?view=markup
+http://svn.gnome.org/viewvc/beagle/tags/BEAGLE_0_3_4/libbeagle/ChangeLog?view=markup
+
+-------------------------------------------------------------------------------
+
Version 0.3.3
2 February 2008
Modified: branches/beagle-rdf/Util/ExceptionHandlingThread.cs
==============================================================================
--- branches/beagle-rdf/Util/ExceptionHandlingThread.cs (original)
+++ branches/beagle-rdf/Util/ExceptionHandlingThread.cs Mon Mar 24 21:09:25 2008
@@ -60,8 +60,8 @@
} catch (ThreadAbortException e) {
Logger.Log.Debug ("Thread aborted: {0}\n{1}\n", this.thread.Name, e.StackTrace);
} catch (Exception e) {
- Logger.Log.Warn (e, "Exception caught while executing {0}:{1}",
- this.method.Target, this.method.Method);
+ Logger.Log.Warn (e, "Exception caught in {2} while executing {0}:{1}",
+ this.method.Target, this.method.Method, this.thread.Name);
}
lock (live_threads)
Modified: branches/beagle-rdf/beagled/BeagleDaemon.cs
==============================================================================
--- branches/beagle-rdf/beagled/BeagleDaemon.cs (original)
+++ branches/beagle-rdf/beagled/BeagleDaemon.cs Mon Mar 24 21:09:25 2008
@@ -500,7 +500,12 @@
QueryDriver.Init ();
Server.Init ();
+#if MONO_1_9
+ Shutdown.SetupSignalHandlers (new Shutdown.SignalHandler (HandleSignal));
+#else
SetupSignalHandlers ();
+#endif
+
Shutdown.ShutdownEvent += OnShutdown;
main_loop = new MainLoop ();
@@ -578,7 +583,8 @@
return;
// Set shutdown flag to true so that other threads can stop initializing
- if ((Mono.Unix.Native.Signum) signal != Mono.Unix.Native.Signum.SIGUSR1)
+ if ((Mono.Unix.Native.Signum) signal != Mono.Unix.Native.Signum.SIGUSR1 &&
+ (Mono.Unix.Native.Signum) signal != Mono.Unix.Native.Signum.SIGUSR2)
Shutdown.ShutdownRequested = true;
// Do all signal handling work in the main loop and not in the signal handler.
@@ -589,17 +595,22 @@
{
Logger.Log.Debug ("Handling signal {0} ({1})", signal, (Mono.Unix.Native.Signum) signal);
+ // Pass the signals to the helper too.
+ GLib.Idle.Add (new GLib.IdleHandler (delegate ()
+ {
+ RemoteIndexer.SignalRemoteIndexer ((Mono.Unix.Native.Signum) signal);
+ return false;
+ }));
+
// If we get SIGUSR1, turn the debugging level up.
if ((Mono.Unix.Native.Signum) signal == Mono.Unix.Native.Signum.SIGUSR1) {
LogLevel old_level = Log.Level;
Log.Level = LogLevel.Debug;
Log.Debug ("Moving from log level {0} to Debug", old_level);
- }
-
- // Send informational signals to the helper too.
- if ((Mono.Unix.Native.Signum) signal == Mono.Unix.Native.Signum.SIGUSR1 ||
- (Mono.Unix.Native.Signum) signal == Mono.Unix.Native.Signum.SIGUSR2) {
- GLib.Idle.Add (new GLib.IdleHandler (delegate () { RemoteIndexer.SignalRemoteIndexer ((Mono.Unix.Native.Signum) signal); return false; }));
+ return;
+ } else if ((Mono.Unix.Native.Signum) signal == Mono.Unix.Native.Signum.SIGUSR2) {
+ // Debugging hook for beagled
+ QueryDriver.DebugHook ();
return;
}
Modified: branches/beagle-rdf/beagled/BuildIndex.cs
==============================================================================
--- branches/beagle-rdf/beagled/BuildIndex.cs (original)
+++ branches/beagle-rdf/beagled/BuildIndex.cs Mon Mar 24 21:09:25 2008
@@ -354,7 +354,16 @@
fa_store = new FileAttributesStore (backing_fa_store);
// Set up signal handlers
+#if MONO_1_9
+ Shutdown.SetupSignalHandlers (delegate (int signal)
+ {
+ if (signal == (int) Mono.Unix.Native.Signum.SIGINT ||
+ signal == (int) Mono.Unix.Native.Signum.SIGTERM)
+ Shutdown.BeginShutdown ();
+ });
+#else
SetupSignalHandlers ();
+#endif
Thread monitor_thread = null;
Modified: branches/beagle-rdf/beagled/ExternalMetadataQueryable.cs
==============================================================================
--- branches/beagle-rdf/beagled/ExternalMetadataQueryable.cs (original)
+++ branches/beagle-rdf/beagled/ExternalMetadataQueryable.cs Mon Mar 24 21:09:25 2008
@@ -74,7 +74,7 @@
return 0;
}
- public ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text)
+ public ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text, int ctx_length, int snp_length)
{
return null;
}
Modified: branches/beagle-rdf/beagled/FileSystemQueryable/FileCrawlTask.cs
==============================================================================
--- branches/beagle-rdf/beagled/FileSystemQueryable/FileCrawlTask.cs (original)
+++ branches/beagle-rdf/beagled/FileSystemQueryable/FileCrawlTask.cs Mon Mar 24 21:09:25 2008
@@ -135,5 +135,14 @@
Reschedule = true;
}
+
+ internal void DebugHook ()
+ {
+ lock (big_lock)
+ Log.Debug ("FSQ:FileCrawlTask Debughook: Crawling {0} (generator is {1}) and is {2}active",
+ (current_dir != null ? current_dir.FullName : "nothing"),
+ (current_generator != null ? "busy" : "empty"),
+ (is_active ? String.Empty : "in"));
+ }
}
}
Modified: branches/beagle-rdf/beagled/FileSystemQueryable/FileSystemQueryable.cs
==============================================================================
--- branches/beagle-rdf/beagled/FileSystemQueryable/FileSystemQueryable.cs (original)
+++ branches/beagle-rdf/beagled/FileSystemQueryable/FileSystemQueryable.cs Mon Mar 24 21:09:25 2008
@@ -464,6 +464,8 @@
lock (dir_models_by_path)
dir_models_by_path.Remove (expired_path);
+
+ FileAttributesStore.Drop (expired_path);
}
public void AddDirectory (DirectoryModel parent, string name)
@@ -1027,6 +1029,14 @@
self_task = null;
}
}
+
+ internal void DebugHook ()
+ {
+ lock (queue_lock)
+ Log.Debug ("FSQ:FileSystemEventsGenerator Debughook: {0} more events to process, generator is {1}active",
+ event_queue.Count,
+ (self_task != null ? String.Empty : "in"));
+ }
}
//////////////////////////////////////////////////////////////////////////
@@ -1843,7 +1853,7 @@
return parent_dirs;
}
- override public ISnippetReader GetSnippet (string [] query_terms, Hit hit, bool full_text)
+ override public ISnippetReader GetSnippet (string [] query_terms, Hit hit, bool full_text, int ctx_length, int snp_length)
{
// Uri remapping from a hit is easy: the internal uri
// is stored in a property.
@@ -1853,12 +1863,12 @@
TextReader reader = TextCache.UserCache.GetReader (uri, ref self_cache);
if (self_cache)
- return SnippetFu.GetSnippetFromFile (query_terms, hit.Uri.LocalPath, full_text);
+ return SnippetFu.GetSnippetFromFile (query_terms, hit.Uri.LocalPath, full_text, ctx_length, snp_length);
if (reader == null)
return null;
- return SnippetFu.GetSnippet (query_terms, reader, full_text);
+ return SnippetFu.GetSnippet (query_terms, reader, full_text, ctx_length, snp_length);
}
override public void Start ()
@@ -2014,6 +2024,45 @@
return internal_uri;
}
+
+ //////////////////////////
+
+ internal override void DebugHook ()
+ {
+ Log.Debug ("FSQ DebugHook: {0} roots, {1} roots by path, {2} dir_models_by_id, {3} name_info_by_id, {4} dir_models_by_path",
+ (roots != null ? roots.Count : -1),
+ (roots_by_path != null ? roots_by_path.Count : -1),
+ (dir_models_by_id != null ? dir_models_by_id.Count : -1),
+ (name_info_by_id != null ? name_info_by_id.Count : -1),
+ (dir_models_by_path != null ? dir_models_by_path.Count : -1));
+
+ Log.Debug ("FSQ Debughook: {0} external_pending_indexables",
+ (external_pending_indexables != null ? external_pending_indexables.Count : -1));
+
+ lock (big_lock) {
+ foreach (DirectoryModel root in roots) {
+ int count = 1;
+ ArrayList queue = new ArrayList ();
+ queue.Add (root);
+ DirectoryModel child;
+ while (queue.Count > 0) {
+ child = (DirectoryModel) queue [0];
+ queue.RemoveAt (0);
+
+ if (child.ChildCount == 0)
+ continue;
+ count += child.ChildCount;
+ queue.AddRange (child.Children);
+ }
+ Console.WriteLine ("FSQ Debughook: {0} has {1} subdirs", root.FullName, count);
+ }
+ }
+
+ tree_crawl_task.DebugHook ();
+ file_crawl_task.DebugHook ();
+ fs_event_generator.DebugHook ();
+ uid_manager.DebugHook ();
+ }
}
}
Modified: branches/beagle-rdf/beagled/FileSystemQueryable/InternalUriManager.cs
==============================================================================
--- branches/beagle-rdf/beagled/FileSystemQueryable/InternalUriManager.cs (original)
+++ branches/beagle-rdf/beagled/FileSystemQueryable/InternalUriManager.cs Mon Mar 24 21:09:25 2008
@@ -61,6 +61,12 @@
////////////////////////////////////////////////////////////////
+ // Ouch! Hardcoding prefix is not good
+ static FieldSelector fields_nameinfo = new MapFieldSelector (new string[] {"Uri",
+ "prop:k:" + Property.ExactFilenamePropKey,
+ "prop:k:" + Property.ParentDirUriPropKey,
+ "prop:k:" + Property.IsDirectoryPropKey
+ });
private NameInfo DocumentToNameInfo (Document doc)
{
@@ -132,7 +138,7 @@
if (match_id != -1) {
Document doc;
- doc = reader.Document (match_id);
+ doc = reader.Document (match_id, fields_nameinfo);
info = DocumentToNameInfo (doc);
}
@@ -269,7 +275,7 @@
break;
Document doc;
- doc = searcher.Doc (i);
+ doc = searcher.Doc (i, fields_nameinfo);
NameInfo info;
info = DocumentToNameInfo (doc);
@@ -384,6 +390,10 @@
return GetAllDirectoryNameInfo (null);
}
+ internal void DebugHook ()
+ {
+ Log.Debug ("FSQ:InternalUriManager Debughook: {0} cached_uid_by_path", cached_uid_by_path.Count);
+ }
}
}
Modified: branches/beagle-rdf/beagled/FileSystemQueryable/TreeCrawlTask.cs
==============================================================================
--- branches/beagle-rdf/beagled/FileSystemQueryable/TreeCrawlTask.cs (original)
+++ branches/beagle-rdf/beagled/FileSystemQueryable/TreeCrawlTask.cs Mon Mar 24 21:09:25 2008
@@ -129,5 +129,10 @@
SetIsActive (false, null);
}
+ internal void DebugHook ()
+ {
+ lock (big_lock)
+ Log.Debug ("FSQ:TreeCrawlTask Debughook: {0} directories left to crawl, current {1}active", to_be_crawled.Count, is_active ? String.Empty : "in");
+ }
}
}
Modified: branches/beagle-rdf/beagled/FilterFactory.cs
==============================================================================
--- branches/beagle-rdf/beagled/FilterFactory.cs (original)
+++ branches/beagle-rdf/beagled/FilterFactory.cs Mon Mar 24 21:09:25 2008
@@ -347,6 +347,7 @@
if (Debug)
Logger.Log.Debug ("None of the matching filters could process the file: {0}", path);
+ indexable.NoContent = true;
return false;
}
Modified: branches/beagle-rdf/beagled/IQueryable.cs
==============================================================================
--- branches/beagle-rdf/beagled/IQueryable.cs (original)
+++ branches/beagle-rdf/beagled/IQueryable.cs Mon Mar 24 21:09:25 2008
@@ -48,7 +48,7 @@
// Just return the number of matches
int DoCountMatchQuery (Query query);
- ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text);
+ ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text, int context_length, int snippet_length);
QueryableStatus GetQueryableStatus ();
}
Modified: branches/beagle-rdf/beagled/IndexHelper/IndexHelper.cs
==============================================================================
--- branches/beagle-rdf/beagled/IndexHelper/IndexHelper.cs (original)
+++ branches/beagle-rdf/beagled/IndexHelper/IndexHelper.cs Mon Mar 24 21:09:25 2008
@@ -137,7 +137,11 @@
Server.Init ();
+#if MONO_1_9
+ Shutdown.SetupSignalHandlers (new Shutdown.SignalHandler (HandleSignal));
+#else
SetupSignalHandlers ();
+#endif
Shutdown.ShutdownEvent += OnShutdown;
Modified: branches/beagle-rdf/beagled/KonversationQueryable/KonversationQueryable.cs
==============================================================================
--- branches/beagle-rdf/beagled/KonversationQueryable/KonversationQueryable.cs (original)
+++ branches/beagle-rdf/beagled/KonversationQueryable/KonversationQueryable.cs Mon Mar 24 21:09:25 2008
@@ -513,7 +513,7 @@
return false;
}
- override public ISnippetReader GetSnippet (string [] query_terms, Hit hit, bool full_text)
+ override public ISnippetReader GetSnippet (string [] query_terms, Hit hit, bool full_text, int ctx_length, int snp_length)
{
if (hit.ParentUri == null)
return null;
@@ -528,7 +528,7 @@
LineTextReader reader;
reader = new LineTextReader (path, begin_offset, end_offset);
- return SnippetFu.GetSnippet (query_terms, reader, full_text);
+ return SnippetFu.GetSnippet (query_terms, reader, full_text, ctx_length, snp_length);
} catch {
return null;
}
Modified: branches/beagle-rdf/beagled/KopeteQueryable/KopeteQueryable.cs
==============================================================================
--- branches/beagle-rdf/beagled/KopeteQueryable/KopeteQueryable.cs (original)
+++ branches/beagle-rdf/beagled/KopeteQueryable/KopeteQueryable.cs Mon Mar 24 21:09:25 2008
@@ -219,7 +219,7 @@
return true;
}
- public override ISnippetReader GetSnippet (string [] query_terms, Hit hit, bool full_text)
+ public override ISnippetReader GetSnippet (string [] query_terms, Hit hit, bool full_text, int ctx_length, int snp_length)
{
TextReader reader = TextCache.UserCache.GetReader (hit.Uri);
@@ -228,7 +228,7 @@
KopeteSnippetReader snippet_reader = new KopeteSnippetReader (reader);
- return SnippetFu.GetSnippet (query_terms, snippet_reader, full_text);
+ return SnippetFu.GetSnippet (query_terms, snippet_reader, full_text, ctx_length, snp_length);
}
private class KopeteSnippetReader : TextReader {
Modified: branches/beagle-rdf/beagled/LuceneBitArray.cs
==============================================================================
--- branches/beagle-rdf/beagled/LuceneBitArray.cs (original)
+++ branches/beagle-rdf/beagled/LuceneBitArray.cs Mon Mar 24 21:09:25 2008
@@ -198,6 +198,8 @@
////////////////////////////////////////////////////////////
+ static FieldSelector fields_uri = new MapFieldSelector (new string[] {"Uri"});
+
public void ProjectOnto (LuceneBitArray other)
{
int j = 0;
@@ -209,7 +211,7 @@
j = i+1;
Document doc;
- doc = searcher.Doc (i, LuceneQueryingDriver.fields_uri);
+ doc = searcher.Doc (i, fields_uri);
other.AddUri (doc.Get ("Uri"));
}
Modified: branches/beagle-rdf/beagled/LuceneCommon.cs
==============================================================================
--- branches/beagle-rdf/beagled/LuceneCommon.cs (original)
+++ branches/beagle-rdf/beagled/LuceneCommon.cs Mon Mar 24 21:09:25 2008
@@ -87,7 +87,7 @@
// 18: add IsPersistent to properties, and adjust coded values
// in AddPropertyToDocument() and GetPropertyFromDocument();
// changed subdate field format rules for better readability
- // 19: store a list of current properties in a field
+ // 19: Update lucene and analyzer (lucene.net-2.1)
private const int MAJOR_VERSION = 19;
private int minor_version = 0;
Modified: branches/beagle-rdf/beagled/LuceneIndexingDriver.cs
==============================================================================
--- branches/beagle-rdf/beagled/LuceneIndexingDriver.cs (original)
+++ branches/beagle-rdf/beagled/LuceneIndexingDriver.cs Mon Mar 24 21:09:25 2008
@@ -357,6 +357,7 @@
if (Shutdown.ShutdownRequested) {
foreach (DeferredInfo di in deferred_indexables)
di.Cleanup ();
+ deferred_indexables.Clear ();
foreach (Indexable indexable in request_indexables)
indexable.Cleanup ();
@@ -503,6 +504,7 @@
if (Shutdown.ShutdownRequested) {
foreach (DeferredInfo di in deferred_indexables)
di.Cleanup ();
+ deferred_indexables.Clear ();
primary_writer.Close ();
if (secondary_writer != null)
@@ -542,7 +544,8 @@
// If we have content, try to find a filter
// we we can use to process the indexable
try {
- FilterFactory.FilterIndexable (indexable, (disable_textcache ? null : text_cache), out filter);
+ if (! FilterFactory.FilterIndexable (indexable, (disable_textcache ? null : text_cache), out filter))
+ indexable.NoContent = true;
} catch (Exception e) {
indexable.NoContent = true;
}
@@ -740,6 +743,7 @@
lock (flush_lock) {
foreach (DeferredInfo di in deferred_indexables)
di.Cleanup ();
+ deferred_indexables.Clear ();
}
}
}
Modified: branches/beagle-rdf/beagled/LuceneQueryable.cs
==============================================================================
--- branches/beagle-rdf/beagled/LuceneQueryable.cs (original)
+++ branches/beagle-rdf/beagled/LuceneQueryable.cs Mon Mar 24 21:09:25 2008
@@ -104,7 +104,8 @@
// If the queryable is in read-only more, don't
// instantiate an indexer for it.
- if (read_only_mode)
+ // FIXME: --indexing-delay -1 is a hack for --read-only; need to fix this
+ if (read_only_mode || QueryDriver.IndexingDelay < 0)
return;
indexer = LocalIndexerHook ();
@@ -368,7 +369,7 @@
/////////////////////////////////////////
- protected SnippetReader GetSnippetFromTextCache (string [] query_terms, Uri uri, bool full_text)
+ protected SnippetReader GetSnippetFromTextCache (string [] query_terms, Uri uri, bool full_text, int ctx_length, int snp_length)
{
// Look up the hit in our text cache. If it is there,
// use the cached version to generate a snippet.
@@ -378,12 +379,12 @@
if (reader == null)
return null;
- return SnippetFu.GetSnippet (query_terms, reader, full_text);
+ return SnippetFu.GetSnippet (query_terms, reader, full_text, ctx_length, snp_length);
}
- virtual public ISnippetReader GetSnippet (string [] query_terms, Hit hit, bool full_text)
+ virtual public ISnippetReader GetSnippet (string [] query_terms, Hit hit, bool full_text, int ctx_length, int snp_length)
{
- return GetSnippetFromTextCache (query_terms, hit.Uri, full_text);
+ return GetSnippetFromTextCache (query_terms, hit.Uri, full_text, ctx_length, snp_length);
}
/////////////////////////////////////////
@@ -1059,5 +1060,11 @@
{
return new LuceneQueryingDriver (index_name, minor_version, read_only_mode);
}
+
+ //////////////////////////////////////////////////////////////////////////////////
+
+ virtual internal void DebugHook ()
+ {
+ }
}
}
Modified: branches/beagle-rdf/beagled/NetworkServicesQueryable/NetworkServicesQueryable.cs
==============================================================================
--- branches/beagle-rdf/beagled/NetworkServicesQueryable/NetworkServicesQueryable.cs (original)
+++ branches/beagle-rdf/beagled/NetworkServicesQueryable/NetworkServicesQueryable.cs Mon Mar 24 21:09:25 2008
@@ -135,7 +135,7 @@
return 0;
}
- public ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text)
+ public ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text, int ctx_length, int snp_length)
{
return null;
}
Modified: branches/beagle-rdf/beagled/PidginQueryable/PidginQueryable.cs
==============================================================================
--- branches/beagle-rdf/beagled/PidginQueryable/PidginQueryable.cs (original)
+++ branches/beagle-rdf/beagled/PidginQueryable/PidginQueryable.cs Mon Mar 24 21:09:25 2008
@@ -54,7 +54,9 @@
private void StartWorker()
{
bool gaim_exists = Directory.Exists (Path.Combine (PathFinder.HomeDir, ".gaim"));
+ gaim_exists = gaim_exists && Directory.Exists (Path.Combine (Path.Combine (PathFinder.HomeDir, ".gaim"), "logs"));
bool pidgin_exists = Directory.Exists (Path.Combine (PathFinder.HomeDir, ".purple"));
+ pidgin_exists = pidgin_exists && Directory.Exists (Path.Combine (Path.Combine (PathFinder.HomeDir, ".purple"), "logs"));
if (!pidgin_exists && !gaim_exists) {
GLib.Timeout.Add (polling_interval, new GLib.TimeoutHandler (CheckForExistence));
@@ -88,7 +90,9 @@
private bool CheckForExistence ()
{
bool gaim_exists = Directory.Exists (Path.Combine (PathFinder.HomeDir, ".gaim"));
+ gaim_exists = gaim_exists && Directory.Exists (Path.Combine (Path.Combine (PathFinder.HomeDir, ".gaim"), "logs"));
bool pidgin_exists = Directory.Exists (Path.Combine (PathFinder.HomeDir, ".purple"));
+ pidgin_exists = pidgin_exists && Directory.Exists (Path.Combine (Path.Combine (PathFinder.HomeDir, ".purple"), "logs"));
if (gaim_exists || pidgin_exists) {
this.Start ();
@@ -125,7 +129,7 @@
return true;
}
- public override ISnippetReader GetSnippet (string [] query_terms, Hit hit, bool full_text)
+ public override ISnippetReader GetSnippet (string [] query_terms, Hit hit, bool full_text, int ctx_length, int snp_length)
{
TextReader reader = TextCache.UserCache.GetReader (hit.Uri);
@@ -137,7 +141,7 @@
if (line[0] == '<')
reader = new HtmlRemovingReader (reader);
- return SnippetFu.GetSnippet (query_terms, reader, full_text);
+ return SnippetFu.GetSnippet (query_terms, reader, full_text, ctx_length, snp_length);
}
public ImBuddyListReader ImBuddyListReader {
Modified: branches/beagle-rdf/beagled/QueryDriver.cs
==============================================================================
--- branches/beagle-rdf/beagled/QueryDriver.cs (original)
+++ branches/beagle-rdf/beagled/QueryDriver.cs Mon Mar 24 21:09:25 2008
@@ -36,7 +36,7 @@
namespace Beagle.Daemon {
- public class QueryDriver {
+ public static class QueryDriver {
// Contains list of queryables explicitly asked by --allow-backend or --backend name
// --allow-backend/--backend name : dont read config information and only start backend 'name'
@@ -130,6 +130,7 @@
public static int IndexingDelay {
set { indexing_delay = value; }
+ get { return indexing_delay; }
}
//////////////////////////////////////////////////////////////////////////////////////
@@ -533,7 +534,7 @@
continue;
if (sb.Length > 0)
sb.Append (' ');
- sb.Append (LuceneCommon.Stem (split [i]));
+ sb.Append (LuceneCommon.Stem (split [i].ToLower ()));
}
response.StemmedText.Add (sb.ToString ());
}
@@ -694,5 +695,25 @@
return false;
}
}
+
+ /////////////////////////////////////////////////////////
+
+ // Various debug hooks; especially persistant state information and memory issues
+
+ public static void DebugHook ()
+ {
+ Log.Debug ("Debughook called:");
+ LuceneQueryable l;
+ foreach (Queryable q in Queryables) {
+ l = q.IQueryable as LuceneQueryable;
+ if (l == null)
+ continue;
+ try {
+ l.DebugHook ();
+ } catch (Exception e) {
+ Log.Debug (e, "Exception during Debughook for {0}", q.Name);
+ }
+ }
+ }
}
}
Modified: branches/beagle-rdf/beagled/Queryable.cs
==============================================================================
--- branches/beagle-rdf/beagled/Queryable.cs (original)
+++ branches/beagle-rdf/beagled/Queryable.cs Mon Mar 24 21:09:25 2008
@@ -98,7 +98,7 @@
}
}
- public ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text)
+ public ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text, int ctx_length, int snp_length)
{
if (hit == null)
return null;
@@ -110,7 +110,7 @@
}
try {
- return iqueryable.GetSnippet (query_terms, hit, full_text);
+ return iqueryable.GetSnippet (query_terms, hit, full_text, ctx_length, snp_length);
} catch (Exception ex) {
Logger.Log.Warn (ex, "Caught exception calling DoQuery on '{0}'", Name);
}
Modified: branches/beagle-rdf/beagled/Server.cs
==============================================================================
--- branches/beagle-rdf/beagled/Server.cs (original)
+++ branches/beagle-rdf/beagled/Server.cs Mon Mar 24 21:09:25 2008
@@ -294,12 +294,14 @@
public UnixConnectionHandler (UnixClient client)
{
this.client = client;
+ this.client.SendBufferSize = 4096;
+ this.client.ReceiveBufferSize = 4096;
}
public override bool SendResponse(ResponseMessage response)
{
bool result = false;
-
+
lock (this.client_lock) {
if (this.client == null)
return false;
@@ -409,6 +411,8 @@
Shutdown.WorkerFinished (network_data);
}
+ private bool closed = false;
+
public override void Close ()
{
CancelIfBlocking ();
@@ -417,17 +421,24 @@
// grab the lock here and close the underlying
// UnixClient, or else we'd deadlock between here and
// the Read() in HandleConnection()
+
+ // Do this in a lock since Close() should be thread-safe (can be called from AsyncCallback handler)
lock (this.client_lock) {
+ if (closed)
+ return;
+
if (this.client != null) {
this.client.Close ();
this.client = null;
}
- }
- if (this.executor != null) {
- this.executor.Cleanup ();
- this.executor.AsyncResponseEvent -= OnAsyncResponse;
- this.executor = null;
+ if (this.executor != null) {
+ this.executor.Cleanup ();
+ this.executor.AsyncResponseEvent -= OnAsyncResponse;
+ this.executor = null;
+ }
+
+ closed = true;
}
Server.RunGC ();
@@ -439,8 +450,10 @@
try {
bytes_read = this.client.GetStream ().EndRead (ar);
- } catch (SocketException) {
- } catch (IOException) { }
+ } catch (IOException e) {
+ if (! (e.InnerException is SocketException))
+ throw e;
+ } catch (ObjectDisposedException) { }
if (bytes_read == 0)
Close ();
@@ -455,8 +468,26 @@
return;
}
- this.client.GetStream ().BeginRead (new byte[1024], 0, 1024,
- new AsyncCallback (WatchCallback), null);
+ try {
+ this.client.GetStream ().BeginRead (new byte[1024], 0, 1024,
+ new AsyncCallback (WatchCallback), null);
+ } catch (ObjectDisposedException) {
+ Log.Debug ("Network stream closed; closing connection at this end");
+ this.Close ();
+ return;
+ } catch (IOException e) {
+ if (e.InnerException is SocketException) {
+ Log.Debug (e.InnerException, "Socket exception while setting up watch");
+ this.Close ();
+ return;
+ } else
+ throw e;
+ } catch (System.ArgumentException e) {
+ // https://bugzilla.novell.com/show_bug.cgi?id=371923
+ Log.Debug (e, "Possibly socket is already closed");
+ this.Close ();
+ return;
+ }
}
}
Modified: branches/beagle-rdf/beagled/Shutdown.cs
==============================================================================
--- branches/beagle-rdf/beagled/Shutdown.cs (original)
+++ branches/beagle-rdf/beagled/Shutdown.cs Mon Mar 24 21:09:25 2008
@@ -27,7 +27,9 @@
using System;
using System.Threading;
using System.Collections;
+using System.Diagnostics;
using Beagle.Util;
+using Mono.Unix;
namespace Beagle.Daemon {
@@ -40,6 +42,7 @@
static Hashtable workers_names = new Hashtable ();
static bool shutdownRequested = false;
static bool shutdownStarted = false;
+ static bool mainloop_finished = false;
public delegate void ShutdownHandler ();
public static event ShutdownHandler ShutdownEvent;
@@ -108,14 +111,6 @@
main_loop = loop;
}
- // Our handler triggers an orderly shutdown when it receives a signal.
- // However, this can be annoying if the process gets wedged during
- // shutdown. To deal with that case, we make a note of the time when
- // the first signal comes in, and we allow signals to unconditionally
- // kill the process after 5 seconds have passed.
-
- static DateTime signal_time = DateTime.MinValue;
-
public static void BeginShutdown ()
{
lock (shutdownLock) {
@@ -129,31 +124,7 @@
shutdownStarted = true;
}
- // FIXME: This whole "unconditional killing after 5 seconds because
- // beagled can hang while shutting down" thing should not occur. Any such
- // incident should be immediately investigated and fixed. Hint: Sending
- // kill -quit `pidof beagled` will probably reveal that beagled got locked
- // when signal handler was called and some thread was executing some native
- // method.
- bool first_signal = false;
- if (signal_time == DateTime.MinValue) {
- Log.Always ("Shutdown requested");
- signal_time = DateTime.Now;
- first_signal = true;
- }
-
- if (! first_signal) {
- double t = (DateTime.Now - signal_time).TotalSeconds;
- const double min_t = 5;
-
- if (t < min_t) {
- Logger.Log.Debug ("Signals can force an immediate shutdown in {0:0.00}s", min_t-t);
- return;
- } else {
- Logger.Log.Debug ("Forcing immediate shutdown.");
- Environment.Exit (0);
- }
- }
+ Log.Always ("Shutdown requested");
if (ShutdownEvent != null) {
try {
@@ -180,6 +151,70 @@
Logger.Log.Info ("All workers have finished. Exiting main loop.");
main_loop.Quit ();
+
+#if MONO_1_9
+ // So (re)sending a SIGTERM or SIGINT after mainloop is over will kill the process.
+ // Is it good or bad ? (some of the exception handling thread might still be running)
+ // I think its good; it sort of emulates the 5sec forceful killing feature of < 0.3.4
+ lock (shutdownLock)
+ mainloop_finished = true;
+
+ // Stop the signal handler thread by sending a signal
+ Mono.Unix.Native.Syscall.kill (Process.GetCurrentProcess ().Id, Mono.Unix.Native.Signum.SIGINT);
+#endif
+ }
+
+#if MONO_1_9
+ static bool MainloopFinished {
+ get {
+ lock (shutdownLock)
+ return mainloop_finished;
+ }
+ }
+
+ public delegate void SignalHandler (int signal);
+
+ public static void SetupSignalHandlers (SignalHandler signal_handler)
+ {
+ UnixSignal[] signals = new UnixSignal [] {
+ new UnixSignal (Mono.Unix.Native.Signum.SIGINT),
+ new UnixSignal (Mono.Unix.Native.Signum.SIGTERM),
+ new UnixSignal (Mono.Unix.Native.Signum.SIGUSR1),
+ new UnixSignal (Mono.Unix.Native.Signum.SIGUSR2)
+ };
+
+ // Ignore SIGPIPE
+ // FIXME: Shouldn't this be done in every thread ?
+ Mono.Unix.Native.Stdlib.SetSignalAction (Mono.Unix.Native.Signum.SIGPIPE, Mono.Unix.Native.SignalAction.Ignore);
+
+ Thread signal_thread = new Thread (delegate () {
+ Log.Debug ("Starting signal handler thread");
+ int signal_handler_timeout = -1;
+ while (! MainloopFinished) {
+ int index = UnixSignal.WaitAny (signals, signal_handler_timeout);
+
+ if (index > 3)
+ continue;
+ Mono.Unix.Native.Signum signal = signals [index].Signum;
+
+ // Set shutdown flag to true so that other threads can stop initializing
+ if (signal == Mono.Unix.Native.Signum.SIGINT ||
+ signal == Mono.Unix.Native.Signum.SIGTERM) {
+ ShutdownRequested = true;
+ signal_handler_timeout = 200; // 200 ms
+ }
+
+ if (signal_handler == null)
+ continue;
+
+ // Do all signal handling work in the main loop and not in the signal handler.
+ GLib.Idle.Add (new GLib.IdleHandler (delegate () { signal_handler ((int) signal); return false; }));
+ }
+ Log.Debug ("Exiting signal handler thread");
+ });
+
+ signal_thread.Start ();
}
+#endif
}
}
Modified: branches/beagle-rdf/beagled/SnippetExecutor.cs
==============================================================================
--- branches/beagle-rdf/beagled/SnippetExecutor.cs (original)
+++ branches/beagle-rdf/beagled/SnippetExecutor.cs Mon Mar 24 21:09:25 2008
@@ -42,13 +42,15 @@
Queryable queryable = QueryDriver.GetQueryable (request.Hit.Source);
ISnippetReader snippet_reader;
bool full_text = request.FullText;
+ int ctx_length = request.ContextLength;
+ int snp_length = request.SnippetLength;
if (queryable == null) {
Log.Error ("SnippetExecutor: No queryable object matches '{0}'", request.Hit.Source);
- snippet_reader = new SnippetReader (null, null, false);
+ snippet_reader = new SnippetReader (null, null, false, -1, -1);
full_text = false;
} else
- snippet_reader = queryable.GetSnippet (request.QueryTerms, request.Hit, full_text);
+ snippet_reader = queryable.GetSnippet (request.QueryTerms, request.Hit, full_text, ctx_length, snp_length);
return new SnippetResponse (new SnippetList (full_text, snippet_reader));
}
Modified: branches/beagle-rdf/beagled/SnippetFu.cs
==============================================================================
--- branches/beagle-rdf/beagled/SnippetFu.cs (original)
+++ branches/beagle-rdf/beagled/SnippetFu.cs Mon Mar 24 21:09:25 2008
@@ -38,7 +38,7 @@
public class SnippetFu {
- static public SnippetReader GetSnippet (string[] query_terms, TextReader line_reader, bool full_text)
+ static public SnippetReader GetSnippet (string[] query_terms, TextReader line_reader, bool full_text, int ctx_length, int snp_length)
{
// FIXME: If the query doesn't have search text (or is null), we should
// generate a 'summary snippet'.
@@ -46,24 +46,24 @@
if (line_reader == null)
return null;
- SnippetReader snippet_reader = new SnippetReader (line_reader, query_terms, full_text);
+ SnippetReader snippet_reader = new SnippetReader (line_reader, query_terms, full_text, ctx_length, snp_length);
return snippet_reader;
}
- static public SnippetReader GetSnippetFromFile (string[] query_terms, string filename, bool full_text)
+ static public SnippetReader GetSnippetFromFile (string[] query_terms, string filename, bool full_text, int ctx_length, int snp_length)
{
FileStream stream = new FileStream (filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
- return GetSnippet (query_terms, new StreamReader (stream), full_text);
+ return GetSnippet (query_terms, new StreamReader (stream), full_text, ctx_length, snp_length);
}
- static public SnippetReader GetSnippetFromTextCache (string[] query_terms, string filename, bool full_text)
+ static public SnippetReader GetSnippetFromTextCache (string[] query_terms, string filename, bool full_text, int ctx_length, int snp_length)
{
TextReader reader = TextCache.UserCache.GetReader (new Uri (filename));
if (reader == null)
return null;
try {
- return GetSnippet (query_terms, reader, full_text);
+ return GetSnippet (query_terms, reader, full_text, ctx_length, snp_length);
} catch (ICSharpCode.SharpZipLib.SharpZipBaseException ex) {
Log.Debug ("Unexpected exception '{0}' while extracting snippet for {1}", ex.Message, filename);
return null;
@@ -127,19 +127,24 @@
// Keep a sliding window of the starting positions of words
SlidingWindow sliding_window;
- const int between_snippet_words = 6;
- const int soft_snippet_limit = 200;
+ private const int context_length_default = 6;
+ private const int snippet_length_default = 200;
- public SnippetReader (TextReader line_reader, string[] query_terms, bool full_text)
+ private int context_length;
+ private int snippet_length;
+
+ public SnippetReader (TextReader line_reader, string[] query_terms, bool full_text, int context_length, int snippet_length)
{
this.line_reader = line_reader;
this.found_snippet_length = 0;
this.full_text = full_text;
+ this.context_length = (context_length > 0 ? context_length : context_length_default);
+ this.snippet_length = (snippet_length > 0 ? snippet_length : snippet_length_default);
if (query_terms == null)
return;
- this.sliding_window = new SlidingWindow (between_snippet_words);
+ this.sliding_window = new SlidingWindow (this.context_length);
// remove stop words from query_terms
query_terms_list = new ArrayList (query_terms.Length);
@@ -182,7 +187,7 @@
SnippetLine snippet_line;
ulong line = 0;
- while (found_snippet_length < soft_snippet_limit) {
+ while (found_snippet_length < snippet_length) {
//Console.WriteLine ("Continue with last line ? {0}", continue_line);
if (! continue_line) {
try {
@@ -273,7 +278,7 @@
// We cache the token, so as to avoid stemming it more than once
// when considering multiple terms.
if (stemmed_token == null) {
- stemmed_token = LuceneCommon.Stem (token);
+ stemmed_token = LuceneCommon.Stem (token.ToLower ());
}
if (String.Compare ((string) stemmed_terms [i], stemmed_token, true) != 0)
@@ -307,7 +312,7 @@
// Add the start pos of the token to the window
sliding_window.Add (pos);
// If we found a match previously and saw enough following words, stop
- if (snippet_line != null && snippet_line.Count > 0 && sliding_window.Count == between_snippet_words) {
+ if (snippet_line != null && snippet_line.Count > 0 && sliding_window.Count == context_length) {
sliding_window.Reset ();
string after_match = text.Substring (prev_match_end_pos, end_pos - prev_match_end_pos);
snippet_line.AddNonMatchFragment (after_match);
Modified: branches/beagle-rdf/beagled/SnippetTest.cs
==============================================================================
--- branches/beagle-rdf/beagled/SnippetTest.cs (original)
+++ branches/beagle-rdf/beagled/SnippetTest.cs Mon Mar 24 21:09:25 2008
@@ -45,7 +45,7 @@
SnippetReader snippet_reader;
// FIXME: Oops ... does not quit by passing empty line
- snippet_reader = SnippetFu.GetSnippet (args, Console.In, false); //delegate {
+ snippet_reader = SnippetFu.GetSnippet (args, Console.In, false, -1, -1); //delegate {
bool first = true;
foreach (SnippetLine snippet_line in snippet_reader.GetSnippet ()) {
Modified: branches/beagle-rdf/beagled/StaticQueryable.cs
==============================================================================
--- branches/beagle-rdf/beagled/StaticQueryable.cs (original)
+++ branches/beagle-rdf/beagled/StaticQueryable.cs Mon Mar 24 21:09:25 2008
@@ -53,7 +53,7 @@
}
}
- override public ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text)
+ override public ISnippetReader GetSnippet (string[] query_terms, Hit hit, bool full_text, int ctx_length, int snp_length)
{
if (text_cache == null)
return null;
@@ -63,7 +63,7 @@
if (reader == null)
return null;
- return SnippetFu.GetSnippet (query_terms, reader, full_text);
+ return SnippetFu.GetSnippet (query_terms, reader, full_text, ctx_length, snp_length);
}
override protected bool HitIsValid (Uri uri)
Modified: branches/beagle-rdf/conf-data/crawl-rules/crawl-manpages
==============================================================================
--- branches/beagle-rdf/conf-data/crawl-rules/crawl-manpages (original)
+++ branches/beagle-rdf/conf-data/crawl-rules/crawl-manpages Mon Mar 24 21:09:25 2008
@@ -2,7 +2,7 @@
CRAWL_INDEX_NAME="manpages"
-CRAWL_PATHS="/usr/share/man/man1 /usr/share/man/man2 /usr/share/man/man3 /usr/share/man/man4 /usr/share/man/man5 /usr/share/man/man6 /usr/share/man/man7 /usr/share/man/man8 /usr/share/man/man9 /usr/share/man/mann"
+CRAWL_PATHS="/usr/share/man/man1 /usr/share/man/man2 /usr/share/man/man3 /usr/share/man/man4 /usr/share/man/man5 /usr/share/man/man6 /usr/share/man/man7 /usr/share/man/man8 /usr/share/man/man9 /usr/share/man/mann /usr/share/info"
CRAWL_RECURSIVE="yes"
Modified: branches/beagle-rdf/conf-data/query-mapping.xml
==============================================================================
--- branches/beagle-rdf/conf-data/query-mapping.xml (original)
+++ branches/beagle-rdf/conf-data/query-mapping.xml Mon Mar 24 21:09:25 2008
@@ -13,8 +13,6 @@
<Mapping QueryKeyword="imagecomment" BeagleProperty="digikam:caption" Tokenize="true">Digikam User comments</Mapping>
<Mapping QueryKeyword="imagecomment" BeagleProperty="fspot:Description" Tokenize="true">F-Spot User comments</Mapping>
<Mapping QueryKeyword="speakingto" BeagleProperty="fixme:speakingto" Tokenize="false">Person engaged in conversation</Mapping>
- <Mapping QueryKeyword="speakingto" BeagleProperty="fixme:speakingto" Tokenize="false">Person engaged in conversation</Mapping>
- <Mapping QueryKeyword="speakingto" BeagleProperty="fixme:speakingto" Tokenize="false">Person engaged in conversation</Mapping>
<Mapping QueryKeyword="mailtoaddr" BeagleProperty="parent:fixme:to_address" Tokenize="true" />
<Mapping QueryKeyword="mailtoaddr" BeagleProperty="fixme:to_address" Tokenize="true">Email address of receipient</Mapping>
<Mapping QueryKeyword="inattachment" BeagleProperty="parent:fixme:hasAttachments" Tokenize="false">Use 'inattachment:true' for email attachments.</Mapping>
@@ -40,5 +38,7 @@
<Mapping QueryKeyword="pkggroup" BeagleProperty="pkg:group" Tokenize="true">Group to which the package belongs e.g. System/Configuration/Packaging</Mapping>
<Mapping QueryKeyword="inarchive" BeagleProperty="fixme:inside_archive" Tokenize="false">Use 'inarchive:true' for files inside an archive.</Mapping>
<Mapping QueryKeyword="album" BeagleProperty="fixme:album" Tokenize="true">Album name of the music</Mapping>
+ <Mapping QueryKeyword="filename" BeagleProperty="beagle:NoPunctFilename" Tokenize="true">File name</Mapping>
+ <Mapping QueryKeyword="filename" BeagleProperty="beagle:SplitFilename" Tokenize="true">File name</Mapping>
</Mappings>
</QueryMapping>
Modified: branches/beagle-rdf/configure.in
==============================================================================
--- branches/beagle-rdf/configure.in (original)
+++ branches/beagle-rdf/configure.in Mon Mar 24 21:09:25 2008
@@ -4,7 +4,7 @@
AC_INIT(beagled/BeagleDaemon.cs)
AC_CANONICAL_SYSTEM
-AM_INIT_AUTOMAKE(beagle, 0.3.3)
+AM_INIT_AUTOMAKE(beagle, 0.3.4)
########################################################################
@@ -66,10 +66,19 @@
AC_MSG_RESULT([found])
fi
+BEAGLE_DEFINES=""
+
# check that we have the require version of mono
-PKG_CHECK_MODULES(MONO, mono >= $MONO_REQUIRED)
-BEAGLE_DEFINES=""
+# Temporary: check for mono-1.9
+PKG_CHECK_MODULES(MONO, mono >= 1.9, mono_1_9=yes, mono_1_9=no)
+if test "x$mono_1_9" = "xno"; then
+ AC_MSG_RESULT([missing mono >= 1.9. Searching for mono >= 1.2.4])
+ PKG_CHECK_MODULES(MONO, mono >= $MONO_REQUIRED)
+else
+ AC_MSG_RESULT([found mono >= 1.9])
+ BEAGLE_DEFINES="$BEAGLE_DEFINES -define:MONO_1_9"
+fi
# check for various mono DLLs that we need.
needed_dlls="Mono.Data.Sqlite Mono.Posix System.Data System.Web ICSharpCode.SharpZipLib"
@@ -224,7 +233,12 @@
])
AC_SUBST(LIBTRAYICON_CFLAGS)
AC_SUBST(LIBTRAYICON_LIBS)
-
+
+ AC_PATH_PROG(MSGFMT, msgfmt, no)
+ if test "x$MSGFMT" = "xno"; then
+ AC_MSG_ERROR([You need to install gettext, or pass --disable-gui to build without GUI support])
+ fi
+
fi
dnl ----------------------------------------------
@@ -399,14 +413,6 @@
GMIME_SHARP_PREFIX=`$PKG_CONFIG --variable=prefix gmime-sharp`
dnl ----------------------------------------------
-AC_ARG_ENABLE([bludgeon],
- AC_HELP_STRING([--disable-bludgeon], [Disable bludgeon testing tool]),
- enable_bludgeon=$enableval,
- enable_bludgeon=yes)
-AM_CONDITIONAL(ENABLE_BLUDGEON, test "x$enable_bludgeon" = "xyes")
-
-
-dnl ----------------------------------------------
dnl Epiphany Extension
AC_ARG_ENABLE([epiphany-extension],
@@ -633,8 +639,6 @@
thunderbird-extension/Makefile
doc/Makefile
doc/api/Makefile
-bludgeon/Makefile
-RDFAdapter/Makefile
conf-data/Makefile
beagle-0.0.pc
beagle-daemon.pc
Modified: branches/beagle-rdf/search/Beagle.Search/Driver.cs
==============================================================================
--- branches/beagle-rdf/search/Beagle.Search/Driver.cs (original)
+++ branches/beagle-rdf/search/Beagle.Search/Driver.cs Mon Mar 24 21:09:25 2008
@@ -124,7 +124,7 @@
if (!String.IsNullOrEmpty (query) || !icon_enabled)
search.Query (query);
- Bus.Session.Register (new ObjectPath (PATH_NAME), search);
+ Bus.Session.Register (BUS_NAME, new ObjectPath (PATH_NAME), search);
program.Run ();
}
Modified: branches/beagle-rdf/search/Makefile.am
==============================================================================
--- branches/beagle-rdf/search/Makefile.am (original)
+++ branches/beagle-rdf/search/Makefile.am Mon Mar 24 21:09:25 2008
@@ -42,7 +42,7 @@
$(srcdir)/Beagle.Search.Tiles/Audio.cs \
$(srcdir)/Beagle.Search.Tiles/CApplet.cs \
$(srcdir)/Beagle.Search.Tiles/Calendar.cs \
- $(srcdir)/Beagle.Search.Tiles/Contact.cs \
+ $(srcdir)/Beagle.Search.Tiles/Contact.cs \
$(srcdir)/Beagle.Search.Tiles/Docbook.cs \
$(srcdir)/Beagle.Search.Tiles/File.cs \
$(srcdir)/Beagle.Search.Tiles/Folder.cs \
Modified: branches/beagle-rdf/tools/Query.cs
==============================================================================
--- branches/beagle-rdf/tools/Query.cs (original)
+++ branches/beagle-rdf/tools/Query.cs Mon Mar 24 21:09:25 2008
@@ -69,8 +69,12 @@
(lastQueryTime - queryStartTime).TotalSeconds);
}
- if (verbose && response.NumMatches >= 0)
- Console.WriteLine ("Returned latest {0} results out of total {1} matches", response.Hits.Count, response.NumMatches);
+ if (verbose) {
+ if (response.NumMatches >= 0)
+ Console.WriteLine ("Returned latest {0} results out of total {1} matches", response.Hits.Count, response.NumMatches);
+ else
+ Console.WriteLine ("Returned latest {0} results", response.Hits.Count);
+ }
if (! display_hits) {
count += response.Hits.Count;
Modified: branches/beagle-rdf/tools/StaticQuery.cs
==============================================================================
--- branches/beagle-rdf/tools/StaticQuery.cs (original)
+++ branches/beagle-rdf/tools/StaticQuery.cs Mon Mar 24 21:09:25 2008
@@ -147,13 +147,15 @@
Queryable queryable = QueryDriver.GetQueryable (request.Hit.Source);
ISnippetReader snippet_reader;
bool full_text = request.FullText;
+ int ctx_length = request.ContextLength;
+ int snp_length = request.SnippetLength;
if (queryable == null) {
Console.WriteLine ("SnippetExecutor: No queryable object matches '{0}'", request.Hit.Source);
- snippet_reader = new SnippetReader (null, null, false);
+ snippet_reader = new SnippetReader (null, null, false, -1, -1);
full_text = false;
} else
- snippet_reader = queryable.GetSnippet (request.QueryTerms, request.Hit, full_text);
+ snippet_reader = queryable.GetSnippet (request.QueryTerms, request.Hit, full_text, ctx_length, snp_length);
List<SnippetLine> snippetlines = new List<SnippetLine> ();
if (snippet_reader == null)
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
