beagle r4764 - trunk/beagle/Util

[dbera svn gnome org]

Author: dbera
Date: Sun May 25 16:41:30 2008
New Revision: 4764
URL: http://svn.gnome.org/viewvc/beagle?rev=4764&view=rev

Log:
* Scheduler.cs: Punish misbehaving tasks. Frustrated by the numerous large-log-file complains, allow at most 200 exceptions before a task is automatically cancelled. One part of my brain wants to disallow even a single exception, but ... to keep with existing behaviour I am allowing a large number of exceptions for long running generator tasks. Ration your exceptions.

Show the list of cancelled tasks in beagle-status.

Also deal with task cancellation correctly i.e. DoTaskReal might itself cancel a task so take that into consideration, do not schedule a cancelled task and some other such things.


Modified:
   trunk/beagle/Util/Scheduler.cs

Modified: trunk/beagle/Util/Scheduler.cs
==============================================================================
--- trunk/beagle/Util/Scheduler.cs	(original)
+++ trunk/beagle/Util/Scheduler.cs	Sun May 25 16:41:30 2008
@@ -191,6 +191,9 @@
 
 			public void Schedule (Scheduler scheduler)
 			{
+				if (this.cancelled)
+					return; // do not schedule a cancelled task
+
 				// Increment the task groups the first
 				// time a task is scheduled.
 				if (this.scheduler == null)
@@ -226,13 +229,32 @@
 
 			public void Cancel ()
 			{
+				Cancel (null);
+			}
+
+			public void Cancel (string reason)
+			{
 				if (! cancelled) {
+					AddToCancelledTaskList (reason);
 					DecrementAllTaskGroups ();
 					Cleanup (); // clean up after cancelled tasks
 				}
 				cancelled = true;
 			}
 
+			private void AddToCancelledTaskList (string reason)
+			{
+				string task_desc = String.Format ("Cancelled task: {5}\n"  +
+								  "        Tag: {0}\n" +
+								  "    Creator: {1}\n" +
+								  "Description: {2}\n" +
+								  "   Priority: {3} ({4})\n", 
+								  Tag, Creator, Description, Priority, SubPriority,
+								  (reason == null ? String.Empty : reason));
+				if (scheduler != null)
+					scheduler.ReportCancelledTask (task_desc);
+			}
+
 			///////////////////////////////
 
 			// The Task's count keeps track of how many
@@ -244,6 +266,18 @@
 				get { return count; }
 			}
 
+			// Keeps track of how many times the task was executed but
+			// there was some exception
+			private int misfires = 0;
+
+			// Allow at most this many exceptions per task
+			// This is a bit high since some tasks like the generators are
+			// long running tasks. This should be a high enough number
+			// to tolerate some exceptional cases but stop the task
+			// in case it went into an infinite loop with exceptions.
+			// Ideally all exceptions should be caught and handled downstream.
+			const int MAX_TASK_EXCEPTION = 200;
+
 			///////////////////////////////
 			
 			public void SpawnChild (Task child_task)
@@ -272,6 +306,7 @@
 					try {
 						DoTaskReal ();
 					} catch (Exception ex) {
+						misfires ++;
 						Logger.Log.Warn (ex,
 								 "Caught exception in DoTaskReal\n" +
 								 "        Tag: {0}\n" +
@@ -279,12 +314,23 @@
 								 "Description: {2}\n" +
 								 "   Priority: {3} ({4})", 
 								 Tag, Creator, Description, Priority, SubPriority);
+						if (misfires >= MAX_TASK_EXCEPTION) {
+							Log.Warn ("More than {5} exceptions in DoTaskReal. Disabling further execution of task:\n" +
+								 "        Tag: {0}\n" +
+								 "    Creator: {1}\n" +
+								 "Description: {2}\n" +
+								 "   Priority: {3} ({4})", 
+								 Tag, Creator, Description, Priority, SubPriority, MAX_TASK_EXCEPTION);
+							Cancel ("Exceptions in DoTaskReal");
+						}
 					}
 					sw.Stop ();
 					if (Debug)
 						Logger.Log.Debug ("Finished task {0} in {1}", Tag, sw);
 
-					if (Reschedule) {
+					if (cancelled) {
+						return;
+					} else if (Reschedule) {
 						++count;
 						if (Debug)
 							Log.Debug ("Rescheduling task {0}", Tag);
@@ -311,6 +357,9 @@
 					DoCleanup ();
 				} catch (Exception ex) {
 					Logger.Log.Warn (ex, "Caught exception cleaning up task '{0}'", Tag);
+				} finally {
+					Reschedule = false;
+					scheduler = null;
 				}
 			}
 
@@ -557,7 +606,7 @@
 
 		private Hashtable tasks_by_tag = new Hashtable ();
 		private int total_executed_task_count = 0;
-		
+
 		public void Add (Task task)
 		{
 			if (task == null)
@@ -1041,6 +1090,18 @@
 		
 		//////////////////////////////////////////////////////////////////////////////
 
+		// A list of descriptions of cancelled tasks
+		// Kept for display purposes; it is useful to know
+		// which tasks were cancelled and why.
+		private ArrayList cancelled_tasks = new ArrayList ();
+
+		internal void ReportCancelledTask (string description)
+		{
+			cancelled_tasks.Add (description);
+		}
+
+		//////////////////////////////////////////////////////////////////////////////
+
 		private static StringBuilder cached_sb = new StringBuilder ();
 		
 		public SchedulerInformation GetCurrentStatus ()
@@ -1090,6 +1151,11 @@
 					current_status.BlockedTasks.Add (cached_sb.ToString ());
 				}
 
+				// Add the cancelled tasks to blocked_task list for the time being
+				// This is avoid ABI change in libbeagle
+				foreach (string description in cancelled_tasks)
+					current_status.BlockedTasks.Add (description);
+
 				current_status.TotalTaskCount = total_executed_task_count;
 				current_status.StatusString = status_str;