r7286 - in firehose: . firehose firehose/jobs

[commits mugshot org]

Author: walters
Date: 2008-02-05 14:29:29 -0600 (Tue, 05 Feb 2008)
New Revision: 7286

Modified:
   firehose/dev.cfg
   firehose/firehose/controllers.py
   firehose/firehose/jobs/master.py
   firehose/firehose/jobs/poller.py
Log:
Flesh out task addition.  Choose another port to not conflict with jboss.


Modified: firehose/dev.cfg
===================================================================
--- firehose/dev.cfg	2008-02-05 00:02:59 UTC (rev 7285)
+++ firehose/dev.cfg	2008-02-05 20:29:29 UTC (rev 7286)
@@ -26,7 +26,7 @@
 # SERVER
 
 # Some server parameters that you may want to tweak
-# server.socket_port=8080
+server.socket_port=6676
 
 # Enable the debug output at the end on pages.
 # log_debug_info_filter.on = False

Modified: firehose/firehose/controllers.py
===================================================================
--- firehose/firehose/controllers.py	2008-02-05 00:02:59 UTC (rev 7285)
+++ firehose/firehose/controllers.py	2008-02-05 20:29:29 UTC (rev 7286)
@@ -29,6 +29,27 @@
         return {}
     
     @expose("json")
+    def settasks(self):
+        if cherrypy.request.method != 'POST':
+            raise Exception("Must invoke this method using POST")        
+        from firehose.jobs.master import MasterPoller        
+        master = MasterPoller.get()        
+        tasks = simplejson.load(cherrypy.request.body)
+        _logger.debug("got tasks: %r", tasks)
+        master.set_tasks(tasks['tasks'])
+        return {}
+        
+    @expose("json")
+    def addtasks(self):
+        if cherrypy.request.method != 'POST':
+            raise Exception("Must invoke this method using POST")        
+        from firehose.jobs.master import MasterPoller        
+        master = MasterPoller.get()        
+        tasks = simplejson.load(cherrypy.request.body)
+        master.add_tasks(tasks['tasks']) 
+        return {}        
+    
+    @expose("json")
     def requeue(self):
         if cherrypy.request.method != 'POST':
             raise Exception("Must invoke this method using POST")
@@ -45,4 +66,4 @@
         master = MasterPoller.get()
         status = simplejson.load(cherrypy.request.body)
         master.taskset_status(status)
-        return {}                
\ No newline at end of file
+        return {}
\ No newline at end of file

Modified: firehose/firehose/jobs/master.py
===================================================================
--- firehose/firehose/jobs/master.py	2008-02-05 00:02:59 UTC (rev 7285)
+++ firehose/firehose/jobs/master.py	2008-02-05 20:29:29 UTC (rev 7286)
@@ -111,12 +111,22 @@
         self.add_task(taskkey)
     
     def add_task(self, taskkey):
-        _logger.debug("adding task key=%r", taskkey)
-        self.__add_task_for_key(taskkey)
+        self.add_tasks([taskkey])
+        
+    def set_tasks(self, taskkeys):
+        # For now we don't support resetting the list; just append
+        self.add_tasks(taskkeys)
+    
+    def add_tasks(self, taskkeys):
+        _logger.debug("adding %d task keys", len(taskkeys))
+        for taskkey in taskkeys:
+            self.__add_task_for_key(taskkey)
         try:
             conn = sqlite3.connect(self.__path, isolation_level=None)        
             cursor = conn.cursor()
-            self.__set_task_status(cursor, taskkey, None, None)
+            for taskkey in taskkeys:
+                cursor.execute('''INSERT OR IGNORE INTO Tasks VALUES (?, ?, ?)''',
+                       (taskkey, None, None))
         finally:
             conn.close()
         

Modified: firehose/firehose/jobs/poller.py
===================================================================
--- firehose/firehose/jobs/poller.py	2008-02-05 00:02:59 UTC (rev 7285)
+++ firehose/firehose/jobs/poller.py	2008-02-05 20:29:29 UTC (rev 7286)
@@ -32,10 +32,10 @@
         raise NotImplementedError() 
 
 class FeedTaskHandler(object):
-    FAMILY = 'feed'
+    FAMILY = 'FEED'
     
     def run(self, id, prev_hash, prev_timestamp):
-        targeturl = urllib.unquote(id)
+        targeturl = id
         parsedurl = urlparse.urlparse(targeturl)
         try:
             _logger.info('Connecting to %r', targeturl)