r7463 - dumbhippo/trunk/firehose/firehose

[commits mugshot org]

Author: walters
Date: 2008-04-30 16:57:15 -0500 (Wed, 30 Apr 2008)
New Revision: 7463

Added:
   dumbhippo/trunk/firehose/firehose/find_smalldiffs.py
   dumbhippo/trunk/firehose/firehose/gendiffs.py
Log:
Scripts for analyzing feeds from firehose


Added: dumbhippo/trunk/firehose/firehose/find_smalldiffs.py
===================================================================
--- dumbhippo/trunk/firehose/firehose/find_smalldiffs.py	2008-04-29 21:49:24 UTC (rev 7462)
+++ dumbhippo/trunk/firehose/firehose/find_smalldiffs.py	2008-04-30 21:57:15 UTC (rev 7463)
@@ -0,0 +1,25 @@
+#!/usr/bin/python
+
+import os,sys,subprocess
+
+import simplejson
+
+def main():
+    srcdir = sys.argv[1]
+    entries = [path for path in os.listdir(srcdir) if path.endswith('.diff')]
+    entries.sort()
+    entry_lines = {}
+    for path in entries:
+        c = 0
+        fullpath = os.path.join(srcdir, path)
+        f = open(fullpath)
+        for line in f:
+            c += 1
+        if c > 3:
+            c -= 3
+        if c > 0 and c < 13:
+            entry_lines[path] = c
+    simplejson.dump(entry_lines, sys.stdout)
+        
+if __name__ == '__main__':
+    main()
\ No newline at end of file

Added: dumbhippo/trunk/firehose/firehose/gendiffs.py
===================================================================
--- dumbhippo/trunk/firehose/firehose/gendiffs.py	2008-04-29 21:49:24 UTC (rev 7462)
+++ dumbhippo/trunk/firehose/firehose/gendiffs.py	2008-04-30 21:57:15 UTC (rev 7463)
@@ -0,0 +1,28 @@
+#!/usr/bin/python
+
+import os,sys,subprocess
+
+def main():
+    srcdir = sys.argv[1]
+    entries = os.listdir(srcdir)
+    entries.sort()
+    items = {}
+    for entry in entries:
+        if entry.endswith('.diff'):
+            continue
+        (key, ts) = entry.rsplit('.', 1)        
+        if key not in items:
+            items[key] = []
+        items[key].append(ts)
+    for key,tslist in items.iteritems():
+        for ts1, ts2 in zip(tslist[:-1], tslist[1:]):
+            ts1path = key + '.' + ts1
+            ts2path = key + '.' + ts2
+            diffpath = os.path.join(srcdir, '%s.%s-%s.diff' % (key, ts1, ts2))
+            if os.path.exists(diffpath):
+                continue
+            diff_f = open(diffpath, 'w')
+            subprocess.call(['diff', '-u', ts1path, ts2path], cwd=srcdir, stdout=diff_f, stderr=subprocess.STDOUT)
+    
+if __name__ == '__main__':
+    main()
\ No newline at end of file