r7491 - dumbhippo/trunk/firehose/firehose

[commits mugshot org]

Author: walters
Date: 2008-06-25 09:29:01 -0500 (Wed, 25 Jun 2008)
New Revision: 7491

Added:
   dumbhippo/trunk/firehose/firehose/logsToLogstats.groovy
   dumbhippo/trunk/firehose/firehose/s3SyncLogs.py
Log:
Scripts for reducing and syncing logs to S3


Added: dumbhippo/trunk/firehose/firehose/logsToLogstats.groovy
===================================================================
--- dumbhippo/trunk/firehose/firehose/logsToLogstats.groovy	2008-06-25 14:27:28 UTC (rev 7490)
+++ dumbhippo/trunk/firehose/firehose/logsToLogstats.groovy	2008-06-25 14:29:01 UTC (rev 7491)
@@ -0,0 +1,32 @@
+# This script processes log files named "server.log.$date" into 
+# files named logstats.$date
+# Usage: groovy -cp /usr/share/java/commons-io.jar /home/cluster/dhdeploy/firehose/logs
+# Intended to be run daily-ish, preferably after logs are rotated
+
+import org.apache.commons.io.IOUtils
+
+println "Scanning ${args[0]}..."
+def logdir = new File(args[0])
+
+def logPrefix = "server.log."
+def logfiles = logdir.list({ d,f -> f.startsWith(logPrefix) } as FilenameFilter)
+logfiles.each { name ->
+  def path = new File(logdir, name)
+  def outName = "logstats." + name.substring(logPrefix.length())
+  def outPath = new File(logdir, outName)
+  
+  if (outPath.exists())
+    return;  
+ 
+  def outtmp = File.createTempFile("logstats", ".tmp", logdir)
+  def outTmpStream = new BufferedOutputStream(new FileOutputStream(outtmp))
+  def procBuilder = new ProcessBuilder(new File(".", "logstats.groovy").toString(), path.toString());
+  def proc = procBuilder.start()
+  IOUtils.copy(proc.getInputStream(), outTmpStream)
+  proc.waitFor()
+  outTmpStream.close()
+
+  println "Renaming to ${outPath}"
+  outtmp.renameTo(outPath)
+}
+println "done."
\ No newline at end of file

Added: dumbhippo/trunk/firehose/firehose/s3SyncLogs.py
===================================================================
--- dumbhippo/trunk/firehose/firehose/s3SyncLogs.py	2008-06-25 14:27:28 UTC (rev 7490)
+++ dumbhippo/trunk/firehose/firehose/s3SyncLogs.py	2008-06-25 14:29:01 UTC (rev 7491)
@@ -0,0 +1,31 @@
+#!/usr/bin/python
+
+import os,sys,time
+
+from boto.s3.connection import S3Connection
+from boto.s3.key import Key
+
+logdir = sys.argv[1]
+
+# Read in AWS credentials
+execfile(os.path.expanduser('~/.aws'))
+s3Conn = S3Connection()
+logStorageBucket = s3Conn.get_bucket('logs.firehose.svc.verbum.org')
+
+# one week
+timedeltaSecs = 7 * 24 * 60 * 60
+
+curtime = time.time()
+for name in os.listdir(logdir):
+  path = os.path.join(logdir, name)
+  stbuf = os.stat(path)
+  mtime = stbuf.st_mtime
+  if (curtime - mtime) < timedeltaSecs:
+    continue
+  k = Key(logStorageBucket)
+  k.key = name
+  print "Uploading %s" % (path,)
+  k.set_contents_from_filename(path)
+  print "Upload complete, unlinking %s" % (path,)
+  os.unlink(path)
+print "done"


Property changes on: dumbhippo/trunk/firehose/firehose/s3SyncLogs.py
___________________________________________________________________
Name: svn:executable
   + *