[gnome-hwtest] Upload an error report on failure

[Owen Taylor <otaylor src gnome org>]

commit baf63c39f876ed0c7cd6b33a1ab72a2373aaac4d
Author: Owen W. Taylor <otaylor fishsoup net>
Date:   Tue Sep 23 10:38:46 2014 -0400

    Upload an error report on failure
    
    If an update or test job fails, upload an error report to perf.gnome.org,
    including the syslog exported from the test machine.

 js/controller.js |   91 ++++++++++++++++++++++++++++++++++++++++++++++++-----
 js/jobrunner.js  |    9 +++--
 js/upload.js     |   70 ++++++++++++++++++++++++++++-------------
 3 files changed, 134 insertions(+), 36 deletions(-)
---
diff --git a/js/controller.js b/js/controller.js
index 878a5b5..8777e85 100644
--- a/js/controller.js
+++ b/js/controller.js
@@ -298,9 +298,12 @@ const UpdateJob = Lang.Class({
 
         let listenerId = machine.logListener.connect('record', next);
 
+        this.log = [];
         try {
             while (true) {
                 let [, record] = yield;
+                this.log.push(record);
+
                 if (record['MESSAGE_ID'] == LogListener.SHUTDOWN_MESSAGE_ID)
                     break;
             }
@@ -327,7 +330,25 @@ const UpdateJob = Lang.Class({
             this.runner.addJob(new TestJob(this.runner, this.partition, this.ref, testset));
     },
 
-    stop: function() {
+    stop: function(message, exception) {
+        if (message) {
+            /* Since test reports are per-target (including a testset), we log an error in
+             * updating against each test-report separately. This is a bit silly, but
+             * since we expect normally the test to fail, not the update, this keeps
+             * the infrastructure simple.
+             */
+            for (let testset of Utils.values(this.partition.testsets)) {
+                this.runner.addJob(new ErrorUploadJob(this.runner,
+                                                      this.partition,
+                                                      this.ref,
+                                                      testset,
+                                                      "Update failed: " + message,
+                                                      this.log));
+            }
+        }
+
+        this.log = null;
+
         let machine = this.partition.machine;
         machine.powerOff();
         machine.state = Machine.STATE_INACTIVE;
@@ -378,9 +399,11 @@ const TestJob = Lang.Class({
 
         let metrics = [];
 
+        this.log = [];
         try {
             while (true) {
                 let [, record] = yield;
+                this.log.push(record);
 
                 if (record['MESSAGE_ID'] == LogListener.METRIC_MESSAGE_ID) {
                     let metric = {
@@ -413,13 +436,13 @@ const TestJob = Lang.Class({
         logger.info("%s: test succeeded", this);
 
         let uploader = Upload.getUploader(this.runner.config);
-        uploader.uploadAsync(this.partition,
-                             this.partition.tree,
-                             this.testset,
-                             this.ref,
-                             new Date(),
-                             metrics,
-                             cancellable, next)
+        uploader.uploadSuccessAsync(this.partition,
+                                    this.partition.tree,
+                                    this.testset,
+                                    this.ref,
+                                    new Date(),
+                                    metrics,
+                                    cancellable, next);
 
         let [, result] = yield cancellable;
         uploader.uploadFinish(result);
@@ -427,7 +450,16 @@ const TestJob = Lang.Class({
         logger.info("%s: upload succeeded", this);
     },
 
-    stop: function() {
+    stop: function(message, exception) {
+        if (message)
+            this.runner.addJob(new ErrorUploadJob(this.runner,
+                                                  this.partition,
+                                                  this.ref,
+                                                  this.testset,
+                                                  "Test failed: " + message,
+                                                  this.log));
+        this.log = null;
+
         let machine = this.partition.machine;
         machine.powerOff();
         machine.state = Machine.STATE_INACTIVE;
@@ -435,6 +467,47 @@ const TestJob = Lang.Class({
     }
 });
 
+
+const ErrorUploadJob = Lang.Class({
+    Name: 'ErrorUploadJob',
+    Extends: JobRunner.Job,
+
+    _init: function(controller, partition, ref, testset, errorMessage, log) {
+        this.parent(controller);
+        this.partition = partition;
+        this.ref = ref;
+        this.testset = testset;
+        this.errorMessage = errorMessage;
+        this.log = log;
+    },
+
+    toString: function() {
+        return Format.vprintf("%s(%s/%s; ref=%s, testset=%s)",
+                              [this.__name__, this.partition.machine.name,
+                               this.partition.name, this.ref, this.testset]);
+    },
+
+    canRun: function() {
+        return true;
+    },
+
+    runAsync: function(cancellable, next) {
+        let uploader = Upload.getUploader(this.runner.config);
+        uploader.uploadErrorAsync(this.partition,
+                                  this.partition.tree,
+                                  this.testset,
+                                  this.ref,
+                                  new Date(),
+                                  this.errorMessage, this.log,
+                                  cancellable, next);
+
+        let [, result] = yield cancellable;
+        uploader.uploadFinish(result);
+
+        logger.info("%s: error upload succeeded", this);
+    }
+});
+
 const Controller = Lang.Class({
     Name: 'Controller',
     Extends: JobRunner.JobRunner,
diff --git a/js/jobrunner.js b/js/jobrunner.js
index 02cd73b..adcd501 100644
--- a/js/jobrunner.js
+++ b/js/jobrunner.js
@@ -68,14 +68,15 @@ const Job = new Lang.Class({
     },
 
     _asyncException: function(e) {
-        this.stop();
-
         if (e instanceof StopIteration) {
+            this.stop();
             this.logger.info("job %s: finished", this);
         } else if (e == AsyncCancel ||
                    e.matches(Gio.IOErrorEnum, Gio.IOErrorEnum.CANCELLED)) {
-            this.logger.error("job %s: cancelled", this);
+            this.stop("Timed out");
+            this.logger.error("job %s: timed out", this);
         } else {
+            this.stop(String(e), e);
             throw e;
         }
     },
@@ -87,7 +88,7 @@ const Job = new Lang.Class({
         return false;
     },
 
-    stop: function(keep) {
+    stop: function(message, exception) {
         this.running = false;
         this._cancellable = null;
         this._iterator = null;
diff --git a/js/upload.js b/js/upload.js
index 4d1dc8b..481190c 100644
--- a/js/upload.js
+++ b/js/upload.js
@@ -46,28 +46,20 @@ const Uploader = new Lang.Class({
         return "RSA-SHA256 "+ signature_base64;
     },
 
-    uploadAsync: function(partition, tree, testset, revision, pullTime, metrics, cancellable, callback) {
-        let pullTimeStr = Format.vprintf("%04d-%02d-%02d %02d:%02d:%02d",
-                                         [pullTime.getUTCFullYear(),
-                                          pullTime.getUTCMonth() + 1,
-                                          pullTime.getUTCDate(),
-                                          pullTime.getUTCHours(),
-                                          pullTime.getUTCMinutes(),
-                                          pullTime.getUTCSeconds()]);
-
-        let report = {
-            "machine": partition.machine.name,
-            "partition": partition.name,
-            "tree": tree,
-            "testset": testset,
-            "revision": revision,
-            "pullTime": pullTimeStr,
-            "metrics": metrics
-        }
+    _formatTime: function(time) {
+        return Format.vprintf("%04d-%02d-%02d %02d:%02d:%02d",
+                              [time.getUTCFullYear(),
+                               time.getUTCMonth() + 1,
+                               time.getUTCDate(),
+                               time.getUTCHours(),
+                               time.getUTCMinutes(),
+                               time.getUTCSeconds()]);
+    },
 
-        let url = this.config.perf_server_api + '/upload?machine=' + partition.machine.name;
-        let data  = JSON.stringify(report);
-        let privkeyfile = '/srv/gnome-hwtest/' + partition.machine.name + '.secret';
+    _uploadAsync: function(machine, jsonData, cancellable, callback) {
+        let url = this.config.perf_server_api + '/upload?machine=' + machine.name;
+        let data  = JSON.stringify(jsonData);
+        let privkeyfile = '/srv/gnome-hwtest/' + machine.name + '.secret';
 
         let signature = this._makeSignature(url, data, privkeyfile);
 
@@ -82,18 +74,50 @@ const Uploader = new Lang.Class({
         }.bind(this));
     },
 
+    uploadSuccessAsync: function(partition, tree, testset, revision, pullTime, metrics, cancellable, 
callback) {
+        let report = {
+            "machine": partition.machine.name,
+            "partition": partition.name,
+            "tree": tree,
+            "testset": testset,
+            "revision": revision,
+            "pullTime": this._formatTime(pullTime),
+            "metrics": metrics
+        }
+
+        this._uploadAsync(partition.machine, report, cancellable, callback);
+    },
+
+    uploadErrorAsync: function(partition, tree, testset, revision, pullTime, errorMessage, log, cancellable, 
callback) {
+        let report = {
+            "machine": partition.machine.name,
+            "partition": partition.name,
+            "tree": tree,
+            "testset": testset,
+            "revision": revision,
+            "pullTime": this._formatTime(pullTime),
+            "error": errorMessage
+        }
+
+        if (log != null) {
+            report['log'] = log;
+        }
+
+        this._uploadAsync(partition.machine, report, cancellable, callback);
+    },
+
     uploadFinish: function(result) {
         let stream = this.session.send_finish(result.soupResult);
         if (result.message.status_code != 200) {
             let output = stream.read_bytes(64*1024, null);
             Utils.stderr.write_bytes(output, null);
             stream.close(null);
-            throw new Error(Format.vprintf("Failed to post report : %d %s\n",
+            throw new Error(Format.vprintf("Failed to post to server : %d %s\n",
                                            [result.message.status_code, result.message.reason_phrase]));
         }
 
         stream.close(null);
-    }
+    },
 });
 
 function getUploader(config) {