[gnome-ostree] Add support for delayed tasks, use it for zdisks

[Colin Walters <walters src gnome org>]

commit d41664c7d28cdf8041cac9592625814d0173456c
Author: Colin Walters <walters verbum org>
Date:   Tue Apr 16 12:38:57 2013 -0400

    Add support for delayed tasks, use it for zdisks
    
    We don't want to be constantly generating new zdisks (compresed disk
    images for download).  A good heuristic would be something like "at
    most once every 2 hours".
    
    This commit adds the concept of a "delayed" task to implement that;
    zdisks now runs after smoketest, at most once an hour.

 Makefile-ostbuild.am               |    2 +
 src/js/builtins/make.js            |   61 +++++++++++++++++++++++++++++----
 src/js/task.js                     |   57 +++++++++++++++++++++++++++++--
 src/js/tasks/task-dummy-delayed.js |   46 ++++++++++++++++++++++++++
 src/js/tasks/task-dummy-resolve.js |   64 ++++++++++++++++++++++++++++++++++++
 src/js/tasks/task-zdisks.js        |    4 +-
 6 files changed, 220 insertions(+), 14 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index c6810c1..278dfd4 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -78,6 +78,8 @@ jsosttasks_DATA= \
        src/js/tasks/task-integrationtest.js \
        src/js/tasks/task-zdisks.js \
        src/js/tasks/testbase.js \
+       src/js/tasks/task-dummy-resolve.js \
+       src/js/tasks/task-dummy-delayed.js \
        $(NULL)
 
 endif
diff --git a/src/js/builtins/make.js b/src/js/builtins/make.js
index df6999d..6233584 100644
--- a/src/js/builtins/make.js
+++ b/src/js/builtins/make.js
@@ -44,6 +44,8 @@ const Make = new Lang.Class({
                                                    help: "Don't process tasks after this" });
        this.parser.addArgument(['-x', '--skip'], { action: 'append',
                                                    help: "Don't process tasks after this" });
+       this.parser.addArgument(['--shell'], { action: 'storeTrue',
+                                              help: "Start an interactive shell" });
        this.parser.addArgument('taskname');
        this.parser.addArgument('parameters', { nargs: '*' });
     },
@@ -52,6 +54,10 @@ const Make = new Lang.Class({
        this._initWorkdir(null, cancellable);
        this._loop = loop;
        this._failed = false;
+       this._shell = false;
+       this._cancellable = cancellable;
+       this._shellComplete = false;
+       this._tasksComplete = false;
        this._oneOnly = args.only;
        let taskmaster = new Task.TaskMaster(this.workdir.get_child('tasks'),
                                             { onEmpty: Lang.bind(this, this._onTasksComplete),
@@ -60,9 +66,24 @@ const Make = new Lang.Class({
        this._taskmaster = taskmaster;
        taskmaster.connect('task-executing', Lang.bind(this, this._onTaskExecuting));
        taskmaster.connect('task-complete', Lang.bind(this, this._onTaskCompleted));
+       let params = this._parseParameters(args.parameters);
+       taskmaster.pushTask(args.taskname, params);
+       if (args.shell) {
+           this._stdin = new Gio.DataInputStream({ base_stream: GSystem.Console.get_stdin() });
+           this._stdin.read_line_async(GLib.PRIORITY_DEFAULT, cancellable,
+                                       Lang.bind(this, this._onStdinRead));
+           this._shell = true;
+       }
+       this._console = GSystem.Console.get();
+       loop.run();
+       if (!this._failed)
+           print("Success!")
+    },
+
+    _parseParameters: function(paramStrings) {
        let params = {};
-       for (let i = 0; i < args.parameters.length; i++) { 
-           let param = args.parameters[i];
+       for (let i = 0; i < paramStrings.length; i++) { 
+           let param = paramStrings[i];
            let idx = param.indexOf('=');
            if (idx == -1)
                throw new Error("Invalid key=value syntax");
@@ -70,11 +91,31 @@ const Make = new Lang.Class({
            let v = JSON.parse(param.substr(idx+1));
            params[k] = v;
        }
-       taskmaster.pushTask(args.taskname, params);
-       this._console = GSystem.Console.get();
-       loop.run();
-       if (!this._failed)
-           print("Success!")
+       return params;
+    },
+
+    _onStdinRead: function(stdin, result) {
+       let cancellable = this._cancellable;
+       let [line, len] = stdin.read_line_finish_utf8(result);
+       let args = line.split(' ');
+       if (args.length > 1) {
+           let cmd = args[0];
+           let params = null; 
+           try {
+               params = this._parseParameters(args.slice(1));
+           } catch (e) {
+               print(e);
+           }
+           if (params !== null) {
+               this._taskmaster.pushTask(cmd, params);
+           }
+           this._stdin.read_line_async(GLib.PRIORITY_DEFAULT, cancellable,
+                                       Lang.bind(this, this._onStdinRead));
+       } else if (line == "") {
+           this._shellComplete = true;
+           if (this._tasksComplete)
+               this._loop.quit();
+       }
     },
 
     _onTaskExecuting: function(taskmaster, task) {
@@ -100,8 +141,12 @@ const Make = new Lang.Class({
     },
 
     _onTasksComplete: function(success, err) {
+       if (this._shell && !this._shellComplete)
+           return;
+       this._tasksComplete = true;
        if (!success)
            this._err = err;
-       this._loop.quit();
+       if (this._shellComplete)
+           this._loop.quit();
     }
 });
diff --git a/src/js/task.js b/src/js/task.js
index 54d1ad2..38c6518 100644
--- a/src/js/task.js
+++ b/src/js/task.js
@@ -119,17 +119,65 @@ const TaskMaster = new Lang.Class({
        this._taskset = TaskSet.prototype.getInstance();
 
        this._taskVersions = {};
+
+       // string -> [ lastExecutedSecs, [timeoutId, parameters]]
+       this._scheduledTaskTimeouts = {};
+    },
+
+    _pushTaskDefImmediate: function(taskDef, parameters) {
+       let name = taskDef.prototype.TaskName;
+       let instance = new taskDef(this, name, [], parameters);
+       instance.onComplete = Lang.bind(this, this._onComplete, instance);
+       this._pendingTasksList.push(instance);
+       this._queueRecalculate();
     },
 
     _pushTaskDef: function(taskDef, parameters) {
        let name = taskDef.prototype.TaskName;
        if (!this._isTaskPending(name)) {
-           let instance = new taskDef(this, name, [], parameters);
-           instance.onComplete = Lang.bind(this, this._onComplete, instance);
-           this._pendingTasksList.push(instance);
-           this._queueRecalculate();
+           let scheduleMinSecs = taskDef.prototype.TaskScheduleMinSecs;
+           if (scheduleMinSecs > 0) {
+               let info = this._scheduledTaskTimeouts[name];
+               if (!info) {
+                   info = [ 0, null ];
+                   this._scheduledTaskTimeouts[name] = info;
+               }
+               let lastExecutedSecs = info[0];
+               let pendingExecData = info[1];
+               let currentTime = GLib.get_monotonic_time() / GLib.USEC_PER_SEC;
+               if (pendingExecData != null) {
+                   // Nothing, already scheduled
+                    let delta = (lastExecutedSecs + scheduleMinSecs) - currentTime;
+                   print("Already scheduled task " + name + " remaining=" + delta);
+               } else if (lastExecutedSecs == 0) {
+                   print("Scheduled task " + name + " executing immediately");
+                   this._pushTaskDefImmediate(taskDef, parameters);
+                   info[0] = currentTime;
+                } else {
+                    let delta = (lastExecutedSecs + scheduleMinSecs) - currentTime;
+                   print("Scheduled task " + name + " delta=" + delta);
+                   let timeoutId = GLib.timeout_add_seconds(GLib.PRIORITY_DEFAULT,
+                                                            Math.max(delta, 0),
+                                                            Lang.bind(this, this._executeScheduledTask, 
taskDef));
+                   info[0] = currentTime;
+                   info[1] = [ timeoutId, parameters ];
+               }
+           } else {
+               this._pushTaskDefImmediate(taskDef, parameters);
+           }
        }
     },
+    
+    _executeScheduledTask: function(taskDef) {
+       let name = taskDef.prototype.TaskName;
+       print("Executing scheduled task " + name);
+       let currentTime = GLib.get_monotonic_time() / GLib.USEC_PER_SEC;
+       let info = this._scheduledTaskTimeouts[name];
+       info[0] = currentTime;
+       let params = info[1][1];
+       info[1] = null;
+       this._pushTaskDefImmediate(taskDef);
+    },
 
     pushTask: function(taskName, parameters) {
        let taskDef = this._taskset.getTask(taskName);
@@ -259,6 +307,7 @@ const TaskDef = new Lang.Class({
 
     TaskPattern: null,
     TaskAfter: [],
+    TaskScheduleMinSecs: 0,
 
     PreserveStdout: true,
     RetainFailed: 1,
diff --git a/src/js/tasks/task-dummy-delayed.js b/src/js/tasks/task-dummy-delayed.js
new file mode 100644
index 0000000..81c5eb2
--- /dev/null
+++ b/src/js/tasks/task-dummy-delayed.js
@@ -0,0 +1,46 @@
+// Copyright (C) 2011 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const JsonDB = imports.jsondb;
+const Builtin = imports.builtin;
+const Task = imports.task;
+const ProcUtil = imports.procutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const TaskDummyDelayed = new Lang.Class({
+    Name: "TaskDummyDelayed",
+    Extends: Task.TaskDef,
+
+    TaskName: "dummy-delayed",
+    TaskAfter: ['dummy-resolve'], 
+    TaskScheduleMinSecs: 5,
+
+    execute: function(cancellable) {
+       print("delayed firing!");
+    }
+});
diff --git a/src/js/tasks/task-dummy-resolve.js b/src/js/tasks/task-dummy-resolve.js
new file mode 100644
index 0000000..73b96e6
--- /dev/null
+++ b/src/js/tasks/task-dummy-resolve.js
@@ -0,0 +1,64 @@
+// Copyright (C) 2011 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const JsonDB = imports.jsondb;
+const Builtin = imports.builtin;
+const Task = imports.task;
+const ProcUtil = imports.procutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const TaskDummyResolve = new Lang.Class({
+    Name: "TaskDummyResolve",
+    Extends: Task.TaskDef,
+
+    TaskName: "dummy-resolve",
+
+    DefaultParameters: {change: false},
+
+    _getDb: function() {
+       if (this._db == null) {
+           let snapshotdir = this.workdir.get_child('dummy-resolve');
+           this._db = new JsonDB.JsonDB(snapshotdir);
+       }
+       return this._db;
+    },
+
+    queryVersion: function() {
+       return this._getDb().getLatestVersion();
+    },
+
+    execute: function(cancellable) {
+       let change = this.parameters.change;
+       let latest = this.queryVersion();
+       if (!latest)
+           change = true;
+       if (change) {
+            let [path, modified] = this._getDb().store({timestamp: GLib.get_real_time() }, cancellable);
+       }
+    }
+});
diff --git a/src/js/tasks/task-zdisks.js b/src/js/tasks/task-zdisks.js
index 4a0ad22..815f43e 100644
--- a/src/js/tasks/task-zdisks.js
+++ b/src/js/tasks/task-zdisks.js
@@ -41,8 +41,8 @@ const TaskZDisks = new Lang.Class({
     Extends: BuildDisks.TaskBuildDisks,
 
     TaskName: "zdisks",
-    TaskAfter: ['builddisks'],
-    TaskScheduleMinSecs: 60*60,  // Only do this once an hour
+    TaskAfter: ['smoketest'],
+    TaskScheduleMinSecs: 3*60*60,  // Only do this every 3 hours
 
     // Legacy
     _VERSION_RE: /^(\d+)\.(\d+)$/,