[gnome-ostree] autobuilder: New builtin, also tweak repoweb to use it
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-ostree] autobuilder: New builtin, also tweak repoweb to use it
- Date: Sun, 21 Oct 2012 00:41:22 +0000 (UTC)
commit 0ef1dbba2cb64a0f19d1b5058291a86a0be40ca5
Author: Colin Walters <walters verbum org>
Date: Sat Oct 20 20:30:15 2012 -0400
autobuilder: New builtin, also tweak repoweb to use it
This new autobuilder outputs JSON that repoweb can read.
Makefile-ostbuild.am | 2 +
qa/repoweb/index.html | 17 ++-
qa/repoweb/repoweb.css | 9 +
qa/repoweb/repoweb.js | 125 ++++++++++-----
src/ostbuild/pyostbuild/builtin_autobuilder.py | 202 ++++++++++++++++++++++++
src/ostbuild/pyostbuild/builtins.py | 2 +
src/ostbuild/pyostbuild/main.py | 1 +
7 files changed, 312 insertions(+), 46 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 3169220..2b3b6aa 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -35,6 +35,7 @@ utilsdir = $(libdir)/ostbuild
pyostbuilddir=$(libdir)/ostbuild/pyostbuild
pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/buildutil.py \
+ src/ostbuild/pyostbuild/builtin_autobuilder.py \
src/ostbuild/pyostbuild/builtin_build.py \
src/ostbuild/pyostbuild/builtin_checkout.py \
src/ostbuild/pyostbuild/builtin_deploy_qemu.py \
@@ -50,6 +51,7 @@ pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/builtin_source_diff.py \
src/ostbuild/pyostbuild/builtins.py \
src/ostbuild/pyostbuild/filemonitor.py \
+ src/ostbuild/pyostbuild/task.py \
src/ostbuild/pyostbuild/fileutil.py \
src/ostbuild/pyostbuild/__init__.py \
src/ostbuild/pyostbuild/jsondb.py \
diff --git a/qa/repoweb/index.html b/qa/repoweb/index.html
index 3111a9b..424d272 100644
--- a/qa/repoweb/index.html
+++ b/qa/repoweb/index.html
@@ -3,10 +3,10 @@
<head>
<title>GNOME-OSTree</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
- <link rel="stylesheet" href="http://code.jquery.com/mobile/1.1.0/jquery.mobile-1.1.0.min.css"; />
+ <link rel="stylesheet" href="jquery.mobile-1.1.0.css" />
<link rel="stylesheet" href="repoweb.css"/>
- <script src="http://code.jquery.com/jquery-1.7.1.min.js";></script>
- <script src="http://code.jquery.com/mobile/1.1.0/jquery.mobile-1.1.0.min.js";></script>
+ <script src="jquery-1.7.1.js"></script>
+ <script src="jquery.mobile-1.1.0.js"></script>
<script src="repoweb.js"></script>
</head>
<body onload="$(document).ready(function(){repoweb_index_init();})">
@@ -24,14 +24,17 @@
the <a href="http://git.gnome.org/browse/gnome-ostree";>gnome-ostree</a>
git module.</p>
- <h3>Summary</h3>
- <div id="repoweb-summary">Loading...</div>
+ <h3>Resolve</h3>
+ <div id="resolve-summary">Loading...</div>
+
+ <h3>Build</h3>
+ <div id="build-summary">Loading...</div>
</div>
<div class="content-secondary">
<ul data-role="listview" data-theme="c" data-dividertheme="d">
<li data-role="list-divider">Tools</li>
- <li><a href="logs" rel="external">Build logs</a></li>
- <li><a href="src" rel="external">Source code</a></li>
+ <li><a href="work/tasks" rel="external">Build logs</a></li>
+ <li><a href="work/src" rel="external">Source code</a></li>
</ul>
</div>
</div> <!-- content -->
diff --git a/qa/repoweb/repoweb.css b/qa/repoweb/repoweb.css
index b1c657d..dd09e95 100644
--- a/qa/repoweb/repoweb.css
+++ b/qa/repoweb/repoweb.css
@@ -1,4 +1,13 @@
/* taken from http://code.jquery.com/mobile/1.1.0/jquery.mobile-1.1.0.css */
+
+.repoweb-build-failed {
+ color: red;
+}
+
+.repoweb-build-success {
+ color: green;
+}
+
/* jqm docs css
Beware: lots of last-minute CSS going on in here
diff --git a/qa/repoweb/repoweb.js b/qa/repoweb/repoweb.js
index ce53059..3f34bc3 100644
--- a/qa/repoweb/repoweb.js
+++ b/qa/repoweb/repoweb.js
@@ -7,6 +7,24 @@ function htmlescape(str) {
return pre.innerHTML.replace(/"/g, """).replace(/'/g, "'");;
}
+function get_page_arg(key) {
+ var url = window.location.toString();
+ var pos = url.indexOf("?");
+ if (pos == -1)
+ return null;
+
+ var search = url.substr(pos + 1);
+ var params = search.split("&");
+
+ for (var n = 0; n < params.length; n++) {
+ var val = params[n].split("=");
+ if (val[0] == key)
+ return unescape(val[1]);
+ }
+
+ return null;
+}
+
var repoDataSignal = {};
var repoData = null;
@@ -17,53 +35,82 @@ function repoweb_on_data_loaded(data) {
}
function repoweb_init() {
- $.getJSON("data.json", repoweb_on_data_loaded);
+ var id = get_page_arg("prefix");
+ var url = "work/autobuilder-" + id + ".json";
+ $.getJSON(url, repoweb_on_data_loaded);
}
-function repoweb_index_init() {
- repoweb_init();
- $(repoDataSignal).on("loaded", function () {
- $("#repoweb-summary").empty();
- var summary = $("#repoweb-summary").get(0);
+function timeago(d, now) {
+ var diffSeconds = (now.getTime() - d.getTime()) / 1000;
+ if (diffSeconds < 0)
+ return "(time format error)";
+ var units = [["seconds", 60],
+ ["minutes", 60*60],
+ ["hours", 60*60*24],
+ ["days", -1]];
+ for (var i = 0; i < units.length; i++) {
+ var unitItem = units[i];
+ var divisor = i == 0 ? 1 : units[i-1][1];
+ if (unitItem[1] == -1 || diffSeconds < unitItem[1]) {
+ return "" + (Math.floor(diffSeconds / divisor)) + " " + unitItem[0] + " ago";
+ }
+ }
+}
- var load = document.createElement('div');
- load.appendChild(document.createTextNode('System load: ' + repoData['load']));
- summary.appendChild(load);
-
- var targets = repoData['targets'];
- for (var name in targets) {
- var elt;
- var targetData = targets[name];
- var div = document.createElement("div");
- summary.appendChild(div);
+function renderBuild(container, build) {
+ var prefix = get_page_arg("prefix");
- elt = document.createElement("h3")
- elt.appendChild(document.createTextNode(name));
- div.appendChild(elt);
- elt = document.createTextNode(targetData['revision']);
- div.appendChild(elt);
- }
- });
+ var now = new Date();
+
+ var div = document.createElement('div');
+ container.appendChild(div);
+ var version = build['meta']['version'];
+ var endTimestamp = null;
+ if (build['timestamp'])
+ endTimestamp = new Date(build['timestamp'] * 1000);
+ var a = document.createElement('a');
+ div.appendChild(a);
+ a.setAttribute('href', 'work/tasks/' + prefix + '-build/' + build['v'] + '/log');
+ a.setAttribute('rel', 'external');
+ a.appendChild(document.createTextNode("Build " + version));
+ div.appendChild(document.createTextNode(": "));
+ var stateSpan = document.createElement('span');
+ div.appendChild(stateSpan);
+ var state = document.createTextNode(build['state']);
+ stateSpan.appendChild(state);
+ if (build['state'] == 'success')
+ $(stateSpan).addClass("repoweb-build-success");
+ else
+ $(stateSpan).addClass("repoweb-build-failed");
+ var status = build['build-status'];
+ if (status)
+ div.appendChild(document.createTextNode(" " + status['description']));
+ else if (endTimestamp)
+ div.appendChild(document.createTextNode(" " + timeago(endTimestamp, now)));
}
-function repoweb_files_init() {
+function repoweb_index_init() {
+ var prefix = get_page_arg("prefix");
repoweb_init();
$(repoDataSignal).on("loaded", function () {
- $("#repoweb-files").empty();
- var files = $("#repoweb-files").get(0);
- var targets = repoData['targets'];
- for (var name in targets) {
- var elt;
- var targetData = targets[name];
- var div = document.createElement("div");
- files.appendChild(div);
+ $("#resolve-summary").empty();
+ var summary = $("#resolve-summary").get(0);
+
+ var div = document.createElement('div');
+ summary.appendChild(div);
+ div.appendChild(document.createTextNode("Current version: "));
+ var a = document.createElement('a');
+ div.appendChild(a);
+ a.setAttribute('href', 'work/snapshots/' + repoData['version-path']);
+ a.setAttribute('rel', 'external');
+ a.appendChild(document.createTextNode(repoData['version']));
- elt = document.createElement("h3")
- elt.appendChild(document.createTextNode(name));
- div.appendChild(elt);
- elt = document.createElement("pre");
- elt.appendChild(document.createTextNode(targetData['files']));
- div.appendChild(elt);
- }
+ $("#build-summary").empty();
+ summary = $("#build-summary").get(0);
+ var buildData = repoData.build;
+ for (var i = buildData.length - 1; i >= 0; i--) {
+ var build = buildData[i];
+ renderBuild(summary, build);
+ }
});
}
diff --git a/src/ostbuild/pyostbuild/builtin_autobuilder.py b/src/ostbuild/pyostbuild/builtin_autobuilder.py
new file mode 100755
index 0000000..8f81f57
--- /dev/null
+++ b/src/ostbuild/pyostbuild/builtin_autobuilder.py
@@ -0,0 +1,202 @@
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import hashlib
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from .subprocess_helpers import run_sync_monitor_log_file
+from . import ostbuildrc
+from . import buildutil
+from . import mainloop
+from . import fileutil
+from . import kvfile
+from . import filemonitor
+from . import jsondb
+from . import odict
+from . import vcs
+from . import task
+
+class OstbuildAutobuilder(builtins.Builtin):
+ name = "autobuilder"
+ short_description = "Run resolve and build"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+ self.resolve_proc = None
+ self.build_proc = None
+ self.loop = mainloop.Mainloop.get(None)
+ self.prev_source_snapshot_path = None
+ self.source_snapshot_path = None
+ self.build_needed = True
+ self.last_build_succeeded = True
+
+ def _status_is_success(self, estatus):
+ return os.WIFEXITED(estatus) and os.WEXITSTATUS(estatus) == 0
+
+ def _on_resolve_exited(self, pid, status):
+ self.resolve_proc = None
+ success = self._status_is_success(status)
+ self._resolve_taskset.finish(success)
+ log("resolve exited success=%s" % (success, ))
+ self.prev_source_snapshot_path = self.source_snapshot_path
+ self.source_snapshot_path = self.get_src_snapshot_db().get_latest_path()
+ changed = self.prev_source_snapshot_path != self.source_snapshot_path
+ if changed:
+ log("New version is %s" % (self.source_snapshot_path, ))
+ log("scheduling next resolve for %d seconds " % (self.resolve_poll_secs, ))
+ self.loop.timeout_add(self.resolve_poll_secs*1000, self._fetch)
+ if not self.build_needed:
+ self.build_needed = self.prev_source_snapshot_path != self.source_snapshot_path
+ if self.build_needed and self.build_proc is None:
+ self._run_build()
+ else:
+ self._write_status()
+
+ def _fetch(self):
+ self._run_resolve(True)
+ return False
+
+ def _run_resolve(self, fetch=False):
+ assert self.resolve_proc is None
+ workdir = self._resolve_taskset.start()
+ f = open(os.path.join(workdir, 'log'), 'w')
+ args = ['ostbuild', 'resolve', '--manifest=' + self.manifest]
+ if fetch:
+ args.append('--fetch')
+ args.append('--fetch-keep-going')
+ self.resolve_proc = subprocess.Popen(args, stdin=open('/dev/null'), stdout=f, stderr=f)
+ f.close()
+ log("started resolve: pid %d workdir: %s" % (self.resolve_proc.pid, workdir))
+ self.loop.watch_pid(self.resolve_proc.pid, self._on_resolve_exited)
+ self._write_status()
+
+ def _on_build_exited(self, pid, status):
+ self.build_proc = None
+ success = self._status_is_success(status)
+ self._build_taskset.finish(success)
+ log("build exited success=%s" % (success, ))
+ filemonitor.FileMonitor.get().remove(self.build_status_mon_id)
+ self.build_status_mon_id = 0
+ if self.build_needed:
+ self._run_build()
+ else:
+ self._write_status()
+
+ def _on_build_status_changed(self):
+ self._write_status()
+
+ def _run_build(self):
+ assert self.build_proc is None
+ assert self.build_needed
+ self.build_needed = False
+ workdir = self._build_taskset.start()
+ statusjson = os.path.join(workdir, 'status.json')
+ f = open(os.path.join(workdir, 'log'), 'w')
+ args = ['ostbuild', 'build', '--skip-vcs-matches',
+ '--src-snapshot=' + self.source_snapshot_path,
+ '--status-json-path=' + statusjson]
+ src_db = self.get_src_snapshot_db()
+ version = src_db.parse_version(os.path.basename(self.source_snapshot_path))
+ meta = {'version': version,
+ 'version-path': os.path.relpath(self.source_snapshot_path, self.snapshot_dir)}
+ meta_path = os.path.join(workdir, 'meta.json')
+ fileutil.write_json_file_atomic(meta_path, meta)
+ self.build_status_json_path = statusjson
+ self.build_status_mon_id = filemonitor.FileMonitor.get().add(self.build_status_json_path,
+ self._on_build_status_changed)
+ self.build_proc = subprocess.Popen(args, stdin=open('/dev/null'), stdout=f, stderr=f)
+ log("started build: pid %d workdir: %s" % (self.build_proc.pid, workdir))
+ self.loop.watch_pid(self.build_proc.pid, self._on_build_exited)
+ self._write_status()
+
+ def _taskhistory_to_json(self, history):
+ MAXITEMS = 5
+ entries = []
+ for item in history[-MAXITEMS:]:
+ data = {'v': '%d.%d' % (item.major, item.minor),
+ 'state': item.state,
+ 'timestamp': item.timestamp}
+ entries.append(data)
+ meta_path = os.path.join(item.path, 'meta.json')
+ if os.path.isfile(meta_path):
+ f = open(meta_path)
+ data['meta'] = json.load(f)
+ f.close()
+ return entries
+
+ def _write_status(self):
+ status = {}
+ if self.source_snapshot_path is not None:
+ src_db = self.get_src_snapshot_db()
+ version = src_db.parse_version(os.path.basename(self.source_snapshot_path))
+ status['version'] = version
+ status['version-path'] = os.path.relpath(self.source_snapshot_path, self.snapshot_dir)
+ else:
+ status['version'] = ''
+
+ status['resolve'] = self._taskhistory_to_json(self._resolve_taskset.get_history())
+ build_history = self._build_taskset.get_history()
+ status['build'] = self._taskhistory_to_json(build_history)
+
+ if self.build_proc is not None:
+ active_build = build_history[-1]
+ active_build_json = status['build'][-1]
+ status_path = os.path.join(active_build.path, 'status.json')
+ if os.path.isfile(status_path):
+ f = open(status_path)
+ build_status = json.load(f)
+ f.close()
+ active_build_json['build-status'] = build_status
+
+ fileutil.write_json_file_atomic(self.status_path, status)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--resolve-poll', type=int, default=10*60)
+ parser.add_argument('--manifest', required=True)
+
+ args = parser.parse_args(argv)
+ self.manifest = args.manifest
+ self.resolve_poll_secs = args.resolve_poll
+
+ self.parse_config()
+ self.parse_prefix(args.prefix)
+ assert self.prefix is not None
+ self.init_repo()
+ self.source_snapshot_path = self.get_src_snapshot_db().get_latest_path()
+
+ taskdir = task.TaskDir(os.path.join(self.workdir, 'tasks'))
+ self._resolve_taskset = taskdir.get('%s-resolve' % (self.prefix, ))
+ self._build_taskset = taskdir.get('%s-build' % (self.prefix, ))
+
+ self.status_path = os.path.join(self.workdir, 'autobuilder-%s.json' % (self.prefix, ))
+
+ self._run_resolve()
+ self._run_build()
+
+ self.loop.run()
+
+builtins.register(OstbuildAutobuilder)
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index cd203bb..ea32006 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -178,6 +178,8 @@ class Builtin(object):
def parse_prefix(self, prefix):
if prefix is not None:
self.prefix = prefix
+ else:
+ self.prefix = self.get_prefix()
def parse_snapshot(self, prefix, path):
self.parse_prefix(prefix)
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 7676f26..1da65da 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -22,6 +22,7 @@ import sys
import argparse
from . import builtins
+from . import builtin_autobuilder
from . import builtin_build
from . import builtin_checkout
from . import builtin_deploy_root
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
