[gnome-ostree/wip/autobuilder] autobuilder: New builtin
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-ostree/wip/autobuilder] autobuilder: New builtin
- Date: Sat, 20 Oct 2012 15:55:16 +0000 (UTC)
commit 200b769bba7243d5c780b9d1a7a7c19f9212e3bf
Author: Colin Walters <walters verbum org>
Date: Wed Oct 17 20:11:07 2012 -0400
autobuilder: New builtin
Makefile-ostbuild.am | 2 +
qa/repoweb/index.html | 13 +-
qa/repoweb/repoweb.js | 116 +++++++++-----
src/ostbuild/pyostbuild/builtin_autobuilder.py | 202 ++++++++++++++++++++++++
src/ostbuild/pyostbuild/builtin_build.py | 66 ++------
src/ostbuild/pyostbuild/builtins.py | 2 +
src/ostbuild/pyostbuild/filemonitor.py | 29 +++-
src/ostbuild/pyostbuild/fileutil.py | 11 ++-
src/ostbuild/pyostbuild/jsondb.py | 8 +
src/ostbuild/pyostbuild/main.py | 1 +
src/ostbuild/pyostbuild/mainloop.py | 26 ++-
src/ostbuild/pyostbuild/task.py | 122 ++++++++++++++
12 files changed, 490 insertions(+), 108 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 3169220..2b3b6aa 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -35,6 +35,7 @@ utilsdir = $(libdir)/ostbuild
pyostbuilddir=$(libdir)/ostbuild/pyostbuild
pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/buildutil.py \
+ src/ostbuild/pyostbuild/builtin_autobuilder.py \
src/ostbuild/pyostbuild/builtin_build.py \
src/ostbuild/pyostbuild/builtin_checkout.py \
src/ostbuild/pyostbuild/builtin_deploy_qemu.py \
@@ -50,6 +51,7 @@ pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/builtin_source_diff.py \
src/ostbuild/pyostbuild/builtins.py \
src/ostbuild/pyostbuild/filemonitor.py \
+ src/ostbuild/pyostbuild/task.py \
src/ostbuild/pyostbuild/fileutil.py \
src/ostbuild/pyostbuild/__init__.py \
src/ostbuild/pyostbuild/jsondb.py \
diff --git a/qa/repoweb/index.html b/qa/repoweb/index.html
index 3111a9b..468e99f 100644
--- a/qa/repoweb/index.html
+++ b/qa/repoweb/index.html
@@ -3,10 +3,10 @@
<head>
<title>GNOME-OSTree</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
- <link rel="stylesheet" href="http://code.jquery.com/mobile/1.1.0/jquery.mobile-1.1.0.min.css"; />
+ <link rel="stylesheet" href="jquery.mobile-1.1.0.css" />
<link rel="stylesheet" href="repoweb.css"/>
- <script src="http://code.jquery.com/jquery-1.7.1.min.js";></script>
- <script src="http://code.jquery.com/mobile/1.1.0/jquery.mobile-1.1.0.min.js";></script>
+ <script src="jquery-1.7.1.js"></script>
+ <script src="jquery.mobile-1.1.0.js"></script>
<script src="repoweb.js"></script>
</head>
<body onload="$(document).ready(function(){repoweb_index_init();})">
@@ -24,8 +24,11 @@
the <a href="http://git.gnome.org/browse/gnome-ostree";>gnome-ostree</a>
git module.</p>
- <h3>Summary</h3>
- <div id="repoweb-summary">Loading...</div>
+ <h3>Resolve</h3>
+ <div id="resolve-summary">Loading...</div>
+
+ <h3>Build</h3>
+ <div id="build-summary">Loading...</div>
</div>
<div class="content-secondary">
<ul data-role="listview" data-theme="c" data-dividertheme="d">
diff --git a/qa/repoweb/repoweb.js b/qa/repoweb/repoweb.js
index ce53059..958dd7b 100644
--- a/qa/repoweb/repoweb.js
+++ b/qa/repoweb/repoweb.js
@@ -7,6 +7,24 @@ function htmlescape(str) {
return pre.innerHTML.replace(/"/g, """).replace(/'/g, "'");;
}
+function get_page_arg(key) {
+ var url = window.location.toString();
+ var pos = url.indexOf("?");
+ if (pos == -1)
+ return null;
+
+ var search = url.substr(pos + 1);
+ var params = search.split("&");
+
+ for (var n = 0; n < params.length; n++) {
+ var val = params[n].split("=");
+ if (val[0] == key)
+ return unescape(val[1]);
+ }
+
+ return null;
+}
+
var repoDataSignal = {};
var repoData = null;
@@ -17,53 +35,71 @@ function repoweb_on_data_loaded(data) {
}
function repoweb_init() {
- $.getJSON("data.json", repoweb_on_data_loaded);
+ var id = get_page_arg("prefix");
+ var url = "work/autobuilder-" + id + ".json";
+ $.getJSON(url, repoweb_on_data_loaded);
+}
+
+function timeago(d, now) {
+ var diffSeconds = (now.getTime() - d.getTime()) / 1000;
+ if (diffSeconds < 0)
+ return "(time format error)";
+ var units = [["seconds", 60],
+ ["minutes", 60*60],
+ ["hours", 60*60*24],
+ ["days", -1]];
+ for (var i = 0; i < units.length; i++) {
+ var unitItem = units[i];
+ var divisor = i == 0 ? 1 : units[i-1][1];
+ if (unitItem[1] == -1 || diffSeconds < unitItem[1]) {
+ return "" + (Math.floor(diffSeconds / divisor)) + " " + unitItem[0] + " ago";
+ }
+ }
}
function repoweb_index_init() {
+ var prefix = get_page_arg("prefix");
repoweb_init();
$(repoDataSignal).on("loaded", function () {
- $("#repoweb-summary").empty();
- var summary = $("#repoweb-summary").get(0);
-
- var load = document.createElement('div');
- load.appendChild(document.createTextNode('System load: ' + repoData['load']));
- summary.appendChild(load);
-
- var targets = repoData['targets'];
- for (var name in targets) {
- var elt;
- var targetData = targets[name];
- var div = document.createElement("div");
- summary.appendChild(div);
+ $("#resolve-summary").empty();
+ var summary = $("#resolve-summary").get(0);
- elt = document.createElement("h3")
- elt.appendChild(document.createTextNode(name));
- div.appendChild(elt);
- elt = document.createTextNode(targetData['revision']);
- div.appendChild(elt);
- }
- });
-}
+ var now = new Date();
-function repoweb_files_init() {
- repoweb_init();
- $(repoDataSignal).on("loaded", function () {
- $("#repoweb-files").empty();
- var files = $("#repoweb-files").get(0);
- var targets = repoData['targets'];
- for (var name in targets) {
- var elt;
- var targetData = targets[name];
- var div = document.createElement("div");
- files.appendChild(div);
+ var div = document.createElement('div');
+ summary.appendChild(div);
+ var a = document.createElement('a');
+ div.appendChild(a);
+ a.setAttribute('href', 'work/snapshots/' + repoData['version-path']);
+ a.setAttribute('rel', 'external');
+ a.appendChild(document.createTextNode(repoData['version']));
- elt = document.createElement("h3")
- elt.appendChild(document.createTextNode(name));
- div.appendChild(elt);
- elt = document.createElement("pre");
- elt.appendChild(document.createTextNode(targetData['files']));
- div.appendChild(elt);
- }
+ $("#build-summary").empty();
+ summary = $("#build-summary").get(0);
+ var buildData = repoData.build;
+ for (var i = 0; i < buildData.length; i++) {
+ var build = buildData[i];
+ div = document.createElement('div');
+ summary.appendChild(div);
+ var version = build['meta']['version'];
+ var endTimestamp = null;
+ if (build['timestamp'])
+ endTimestamp = new Date(build['timestamp'] * 1000);
+ var a = document.createElement('a');
+ div.appendChild(a);
+ a.setAttribute('href', 'work/tasks/' + prefix + '-build/' + build['v'] + '/log');
+ a.setAttribute('rel', 'external');
+ a.appendChild(document.createTextNode("Build " + version));
+ div.appendChild(document.createTextNode(": "));
+ var b = document.createElement("b");
+ div.appendChild(b);
+ b.appendChild(document.createTextNode(build['state']));
+ var status = build['build-status'];
+ console.log("status=" + status);
+ if (status)
+ div.appendChild(document.createTextNode(" " + status['description']));
+ else if (endTimestamp)
+ div.appendChild(document.createTextNode(" " + timeago(endTimestamp, now)));
+ }
});
}
diff --git a/src/ostbuild/pyostbuild/builtin_autobuilder.py b/src/ostbuild/pyostbuild/builtin_autobuilder.py
new file mode 100755
index 0000000..8f81f57
--- /dev/null
+++ b/src/ostbuild/pyostbuild/builtin_autobuilder.py
@@ -0,0 +1,202 @@
+# Copyright (C) 2012 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil
+import argparse
+import time
+import urlparse
+import hashlib
+import json
+from StringIO import StringIO
+
+from . import builtins
+from .ostbuildlog import log, fatal
+from .subprocess_helpers import run_sync, run_sync_get_output
+from .subprocess_helpers import run_sync_monitor_log_file
+from . import ostbuildrc
+from . import buildutil
+from . import mainloop
+from . import fileutil
+from . import kvfile
+from . import filemonitor
+from . import jsondb
+from . import odict
+from . import vcs
+from . import task
+
+class OstbuildAutobuilder(builtins.Builtin):
+ name = "autobuilder"
+ short_description = "Run resolve and build"
+
+ def __init__(self):
+ builtins.Builtin.__init__(self)
+ self.resolve_proc = None
+ self.build_proc = None
+ self.loop = mainloop.Mainloop.get(None)
+ self.prev_source_snapshot_path = None
+ self.source_snapshot_path = None
+ self.build_needed = True
+ self.last_build_succeeded = True
+
+ def _status_is_success(self, estatus):
+ return os.WIFEXITED(estatus) and os.WEXITSTATUS(estatus) == 0
+
+ def _on_resolve_exited(self, pid, status):
+ self.resolve_proc = None
+ success = self._status_is_success(status)
+ self._resolve_taskset.finish(success)
+ log("resolve exited success=%s" % (success, ))
+ self.prev_source_snapshot_path = self.source_snapshot_path
+ self.source_snapshot_path = self.get_src_snapshot_db().get_latest_path()
+ changed = self.prev_source_snapshot_path != self.source_snapshot_path
+ if changed:
+ log("New version is %s" % (self.source_snapshot_path, ))
+ log("scheduling next resolve for %d seconds " % (self.resolve_poll_secs, ))
+ self.loop.timeout_add(self.resolve_poll_secs*1000, self._fetch)
+ if not self.build_needed:
+ self.build_needed = self.prev_source_snapshot_path != self.source_snapshot_path
+ if self.build_needed and self.build_proc is None:
+ self._run_build()
+ else:
+ self._write_status()
+
+ def _fetch(self):
+ self._run_resolve(True)
+ return False
+
+ def _run_resolve(self, fetch=False):
+ assert self.resolve_proc is None
+ workdir = self._resolve_taskset.start()
+ f = open(os.path.join(workdir, 'log'), 'w')
+ args = ['ostbuild', 'resolve', '--manifest=' + self.manifest]
+ if fetch:
+ args.append('--fetch')
+ args.append('--fetch-keep-going')
+ self.resolve_proc = subprocess.Popen(args, stdin=open('/dev/null'), stdout=f, stderr=f)
+ f.close()
+ log("started resolve: pid %d workdir: %s" % (self.resolve_proc.pid, workdir))
+ self.loop.watch_pid(self.resolve_proc.pid, self._on_resolve_exited)
+ self._write_status()
+
+ def _on_build_exited(self, pid, status):
+ self.build_proc = None
+ success = self._status_is_success(status)
+ self._build_taskset.finish(success)
+ log("build exited success=%s" % (success, ))
+ filemonitor.FileMonitor.get().remove(self.build_status_mon_id)
+ self.build_status_mon_id = 0
+ if self.build_needed:
+ self._run_build()
+ else:
+ self._write_status()
+
+ def _on_build_status_changed(self):
+ self._write_status()
+
+ def _run_build(self):
+ assert self.build_proc is None
+ assert self.build_needed
+ self.build_needed = False
+ workdir = self._build_taskset.start()
+ statusjson = os.path.join(workdir, 'status.json')
+ f = open(os.path.join(workdir, 'log'), 'w')
+ args = ['ostbuild', 'build', '--skip-vcs-matches',
+ '--src-snapshot=' + self.source_snapshot_path,
+ '--status-json-path=' + statusjson]
+ src_db = self.get_src_snapshot_db()
+ version = src_db.parse_version(os.path.basename(self.source_snapshot_path))
+ meta = {'version': version,
+ 'version-path': os.path.relpath(self.source_snapshot_path, self.snapshot_dir)}
+ meta_path = os.path.join(workdir, 'meta.json')
+ fileutil.write_json_file_atomic(meta_path, meta)
+ self.build_status_json_path = statusjson
+ self.build_status_mon_id = filemonitor.FileMonitor.get().add(self.build_status_json_path,
+ self._on_build_status_changed)
+ self.build_proc = subprocess.Popen(args, stdin=open('/dev/null'), stdout=f, stderr=f)
+ log("started build: pid %d workdir: %s" % (self.build_proc.pid, workdir))
+ self.loop.watch_pid(self.build_proc.pid, self._on_build_exited)
+ self._write_status()
+
+ def _taskhistory_to_json(self, history):
+ MAXITEMS = 5
+ entries = []
+ for item in history[-MAXITEMS:]:
+ data = {'v': '%d.%d' % (item.major, item.minor),
+ 'state': item.state,
+ 'timestamp': item.timestamp}
+ entries.append(data)
+ meta_path = os.path.join(item.path, 'meta.json')
+ if os.path.isfile(meta_path):
+ f = open(meta_path)
+ data['meta'] = json.load(f)
+ f.close()
+ return entries
+
+ def _write_status(self):
+ status = {}
+ if self.source_snapshot_path is not None:
+ src_db = self.get_src_snapshot_db()
+ version = src_db.parse_version(os.path.basename(self.source_snapshot_path))
+ status['version'] = version
+ status['version-path'] = os.path.relpath(self.source_snapshot_path, self.snapshot_dir)
+ else:
+ status['version'] = ''
+
+ status['resolve'] = self._taskhistory_to_json(self._resolve_taskset.get_history())
+ build_history = self._build_taskset.get_history()
+ status['build'] = self._taskhistory_to_json(build_history)
+
+ if self.build_proc is not None:
+ active_build = build_history[-1]
+ active_build_json = status['build'][-1]
+ status_path = os.path.join(active_build.path, 'status.json')
+ if os.path.isfile(status_path):
+ f = open(status_path)
+ build_status = json.load(f)
+ f.close()
+ active_build_json['build-status'] = build_status
+
+ fileutil.write_json_file_atomic(self.status_path, status)
+
+ def execute(self, argv):
+ parser = argparse.ArgumentParser(description=self.short_description)
+ parser.add_argument('--prefix')
+ parser.add_argument('--resolve-poll', type=int, default=10*60)
+ parser.add_argument('--manifest', required=True)
+
+ args = parser.parse_args(argv)
+ self.manifest = args.manifest
+ self.resolve_poll_secs = args.resolve_poll
+
+ self.parse_config()
+ self.parse_prefix(args.prefix)
+ assert self.prefix is not None
+ self.init_repo()
+ self.source_snapshot_path = self.get_src_snapshot_db().get_latest_path()
+
+ taskdir = task.TaskDir(os.path.join(self.workdir, 'tasks'))
+ self._resolve_taskset = taskdir.get('%s-resolve' % (self.prefix, ))
+ self._build_taskset = taskdir.get('%s-build' % (self.prefix, ))
+
+ self.status_path = os.path.join(self.workdir, 'autobuilder-%s.json' % (self.prefix, ))
+
+ self._run_resolve()
+ self._run_build()
+
+ self.loop.run()
+
+builtins.register(OstbuildAutobuilder)
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index 621ce1d..b2c8077 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -29,6 +29,7 @@ from .subprocess_helpers import run_sync, run_sync_get_output
from .subprocess_helpers import run_sync_monitor_log_file
from . import ostbuildrc
from . import buildutil
+from . import task
from . import fileutil
from . import kvfile
from . import odict
@@ -198,29 +199,6 @@ class OstbuildBuild(builtins.Builtin):
result.append(csum.hexdigest())
return result
- def _create_task_workdir(self, taskname):
- workdir = os.path.join(self.workdir, 'tasks')
- fileutil.ensure_dir(workdir)
- serialfile = os.path.join(workdir, 'serial')
- if not os.path.isfile(serialfile):
- serial = 0
- else:
- f = open(serialfile)
- serial = int(f.read().strip())
- f.close()
-
- serial += 1
-
- f = open(serialfile, 'w')
- f.write('%d\n' % (serial, ))
- f.close()
-
- taskdir = os.path.join(workdir, '%s/%d' % (taskname, serial))
- if os.path.isdir(taskdir):
- shutil.rmtree(taskdir)
- fileutil.ensure_dir(taskdir)
- return taskdir
-
def _build_one_component(self, component, architecture):
basename = component['name']
@@ -291,12 +269,12 @@ class OstbuildBuild(builtins.Builtin):
else:
log("Need rebuild of %s: %s" % (buildname, rebuild_reason, ) )
- workdir = self._create_task_workdir(buildname)
+ taskdir = task.TaskDir(os.path.join(self.workdir, 'tasks'))
+ build_taskset = taskdir.get(buildname)
+ workdir = build_taskset.start()
temp_metadata_path = os.path.join(workdir, '_ostbuild-meta.json')
- f = open(temp_metadata_path, 'w')
- json.dump(expanded_component, f, indent=4, sort_keys=True)
- f.close()
+ fileutil.write_json_file_atomic(temp_metadata_path, expanded_component)
checkoutdir = os.path.join(self.workdir, 'checkouts')
component_src = os.path.join(checkoutdir, buildname)
@@ -320,8 +298,7 @@ class OstbuildBuild(builtins.Builtin):
component_resultdir = os.path.join(workdir, 'results')
fileutil.ensure_dir(component_resultdir)
- self._write_status({'status': 'building',
- 'target': build_ref})
+ self._write_status('Building ' + build_ref)
rootdir = self._compose_buildroot(workdir, basename, architecture)
@@ -363,14 +340,11 @@ class OstbuildBuild(builtins.Builtin):
if not success:
self._analyze_build_failure(architecture, component, component_src,
current_vcs_version, previous_vcs_version)
- self._write_status({'status': 'failed',
- 'target': build_ref})
+ self._write_status('Failed building ' + build_ref)
fatal("Exiting due to build failure in component:%s arch:%s" % (component, architecture))
recorded_meta_path = os.path.join(component_resultdir, '_ostbuild-meta.json')
- recorded_meta_f = open(recorded_meta_path, 'w')
- json.dump(expanded_component, recorded_meta_f, indent=4, sort_keys=True)
- recorded_meta_f.close()
+ fileutil.write_json_file_atomic(recorded_meta_path, expanded_component)
args = ['ostree', '--repo=' + self.repo,
'commit', '-b', build_ref, '-s', 'Build',
@@ -462,9 +436,7 @@ class OstbuildBuild(builtins.Builtin):
os.unlink(contents_tmppath)
contents_path = os.path.join(compose_rootdir, 'contents.json')
- f = open(contents_path, 'w')
- json.dump(self.snapshot, f, indent=4, sort_keys=True)
- f.close()
+ fileutil.write_json_file_atomic(contents_path, self.snapshot)
treename = 'trees/%s' % (target['name'], )
@@ -479,16 +451,11 @@ class OstbuildBuild(builtins.Builtin):
os.unlink(related_tmppath)
shutil.rmtree(compose_rootdir)
- def _write_status(self, data):
+ def _write_status(self, description):
if not self.args.status_json_path:
return
- (fd, temppath) = tempfile.mkstemp(suffix='.tmp', prefix='status-json-',
- dir=os.path.dirname(self.args.status_json_path))
- os.close(fd)
- f = open(temppath, 'w')
- json.dump(data, f, indent=4, sort_keys=True)
- f.close()
- os.rename(temppath, self.args.status_json_path)
+ fileutil.write_json_file_atomic(self.args.status_json_path,
+ {'description': description})
def _initialize_repo(self):
"""Set up an OSTree repository in $workdir/repo.
@@ -542,7 +509,7 @@ and the manifest input."""
log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
- self._write_status({'state': 'build-starting'})
+ self._write_status('Starting')
self.buildopts = BuildOptions()
self.buildopts.force_rebuild = args.force_rebuild
@@ -603,7 +570,7 @@ and the manifest input."""
for (component, architecture) in components_to_build:
archname = '%s/%s' % (component['name'], architecture)
build_rev = self._build_one_component(component, architecture)
- self._write_status({'status': 'scanning'})
+ self._write_status('Scanning')
component_build_revs[archname] = build_rev
targets_list = []
@@ -623,8 +590,7 @@ and the manifest input."""
'runtime': runtime_ref,
'devel': buildroot_ref}
- self._write_status({'status': 'composing',
- 'target': target['name']})
+ self._write_status('Composing ' + target['name'])
if target_component_type == 'runtime':
target_components = runtime_components
@@ -651,6 +617,6 @@ and the manifest input."""
log("Composing %r from %d components" % (target['name'], len(target['contents'])))
self._compose_one_target(target, component_build_revs)
- self._write_status({'status': 'complete'})
+ self._write_status('Complete')
builtins.register(OstbuildBuild)
diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py
index cd203bb..ea32006 100755
--- a/src/ostbuild/pyostbuild/builtins.py
+++ b/src/ostbuild/pyostbuild/builtins.py
@@ -178,6 +178,8 @@ class Builtin(object):
def parse_prefix(self, prefix):
if prefix is not None:
self.prefix = prefix
+ else:
+ self.prefix = self.get_prefix()
def parse_snapshot(self, prefix, path):
self.parse_prefix(prefix)
diff --git a/src/ostbuild/pyostbuild/filemonitor.py b/src/ostbuild/pyostbuild/filemonitor.py
index 0eec07b..c0ce404 100644
--- a/src/ostbuild/pyostbuild/filemonitor.py
+++ b/src/ostbuild/pyostbuild/filemonitor.py
@@ -30,6 +30,7 @@ class FileMonitor(object):
self._timeout = 1000
self._timeout_installed = False
self._loop = mainloop.Mainloop.get(None)
+ self._counter = 0
@classmethod
def get(cls):
@@ -49,16 +50,38 @@ class FileMonitor(object):
if path not in self._paths:
self._paths[path] = []
self._path_modtimes[path] = self._stat(path)
- self._paths[path].append(callback)
+ self._counter += 1
+ self._paths[path].append((self._counter, callback))
if not self._timeout_installed:
self._timeout_installed = True
self._loop.timeout_add(self._timeout, self._check_files)
+ return self._counter
+
+ def remove(self, cb_id):
+ found = False
+ for path in self._paths:
+ cbs = self._paths[path]
+ idx = -1
+ for i,(iter_cb_id, callback) in enumerate(cbs):
+ if iter_cb_id == cb_id:
+ idx = i
+ break
+ if idx != -1:
+ cbs.pop(idx)
+ found = True
+ break
+ assert found
def _check_files(self):
+ cbs = []
for (path,callbacks) in self._paths.iteritems():
mtime = self._stat(path)
orig_mtime = self._path_modtimes[path]
if (mtime is not None) and (orig_mtime is None or (mtime > orig_mtime)):
self._path_modtimes[path] = mtime
- for cb in callbacks:
- cb()
+ for (counter, cb) in callbacks:
+ cbs.append(cb)
+ for cb in cbs:
+ cb()
+ self._timeout_installed = len(self._paths) > 0
+ return self._timeout_installed
diff --git a/src/ostbuild/pyostbuild/fileutil.py b/src/ostbuild/pyostbuild/fileutil.py
index d9ae91f..30bf192 100644
--- a/src/ostbuild/pyostbuild/fileutil.py
+++ b/src/ostbuild/pyostbuild/fileutil.py
@@ -16,7 +16,7 @@
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
-import os
+import os,json
def ensure_dir(path):
if not os.path.isdir(path):
@@ -24,3 +24,12 @@ def ensure_dir(path):
def ensure_parent_dir(path):
ensure_dir(os.path.dirname(path))
+
+def write_json_file_atomic(path, data):
+ path_tmp = path + '.tmp'
+ f = open(path_tmp, 'w')
+ json.dump(data, f, indent=4, sort_keys=True)
+ f.close()
+ os.chmod(path_tmp, 0644)
+ os.rename(path_tmp, path)
+
diff --git a/src/ostbuild/pyostbuild/jsondb.py b/src/ostbuild/pyostbuild/jsondb.py
index 35103b4..897aaf4 100644
--- a/src/ostbuild/pyostbuild/jsondb.py
+++ b/src/ostbuild/pyostbuild/jsondb.py
@@ -25,6 +25,8 @@ import shutil
import hashlib
import json
+from . import buildutil
+
class JsonDB(object):
def __init__(self, dirpath, prefix):
self._dirpath = dirpath
@@ -70,6 +72,11 @@ class JsonDB(object):
return None
return os.path.join(self._dirpath, files[0][3])
+ def parse_version(self, name):
+ match = self._version_csum_re.search(name)
+ assert match is not None
+ return '%s.%s' % (match.group(1), match.group(2))
+
def store(self, obj):
files = self._get_all()
if len(files) == 0:
@@ -85,6 +92,7 @@ class JsonDB(object):
f = open(tmppath, 'w')
json.dump(obj, f, indent=4, sort_keys=True)
f.close()
+ os.chmod(tmppath, 0644)
csum = hashlib.sha256()
f = open(tmppath)
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 7676f26..1da65da 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -22,6 +22,7 @@ import sys
import argparse
from . import builtins
+from . import builtin_autobuilder
from . import builtin_build
from . import builtin_checkout
from . import builtin_deploy_root
diff --git a/src/ostbuild/pyostbuild/mainloop.py b/src/ostbuild/pyostbuild/mainloop.py
index 40a67be..ec34799 100644
--- a/src/ostbuild/pyostbuild/mainloop.py
+++ b/src/ostbuild/pyostbuild/mainloop.py
@@ -29,6 +29,7 @@ class Mainloop(object):
self._timeouts = []
self._pid_watches = {}
self._fd_callbacks = {}
+ self._source_counter = 0
@classmethod
def get(cls, context):
@@ -55,7 +56,9 @@ class Mainloop(object):
self._pid_watches[pid] = callback
def timeout_add(self, ms, callback):
- self._timeouts.append((ms, callback))
+ self._source_counter += 1
+ self._timeouts.append((self._source_counter, ms, callback))
+ return self._source_counter
def quit(self):
self._running = False
@@ -64,7 +67,7 @@ class Mainloop(object):
min_timeout = None
if len(self._pid_watches) > 0:
min_timeout = 500
- for (ms, callback) in self._timeouts:
+ for (source_id, ms, callback) in self._timeouts:
if (min_timeout is None) or (ms < min_timeout):
min_timeout = ms
origtime = time.time() * 1000
@@ -72,23 +75,28 @@ class Mainloop(object):
fds = self.poll.poll(min_timeout)
for fd in fds:
self._fd_callbacks[fd]()
- to_delete_pids = []
+ hit_pids = []
for pid in self._pid_watches:
(opid, status) = os.waitpid(pid, os.WNOHANG)
if opid == pid:
- to_delete_pids.append(pid)
- self._pid_watches[pid](pid, status)
- for pid in to_delete_pids:
+ hit_pids.append(pid)
+ for pid in hit_pids:
+ watch = self._pid_watches[pid]
del self._pid_watches[pid]
+ watch(pid, status)
newtime = time.time() * 1000
diff = int(newtime - origtime)
if diff < 0: diff = 0
- for i,(ms, callback) in enumerate(self._timeouts):
+ new_timeouts = []
+ for i,(source_id, ms, callback) in enumerate(self._timeouts):
remaining_ms = ms - diff
if remaining_ms <= 0:
- callback()
+ result = callback()
+ if result:
+ new_timeouts.append((source_id, ms, callback))
else:
- self._timeouts[i] = (remaining_ms, callback)
+ new_timeouts.append((source_id, remaining_ms, callback))
+ self._timeouts = new_timeouts
def run(self):
self._running = True
diff --git a/src/ostbuild/pyostbuild/task.py b/src/ostbuild/pyostbuild/task.py
new file mode 100644
index 0000000..8f6dfae
--- /dev/null
+++ b/src/ostbuild/pyostbuild/task.py
@@ -0,0 +1,122 @@
+# Copyright (C) 2011 Colin Walters <walters verbum org>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+import os,sys,subprocess,tempfile,re,shutil,stat
+import argparse
+import time
+import urlparse
+import hashlib
+import json
+
+from . import fileutil
+
+VERSION_RE = re.compile(r'(\d+)\.(\d+)')
+
+class TaskDir(object):
+ def __init__(self, path):
+ self.path = path
+
+ def get(self, name):
+ task_path = os.path.join(self.path, name)
+ fileutil.ensure_dir(task_path)
+
+ return TaskSet(task_path)
+
+class TaskHistoryEntry(object):
+ def __init__(self, path, state=None):
+ self.path = path
+ match = VERSION_RE.match(os.path.basename(path))
+ assert match is not None
+ self.major = int(match.group(1))
+ self.minor = int(match.group(2))
+ self.timestamp = None
+ if state is None:
+ statuspath = os.path.join(self.path, 'status')
+ if os.path.isfile(statuspath):
+ f = open(statuspath)
+ self.state = f.read()
+ f.close()
+ self.timestamp = int(os.stat(statuspath)[stat.ST_MTIME])
+ else:
+ self.state = 'interrupted'
+ else:
+ self.state = state
+
+ def finish(self, success):
+ statuspath = os.path.join(self.path, 'status')
+ f = open(statuspath, 'w')
+ if success:
+ success_str = 'success'
+ else:
+ success_str = 'failed'
+ self.state = success_str
+ self.timestamp = int(time.time())
+ f.write(success_str)
+ f.close()
+
+ def __cmp__(self, other):
+ if not isinstance(other, TaskHistoryEntry):
+ return -1
+ elif (self.major != other.major):
+ return cmp(self.major, other.major)
+ else:
+ return cmp(self.minor, other.minor)
+
+class TaskSet(object):
+ def __init__(self, path):
+ self.path = path
+
+ self._history = []
+ self._running = False
+ self._running_version = None
+
+ self._load()
+
+ def _load(self):
+ for item in os.listdir(self.path):
+ match = VERSION_RE.match(item)
+ if match is None:
+ continue
+ history_path = os.path.join(self.path, item)
+ self._history.append(TaskHistoryEntry(history_path))
+ self._history.sort()
+
+ def start(self):
+ assert not self._running
+ self._running = True
+ yearver = time.gmtime().tm_year
+ if len(self._history) == 0:
+ lastversion = -1
+ else:
+ last = self._history[-1]
+ if last.major == yearver:
+ lastversion = last.minor
+ else:
+ lastversion = -1
+ history_path = os.path.join(self.path, '%d.%d' % (yearver, lastversion + 1))
+ fileutil.ensure_dir(history_path)
+ self._history.append(TaskHistoryEntry(history_path, state='running'))
+ return history_path
+
+ def finish(self, success):
+ assert self._running
+ last = self._history[-1]
+ last.finish(success)
+ self._running = False
+
+ def get_history(self):
+ return self._history
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
