[gnome-ostree/wip/gjs-round2] Work on porting build to gjs
- From: Colin Walters <walters src gnome org>
- To: commits-list gnome org
- Cc:
- Subject: [gnome-ostree/wip/gjs-round2] Work on porting build to gjs
- Date: Mon, 10 Dec 2012 00:01:01 +0000 (UTC)
commit 42d33012e0aefaee1b78b733b439681a11ed4e0b
Author: Colin Walters <walters verbum org>
Date: Sun Dec 9 18:59:58 2012 -0500
Work on porting build to gjs
src/ostbuild/js/build.js | 721 ++++++++++++++++++++++++++++++
src/ostbuild/js/procutil.js | 25 +-
src/ostbuild/js/resolve.js | 2 -
src/ostbuild/pyostbuild/builtin_build.py | 60 +--
src/ostbuild/pyostbuild/main.py | 2 +-
5 files changed, 766 insertions(+), 44 deletions(-)
---
diff --git a/src/ostbuild/js/build.js b/src/ostbuild/js/build.js
new file mode 100644
index 0000000..d6eb33b
--- /dev/null
+++ b/src/ostbuild/js/build.js
@@ -0,0 +1,721 @@
+// Copyright (C) 2011 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+const Lang = imports.lang;
+const Format = imports.format;
+
+const GSystem = imports.gi.GSystem;
+
+const Task = imports.task;
+const JsonDB = imports.jsondb;
+const ProcUtil = imports.procutil;
+const JsonUtil = imports.jsonutil;
+const Snapshot = imports.snapshot;
+const Config = imports.config;
+const BuildUtil = imports.buildutil;
+const Vcs = imports.vcs;
+const ArgParse = imports.argparse;
+
+const OPT_COMMON_CFLAGS = {'i686': '-O2 -g -m32 -march=i686 -mtune=atom -fasynchronous-unwind-tables',
+ 'x86_64': '-O2 -g -m64 -mtune=generic'}
+
+var loop = GLib.MainLoop.new(null, true);
+
+const Build = new Lang.Class({
+ Name: "Build",
+
+ def _resolve_refs(self, refs):
+ if len(refs) == 0:
+ return []
+ args = ['ostree', '--repo=' + self.repo, 'rev-parse']
+ args.extend(refs)
+ output = run_sync_get_output(args)
+ return output.split('\n')
+
+ def _clean_stale_buildroots(self, buildroot_cachedir, keep_root):
+ roots = os.listdir(buildroot_cachedir)
+ root_mtimes = {}
+ for root in roots:
+ path = os.path.join(buildroot_cachedir, root)
+ root_mtimes[root] = os.lstat(path).st_mtime
+ roots.sort(lambda a,b: cmp(root_mtimes[a], root_mtimes[b]))
+ remaining = roots[:-2]
+ for root in remaining:
+ path = os.path.join(buildroot_cachedir, root)
+ if path == keep_root:
+ continue
+ log("Removing old cached buildroot %s" % (root, ))
+ shutil.rmtree(path)
+
+ def _compose_buildroot(self, workdir, component_name, architecture):
+ starttime = time.time()
+
+ buildname = '%s/%s/%s' % (this._snapshot['prefix'], component_name, architecture)
+ buildroot_cachedir = os.path.join(self.workdir, 'roots', buildname)
+ fileutil.ensure_dir(buildroot_cachedir)
+
+ components = this._snapshot['components']
+ component = None
+ build_dependencies = []
+ for component in components:
+ if component['name'] == component_name:
+ break
+ build_dependencies.append(component)
+
+ ref_to_rev = {}
+
+ prefix = this._snapshot['prefix']
+
+ arch_buildroot_name = 'bases/%s/%s-%s-devel' % (this._snapshot['base']['name'],
+ prefix,
+ architecture)
+
+ log("Computing buildroot contents")
+
+ arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse',
+ arch_buildroot_name]).strip()
+
+ ref_to_rev[arch_buildroot_name] = arch_buildroot_rev
+ checkout_trees = [(arch_buildroot_name, '/')]
+ refs_to_resolve = []
+ for dependency in build_dependencies:
+ buildname = 'components/%s/%s/%s' % (prefix, dependency['name'], architecture)
+ refs_to_resolve.append(buildname)
+ checkout_trees.append((buildname, '/runtime'))
+ checkout_trees.append((buildname, '/devel'))
+
+ resolved_refs = self._resolve_refs(refs_to_resolve)
+ for ref,rev in zip(refs_to_resolve, resolved_refs):
+ ref_to_rev[ref] = rev
+
+ sha = hashlib.sha256()
+
+ uid = os.getuid()
+ gid = os.getgid()
+ etc_passwd = 'root:x:0:0:root:/root:/bin/bash\nbuilduser:x:%u:%u:builduser:/:/bin/bash\n' % (uid, gid)
+ etc_group = 'root:x:0:root\nbuilduser:x:%u:builduser\n' % (gid, )
+
+ (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-buildroot-')
+ f = os.fdopen(fd, 'w')
+ for (branch, subpath) in checkout_trees:
+ f.write(ref_to_rev[branch])
+ f.write('\0')
+ f.write(subpath)
+ f.write('\0')
+ f.close()
+
+ f = open(tmppath)
+ buf = f.read(8192)
+ while buf != '':
+ sha.update(buf)
+ buf = f.read(8192)
+ f.close()
+
+ sha.update(etc_passwd)
+ sha.update(etc_group)
+
+ new_root_cacheid = sha.hexdigest()
+
+ cached_root = os.path.join(buildroot_cachedir, new_root_cacheid)
+ if os.path.isdir(cached_root):
+ log("Reusing cached buildroot: %s" % (cached_root, ))
+ self._clean_stale_buildroots(buildroot_cachedir, os.path.basename(cached_root))
+ os.unlink(tmppath)
+ return cached_root
+
+ if len(checkout_trees) > 0:
+ log("composing buildroot from %d parents (last: %r)" % (len(checkout_trees),
+ checkout_trees[-1][0]))
+
+ cached_root_tmp = cached_root + '.tmp'
+ if os.path.isdir(cached_root_tmp):
+ shutil.rmtree(cached_root_tmp)
+ run_sync(['ostree', '--repo=' + self.repo,
+ 'checkout', '--user-mode', '--union',
+ '--from-file=' + tmppath, cached_root_tmp])
+
+ os.unlink(tmppath)
+
+ builddir_tmp = os.path.join(cached_root_tmp, 'ostbuild')
+ fileutil.ensure_dir(os.path.join(builddir_tmp, 'source', component_name))
+ fileutil.ensure_dir(os.path.join(builddir_tmp, 'results'))
+ f = open(os.path.join(cached_root_tmp, 'etc', 'passwd'), 'w')
+ f.write(etc_passwd)
+ f.close()
+ f = open(os.path.join(cached_root_tmp, 'etc', 'group'), 'w')
+ f.write(etc_group)
+ f.close()
+ os.rename(cached_root_tmp, cached_root)
+
+ self._clean_stale_buildroots(buildroot_cachedir, os.path.basename(cached_root))
+
+ endtime = time.time()
+ log("Composed buildroot; %d seconds elapsed" % (int(endtime - starttime),))
+
+ return cached_root
+
+ def _analyze_build_failure(self, t, architecture, component, component_srcdir,
+ current_vcs_version, previous_vcs_version):
+ # Dump last bit of log
+ print "LOGFILE: " + t.logfile_path
+ f = open(t.logfile_path)
+ lines = f.readlines()
+ lines = lines[-250:]
+ for line in lines:
+ print "| " + line.strip()
+ f.close()
+ if (current_vcs_version is not None and previous_vcs_version is not None):
+ git_args = ['git', 'log', '--format=short']
+ git_args.append(previous_vcs_version + '...' + current_vcs_version)
+ subproc_env = dict(os.environ)
+ subproc_env['GIT_PAGER'] = 'cat'
+ run_sync(git_args, cwd=component_srcdir, stdin=open('/dev/null'),
+ stdout=sys.stdout, env=subproc_env, log_success=False)
+ else:
+ log("No previous build; skipping source diff")
+
+ def _needs_rebuild(self, previous_metadata, new_metadata):
+ build_keys = ['config-opts', 'src', 'revision']
+ for k in build_keys:
+ if (k not in new_metadata):
+ return 'key %r removed from new_metadata' % (k, )
+ if k in previous_metadata:
+ oldval = previous_metadata[k]
+ newval = new_metadata[k]
+ if oldval != newval:
+ return 'key %r differs (%r -> %r)' % (k, oldval, newval)
+
+ if (k not in new_metadata) or (previous_metadata[k] != new_metadata[k]):
+ return 'key %r differs' % (k, )
+
+ if 'patches' in previous_metadata:
+ if 'patches' not in new_metadata:
+ return 'patches differ'
+ old_patches = previous_metadata['patches']
+ new_patches = new_metadata['patches']
+ old_files = old_patches['files']
+ new_files = new_patches['files']
+ if len(old_files) != len(new_files):
+ return 'patches differ'
+ old_sha256sums = old_patches.get('files_sha256sums')
+ new_sha256sums = new_patches.get('files_sha256sums')
+ if ((old_sha256sums is None or new_sha256sums is None) or
+ len(old_sha256sums) != len(new_sha256sums) or
+ old_sha256sums != new_sha256sums):
+ return 'patch sha256sums differ'
+ return None
+
+ def _compute_sha256sums_for_patches(self, patchdir, component):
+ patches = buildutil.get_patch_paths_for_component(patchdir, component)
+ result = []
+
+ for patch in patches:
+ csum = hashlib.sha256()
+ f = open(patch)
+ patchdata = f.read()
+ csum.update(patchdata)
+ f.close()
+ result.append(csum.hexdigest())
+ return result
+
+ def _save_component_build(self, buildname, expanded_component):
+ build_ref = 'components/%s' % (buildname, )
+ cachedata = dict(expanded_component)
+ cachedata['ostree'] = run_sync_get_output(['ostree', '--repo=' + self.repo,
+ 'rev-parse', build_ref])
+ self._component_build_cache[buildname] = cachedata
+ fileutil.write_json_file_atomic(self._component_build_cache_path, self._component_build_cache)
+ return cachedata['ostree']
+
+ function _buildOneComponent(component, architecture, cancellable) {
+ let basename = component['name'];
+
+ let buildname = Format.vprintf('%s/%s/%s', [this._snapshot['prefix'], basename, architecture]);
+ let buildRef = 'components/' + buildname;
+
+ let currentVcsVersion = component['revision'];
+ let expandedComponent = Snapshot.expandComponent(this._snapshot, component);
+ let previousMetadata = this._componentBuildCache[buildname];
+ let wasInBuildCache = (previousMetadata != null);
+ let previousBuildVersion;
+ if wasInBuildCache:
+ previousBuildVersion = previousMetadata['ostree'];
+ else:
+ previousBuildVersion = ProcUtil.runSyncGetOutputUTF8StrippedOrNull(['ostree', '--repo=' + self.repo.get_path(),
+ 'rev-parse', buildRef]);
+ let previousVcsVersion;
+ if (previousMetadata != null) {
+ previousVcsVersion = previousMetadata['revision'];
+ } else if (previousBuildVersion != null) {
+ let jsonstr = ProcUtil.runSyncGetOutputUTF8(['ostree', '--repo=' + self.repo.get_path(),
+ 'cat', previousBuildVersion,
+ '/_ostbuild-meta.json']);
+ previousMetadata = JSON.parse(jsonstr);
+ previousVcsVersion = previousMetadata['revision'];
+ } else {
+ log("No previous build for " + buildname);
+ previousVcsVersion = null;
+ }
+
+ if expandedComponent['patches'] {
+ patchesRevision = expandedComponent['patches']['revision'];
+ let patchdir;
+ if (this.args.patches_path) {
+ patchdir = Gio.File.new_for_path(this.args.patches_path);
+ else if (this._cachedPatchdirRevision == patchesRevision)
+ patchdir = this._patchdir;
+ else {
+ patchdir = Vcs.checkoutPatches(this._mirrordir,
+ this._patchdir,
+ expandedComponent,
+ this.args.patches_path);
+ self.cached_patchdir_revision = patches_revision
+ }
+ if ((previous_metadata is not None) and
+ 'patches' in previous_metadata and
+ 'revision' in previous_metadata['patches'] and
+ previous_metadata['patches']['revision'] == patches_revision):
+ # Copy over the sha256sums
+ expanded_component['patches'] = previous_metadata['patches']
+ else:
+ patches_sha256sums = self._compute_sha256sums_for_patches(patchdir, expanded_component)
+ expanded_component['patches']['files_sha256sums'] = patches_sha256sums
+ else:
+ patchdir = None
+
+ force_rebuild = (self.buildopts.force_rebuild or
+ basename in self.force_build_components or
+ expanded_component['src'].startswith('local:'))
+
+ if previous_metadata is not None:
+ rebuild_reason = self._needs_rebuild(previous_metadata, expanded_component)
+ if rebuild_reason is None:
+ if not force_rebuild:
+ log("Reusing cached build of %s at %s" % (buildname, previous_vcs_version))
+ if not was_in_build_cache:
+ return self._save_component_build(buildname, expanded_component)
+ return previous_build_version
+ else:
+ log("Build forced regardless")
+ else:
+ log("Need rebuild of %s: %s" % (buildname, rebuild_reason, ) )
+
+ taskdir = task.TaskDir(os.path.join(self.workdir, 'tasks'))
+ build_taskset = taskdir.get(buildname)
+ t = build_taskset.start()
+ workdir = t.path
+
+ temp_metadata_path = os.path.join(workdir, '_ostbuild-meta.json')
+ fileutil.write_json_file_atomic(temp_metadata_path, expanded_component)
+
+ checkoutdir = os.path.join(self.workdir, 'checkouts')
+ component_src = os.path.join(checkoutdir, buildname)
+ fileutil.ensure_parent_dir(component_src)
+ child_args = ['ostbuild', 'checkout', '--snapshot=' + this._snapshot_path,
+ '--checkoutdir=' + component_src,
+ '--metadata-path=' + temp_metadata_path]
+ if not self.buildopts.no_clean:
+ child_args.append('--clean')
+ child_args.extend(['--overwrite', basename])
+ if self.args.patches_path:
+ child_args.append('--patches-path=' + self.args.patches_path)
+ elif patchdir is not None:
+ child_args.append('--patches-path=' + patchdir)
+ run_sync(child_args)
+
+ os.unlink(temp_metadata_path)
+
+ component_resultdir = os.path.join(workdir, 'results')
+ fileutil.ensure_dir(component_resultdir)
+
+ self._write_status('Building ' + build_ref)
+
+ rootdir = self._compose_buildroot(workdir, basename, architecture)
+
+ tmpdir=os.path.join(workdir, 'tmp')
+ fileutil.ensure_dir(tmpdir)
+
+ src_compile_one_path = os.path.join(LIBDIR, 'ostbuild', 'ostree-build-compile-one')
+ dest_compile_one_path = os.path.join(rootdir, 'ostree-build-compile-one')
+ shutil.copy(src_compile_one_path, dest_compile_one_path)
+ os.chmod(dest_compile_one_path, 0755)
+
+ chroot_sourcedir = os.path.join('/ostbuild', 'source', basename)
+
+ current_machine = os.uname()[4]
+ if current_machine != architecture:
+ child_args = ['setarch', architecture]
+ else:
+ child_args = []
+ child_args.extend(buildutil.get_base_user_chroot_args())
+ child_args.extend([
+ '--mount-readonly', '/',
+ '--mount-proc', '/proc',
+ '--mount-bind', '/dev', '/dev',
+ '--mount-bind', tmpdir, '/tmp',
+ '--mount-bind', component_src, chroot_sourcedir,
+ '--mount-bind', component_resultdir, '/ostbuild/results',
+ '--chdir', chroot_sourcedir])
+ child_args.extend([rootdir, '/ostree-build-compile-one',
+ '--ostbuild-resultdir=/ostbuild/results',
+ '--ostbuild-meta=_ostbuild-meta.json'])
+ env_copy = dict(buildutil.BUILD_ENV)
+ env_copy['PWD'] = chroot_sourcedir
+ env_copy['CFLAGS'] = OPT_COMMON_CFLAGS[architecture]
+ env_copy['CXXFLAGS'] = OPT_COMMON_CFLAGS[architecture]
+
+ success = run_sync(child_args, stdout=t.logfile_stream,
+ stderr=t.logfile_stream, env=env_copy,
+ fatal_on_error=False)
+ if not success:
+ build_taskset.finish(False)
+ self._analyze_build_failure(t, architecture, component, component_src,
+ current_vcs_version, previous_vcs_version)
+ self._write_status('Failed building ' + build_ref)
+ fatal("Exiting due to build failure in component:%s arch:%s" % (component, architecture))
+
+ recorded_meta_path = os.path.join(component_resultdir, '_ostbuild-meta.json')
+ fileutil.write_json_file_atomic(recorded_meta_path, expanded_component)
+
+ args = ['ostree', '--repo=' + self.repo,
+ 'commit', '-b', build_ref, '-s', 'Build',
+ '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
+ '--skip-if-unchanged']
+
+ setuid_files = expanded_component.get('setuid', [])
+ statoverride_path = None
+ if len(setuid_files) > 0:
+ (fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-')
+ f = os.fdopen(fd, 'w')
+ for path in setuid_files:
+ f.write('+2048 ' + path)
+ f.write('\n')
+ f.close()
+ args.append('--statoverride=' + statoverride_path)
+
+ run_sync(args, stdout=t.logfile_stream,
+ stderr=t.logfile_stream,
+ cwd=component_resultdir)
+ if statoverride_path is not None:
+ os.unlink(statoverride_path)
+
+ if not self.args.no_clean_results:
+ if os.path.islink(component_src):
+ os.unlink(component_src)
+ else:
+ shutil.rmtree(component_src)
+ shutil.rmtree(component_resultdir)
+
+ shutil.rmtree(tmpdir)
+
+ ostree_revision = self._save_component_build(buildname, expanded_component)
+
+ build_taskset.finish(True)
+
+ return ostree_revision
+ },
+
+ function _composeOneTarget(self, target, componentBuildRevs, cancellable) {
+ let base = target['base'];
+ let baseName = 'bases/' + base['name'];
+ let runtimeName = 'bases/' + base['runtime'];
+ let develName = 'bases/' + base['devel'];
+
+ let rootdir = self.workdir.get_child('roots');
+ let composeRootdir = rootdir.get_child(target['name']);
+ GSystem.shutil_rm_rf(composeRootdir);
+ GSystem.file_ensure_directory(composeRootdir, cancellable);
+
+ let relatedRefs = {};
+ let baseRevision = ProcUtil.runSyncGetOutputUTF8Stripped(['ostree', '--repo=' + this.repo.get_path(),
+ 'rev-parse', baseName], cancellable);
+
+ let runtimeRevision = ProcUtil.runSyncGetOutputUTF8Stripped(['ostree', '--repo=' + self.repo.get_path(),
+ 'rev-parse', runtimeName], cancellable);
+ relatedRefs[runtimeName] = runtimeRevision;
+ develRevision = ProcUtil.runSyncGetOutputUTF8Stripped(['ostree', '--repo=' + self.repo.get_path(),
+ 'rev-parse', develName], cancellable);
+ relatedRefs[develName] = develRevision;
+
+ for (let name in componentBuildRevs) {
+ let rev = componentBuildRevs[name];
+ let buildRef = 'components/' + this._snapshot['prefix'] + '/' + name;
+ relatedRefs[buildRef] = rev;
+ }
+
+ let [relatedTmpPath, stream] = Gio.File.new_tmp("ostbuild-compose-XXXXXX.txt");
+ let dataOut = Gio.DataOutputStream.new(stream);
+ for (let name in relatedRefs) {
+ let rev = relatedRefs[name];
+ dataOut.put_string(name, cancellable);
+ dataOut.put_string(' ', cancellable);
+ dataOut.put_string(rev, cancellable);
+ dataOut.put_string('\n', cancellable);
+ }
+ dataOut.close(cancellable);
+
+ let composeContents = [[baseRevision, '/']];
+ for (let i = 0; i < target['contents'].length; i++) {
+ let treeContent = target['contents'][i];
+ let name = treeContent['name'];
+ let rev = componentBuildRevs[name];
+ let subtrees = treeContent['trees'];
+ for (let j = 0; j < subtrees.length; j++) {
+ let subpath = subtrees[j];
+ compose_contents.append([rev, subpath]);
+ }
+ }
+
+ let [contentsTmpPath, stream] = Gio.File.new_tmp("ostbuild-compose-XXXXXX.txt");
+ let dataOut = Gio.DataOutputStream.new(stream);
+ for (let i = 0; i < composeContents.length; i++) {
+ let [branch, subpath] = composeContents[i];
+ dataOut.put_string(branch, cancellable);
+ dataOut.put_byte(0, cancellable);
+ dataOut.put_string(subpath, cancellable);
+ dataOut.put_byte(0, cancellable);
+ }
+ dataOut.close(cancellable);
+
+ ProcUtil.runSync(['ostree', '--repo=' + self.repo.get_path(),
+ 'checkout', '--user-mode', '--no-triggers', '--union',
+ '--from-file=' + contentsTmpPath.get_path(), composeRootdir.get_path()],
+ cancellable);
+ GSystem.file_unlink(contentsTmppath, cancellable);
+
+ contentsPath = composeRootdir.get_child('contents.json');
+ JsonUtil.writeJsonFileAtomic(contentsPath, this._snapshot);
+
+ let treename = 'trees/' + target['name'];
+
+ ProcUtil.runSync(['ostree', '--repo=' + self.repo.get_path(),
+ 'commit', '-b', treename, '-s', 'Compose',
+ '--owner-uid=0', '--owner-gid=0', '--no-xattrs',
+ '--related-objects-file=' + relatedTmpPath.get_path(),
+ '--skip-if-unchanged'
+ ], cancellable, {cwd: composeRootdir.get_path()});
+ GSystem.file_unlink(relatedTmpPath, cancellable);
+ GSystem.shutil_rm_rf(composeRootdir, cancellable);
+ },
+
+ /* Build the Yocto base system. */
+ _buildBase: function(architecture, cancellable) {
+ let basemeta = this._snapshot['base'];
+ let checkoutdir = this.workdir.get_child('checkouts').get_child(basemeta['name']);
+ GSystem.file_ensure_directory(checkoutdir, cancellable);
+
+ let ftype = checkoutdir.query_file_type(Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS, cancellable);
+ if (ftype == Gio.FileType.SYMBOLIC_LINK)
+ GSystem.file_unlink(checkoutdir, cancellable);
+
+ let [keytype, uri] = Vcs.parseSrcKey(basemeta['src']);
+ if (keytype == 'local') {
+ GSystem.shutil_rm_rf(checkoutdir, cancellable);
+ checkoutdir.make_symbolic_link(uri, cancellable);
+ } else {
+ Vcs.getVcsCheckout(this._mirrordir, keytype, uri, checkoutdir,
+ basemeta['revision'], cancellable,
+ {overwrite:false});
+ }
+
+ let builddirName = Format.vprintf('build-%s-%s', [basemeta['name'], architecture]);
+ let builddir = this.workdir.get_child(builddirName);
+
+ // Just keep reusing the old working directory downloads and sstate
+ let oldBuilddir = this.workdir.get_child('build-' + basemeta['name']);
+ let sstateDir = oldBuilddir.get_child('sstate-cache');
+ let downloads = oldBuilddir.get_child('downloads');
+
+ let cmd = ['linux-user-chroot', '--unshare-pid', '/',
+ LIBDIR + '/ostbuild/ostree-build-yocto',
+ checkoutdir.get_path(), builddir.get_path(), architecture,
+ this.repo.get_path()];
+ // We specifically want to kill off any environment variables jhbuild
+ // may have set.
+ env = {};
+ Lang.copyProperties(BuildUtil.BUILD_ENV, env);
+ env['DL_DIR'] = downloads.get_path();
+ env['SSTATE_DIR'] = sstate_dir.get_path();
+ ProcUtil.runSync(cmd, cancellable, {env:env});
+ },
+
+ execute: function(argv) {
+ let cancellable = null;
+
+ let parser = new ArgParse.ArgumentParser("Build multiple components and generate trees");
+ parser.addArgument('--prefix');
+ parser.addArgument('--src-snapshot');
+ parser.addArgument('--patches-path');
+ parser.addArgument('components', {nargs:'*'});
+
+ let args = parser.parseArgs(argv);
+
+ this.config = Config.get();
+ this.workdir = Gio.File.new_for_path(this.config.getGlobal('workdir'));
+ this.mirrordir = Gio.File.new_for_path(this.config.getGlobal('mirrordir'));
+ this.prefix = args.prefix || this.config.getPrefix();
+ this._snapshotDir = this.workdir.get_child('snapshots');
+
+ this._srcDb = new JsonDB.JsonDB(this._snapshotDir, this.prefix + '-src-snapshot');
+ this._snapshot = Snapshot.load(this._srcDb, this.prefix, args.snapshot, cancellable);
+
+
+ this.forceBuildComponents = {};
+ this.cachedPatchdirRevision = null;
+
+ this.repo = this.workdir.get_child('repo');
+
+ GSystem.file_ensure_directory(this.repo, cancellable);
+ if (!this.repo.get_child('objects').query_exists(cancellable)) {
+ ProcUtil.runSync(['ostree', '--repo=' + self.repo.get_path(), 'init', '--archive'],
+ cancellable);
+ }
+
+ let components = this._snapshot['components'];
+
+ let prefix = this._snapshot['prefix'];
+ let basePrefix = this._snapshot['base']['name'] + '/' + prefix;
+ let architectures = this._snapshot['architectures'];
+
+ for (let i = 0; i < architectures.length; i++) {
+ this._buildBase(architectures[i], cancellable);
+ }
+
+ let componentToArches = {};
+
+ let runtimeComponents = [];
+ let develComponents = [];
+
+ for (let i = 0; i < components.length; i++) {
+ let component = components[i];
+ let name = component['name']
+
+ let isRuntime = (component['component'] || 'runtime') == 'runtime';
+
+ if (isRuntime) {
+ runtimeComponents.push(component);
+ }
+ develComponents.push(component);
+
+ let isNoarch = component['noarch'] || false;
+ let componentArches;
+ if (isNoarch) {
+ // Just use the first specified architecture
+ componentArches = [architectures[0]];
+ } else {
+ componentArches = component['architectures'] || architectures;
+ }
+ componentToArches[name] = componentArches;
+ }
+
+ for (let i = 0; i < args.components.length; i++) {
+ let name = args.components[i];
+ let component = Snapshot.getComponent(this._snapshot, name);
+ this._forceBuildComponents[name] = true;
+ }
+
+ let componentsToBuild = [];
+ let componentSkippedCount = 0;
+ let componentBuildRevs = {};
+
+ for (let i = 0; i < components.length; i++) {
+ let component = components[i];
+ for (let j = 0; j < architectures.length; j++) {
+ componentsToBuild.push([component, architectures[j]]);
+ }
+ }
+
+ this._componentBuildCachePath = this.workdir.get_child('component-builds.json');
+ if (this._componentBuildCachePath.query_exists(cancellable)) {
+ this._componentBuildCache = JsonUtil.loadJson(self._componentBuildCachePath));
+ } else {
+ this._componentBuildCache = {};
+ }
+
+ for (let i = 0; i < componentsToBuild.length; i++) {
+ let [component, architecture] = componentsToBuild[i];
+ let archname = component['name'] + '/' + architecture;
+ let buildRev = this._buildOneComponent(component, architecture, cancellable);
+ this._componentBuildRevs[archname] = buildRev;
+ }
+
+ let targetsList = [];
+ for (let targetComponentType in {'runtime', 'devel'}) {
+ for (let i = 0; i < architectures.length; i++) {
+ let architecture = architectures[i];
+ let target = {};
+ targetsList.push(target);
+ target['name'] = prefix + '-' + architecture + '-' + target_component_type;
+
+ let runtimeRef = base_prefix + '-' + architecture + '-runtime';
+ let buildrootRef = base_prefix + '-' + architecture + '-devel';
+ let baseRef;
+ if (targetComponentType == 'runtime') {
+ baseRef = runtimeRef;
+ } else {
+ baseFef = buildrootRef;
+ }
+ target['base'] = {'name': baseRef,
+ 'runtime': runtimeRef,
+ 'devel': buildrootRef};
+
+ let targetComponents;
+ if (targetComponentType == 'runtime') {
+ targetComponents = runtimeComponents;
+ } else {
+ targetComponents = develComponents;
+ }
+
+ let contents = [];
+ for (let i = 0; i < target_components.length; i++) {
+ let component = target_components[i];
+ if component['bootstrap'] {
+ continue;
+ }
+ let buildsForComponent = componentToArches[component['name']];
+ if (buildsForComponent.indexOf(architecture) == -1) {
+ continue;
+ }
+ let binaryName = component['name'] + '/' + architecture;
+ let componentRef = {'name': binaryName};
+ if (targetComponentType == 'runtime') {
+ componentRef['trees'] = ['/runtime'];
+ } else {
+ componentRef['trees'] = ['/runtime', '/devel', '/doc']
+ }
+ contents.push(componentRef);
+ }
+ target['contents'] = contents;
+ }
+ }
+
+ for (let i = 0; i < targets_list.length; i++) {
+ let target = targets_list[i];
+ log(Format.vprintf("Composing %r from %d components", [target['name'], target['contents'].length]));
+ this._composeOneTarget(target, componentBuildRevs, cancellable);
+ }
+ }
+});
+
+
+var app = new Build();
+GLib.idle_add(GLib.PRIORITY_DEFAULT,
+ function() { try { app.execute(ARGV); } finally { loop.quit(); }; return false; });
+loop.run();
+
diff --git a/src/ostbuild/js/procutil.js b/src/ostbuild/js/procutil.js
index a42b654..ed76aee 100644
--- a/src/ostbuild/js/procutil.js
+++ b/src/ostbuild/js/procutil.js
@@ -5,11 +5,19 @@ const GSystem = imports.gi.GSystem;
const Params = imports.params;
function _setContextFromParams(context, params) {
- params = Params.parse(params, {cwd: null});
+ params = Params.parse(params, {cwd: null,
+ env: null,
+ stderr: null });
if (typeof(params.cwd) == 'string')
context.set_cwd(params.cwd);
else if (params.cwd)
context.set_cwd(params.cwd.get_path());
+
+ if (params.env)
+ context.set_environment(params.env);
+
+ if (params.stderr)
+ context.set_stderr_disposition(params.stderr);
}
function _wait_sync_check_internal(proc, cancellable) {
@@ -65,6 +73,21 @@ function runSyncGetOutputUTF8(args, cancellable, params) {
return _runSyncGetOutputInternal(args, cancellable, params, false);
}
+function runSyncGetOutputUTF8Stripped(args, cancellable, params) {
+ return _runSyncGetOutputInternal(args, cancellable, params, false).replace(/[ \n]+$/, '');
+}
+
+function runSyncGetOutputUTF8StrippedOrNull(args, cancellable, params) {
+ try {
+ params.stderr = Gio.SubprocessStreamDisposition.NULL;
+ return runSyncGetOutputUTF8Stripped(args, cancellable, params);
+ } catch (e) {
+ if (e.domain == GLib.spawn_exit_error_quark())
+ return null;
+ throw e;
+ }
+}
+
function asyncWaitCheckFinish(process, result) {
let [waitSuccess, estatus] = process.wait_finish(result);
let success = false;
diff --git a/src/ostbuild/js/resolve.js b/src/ostbuild/js/resolve.js
index 48c547e..025d1a6 100644
--- a/src/ostbuild/js/resolve.js
+++ b/src/ostbuild/js/resolve.js
@@ -49,8 +49,6 @@ const Resolve = new Lang.Class({
help:"Also perform a git fetch"});
parser.addArgument('--fetch-keep-going', {action:'storeTrue',
help:"Don't exit on fetch failures"});
- parser.addArgument('--stamp-file',
- {help: "If manifest changes, create this file"});
parser.addArgument('components', {nargs:'*',
help:"List of component names to git fetch"});
diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py
index e86f40c..ee7bc28 100755
--- a/src/ostbuild/pyostbuild/builtin_build.py
+++ b/src/ostbuild/pyostbuild/builtin_build.py
@@ -1,43 +1,23 @@
-# Copyright (C) 2011 Colin Walters <walters verbum org>
-#
-# This library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library; if not, write to the
-# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-# Boston, MA 02111-1307, USA.
-
-import os,sys,subprocess,tempfile,re,shutil
-import argparse
-import time
-import urlparse
-import hashlib
-import json
-from StringIO import StringIO
-
-from . import builtins
-from .ostbuildlog import log, fatal
-from .subprocess_helpers import run_sync, run_sync_get_output
-from .subprocess_helpers import run_sync_monitor_log_file
-from . import ostbuildrc
-from . import buildutil
-from . import task
-from . import fileutil
-from . import kvfile
-from . import snapshot
-from . import odict
-from . import vcs
-
-OPT_COMMON_CFLAGS = {'i686': '-O2 -g -m32 -march=i686 -mtune=atom -fasynchronous-unwind-tables',
- 'x86_64': '-O2 -g -m64 -mtune=generic'}
+// Copyright (C) 2011 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+
+const OPT_COMMON_CFLAGS = {'i686': '-O2 -g -m32 -march=i686 -mtune=atom -fasynchronous-unwind-tables',
+ 'x86_64': '-O2 -g -m64 -mtune=generic'}
class BuildOptions(object):
pass
diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py
index 708e3dd..df97a03 100755
--- a/src/ostbuild/pyostbuild/main.py
+++ b/src/ostbuild/pyostbuild/main.py
@@ -29,7 +29,7 @@ JS_BUILTINS = {'autobuilder': "Run resolve and build",
'prefix': "Display or modify \"prefix\" (build target)",
'git-mirror': "Update internal git mirror for one or more components",
'resolve': "Expand git revisions in source to exact targets",
- 'qemu-pull-deploy': "Copy from local repository into qemu disk and deploy"};
+ 'build': "Build multiple components and generate trees"};
def usage(ecode):
print "Builtins:"
[
Date Prev][
Date Next] [
Thread Prev][
Thread Next]
[
Thread Index]
[
Date Index]
[
Author Index]
