GNOME.org
 

[gnome-ostree] build: Move more of build-compile-one into JS code

commit ba6098a9e4cbd716088b02026429d3654914bd4c
Author: Colin Walters <walters verbum org>
Date:   Fri Jan 18 18:23:28 2013 -0500

    build: Move more of build-compile-one into JS code
    
    Longer term we want to analysis here, etc.

 Makefile-ostbuild.am                  |    1 +
 src/ostbuild/js/fileutil.js           |   62 +++++++++++++++++
 src/ostbuild/js/tasks/task-build.js   |  111 ++++++++++++++++++++++++++++++-
 src/ostbuild/ostree-build-compile-one |  117 +--------------------------------
 4 files changed, 173 insertions(+), 118 deletions(-)
---
diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am
index 24c73ce..ed921e5 100644
--- a/Makefile-ostbuild.am
+++ b/Makefile-ostbuild.am
@@ -43,6 +43,7 @@ jsostbuild_DATA= \
 	src/ostbuild/js/buildutil.js \
 	src/ostbuild/js/builtin.js \
 	src/ostbuild/js/config.js \
+	src/ostbuild/js/fileutil.js \
 	src/ostbuild/js/task.js \
 	src/ostbuild/js/jsondb.js \
 	src/ostbuild/js/jsonutil.js \
diff --git a/src/ostbuild/js/fileutil.js b/src/ostbuild/js/fileutil.js
new file mode 100644
index 0000000..fee9ecc
--- /dev/null
+++ b/src/ostbuild/js/fileutil.js
@@ -0,0 +1,62 @@
+// Copyright (C) 2012,2013 Colin Walters <walters verbum org>
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the
+// Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+// Boston, MA 02111-1307, USA.
+
+const GLib = imports.gi.GLib;
+const Gio = imports.gi.Gio;
+
+const Params = imports.params;
+
+function walkDirInternal(dir, matchParams, callback, cancellable, queryStr) {
+    let denum = dir.enumerate_children(queryStr, Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS,
+				       cancellable);
+    let info;
+    let subdirs = [];
+    while ((info = denum.next_file(cancellable)) != null) {
+	let name = info.get_name();
+	let child = dir.get_child(name);
+	let ftype = info.get_file_type();
+	
+	if (ftype == Gio.FileType.DIRECTORY) {
+	    subdirs.push(child);
+	    continue;
+	}
+
+	if (matchParams.nameRegex && matchParams.nameRegex.exec(name) === null)
+	    continue;
+	if (matchParams.fileType !== null && matchParams.fileType != info.get_file_type())
+	    continue;
+	if (matchParams.contentType != null && matchParams.contentType != info.get_content_type())
+	    continue;
+	callback(child, cancellable);
+    }
+
+    denum.close(cancellable);
+
+    for (let i = 0; i < subdirs.length; i++) {
+	walkDirInternal(subdirs[i], matchParams, callback, cancellable, queryStr);
+    }
+}
+
+function walkDir(dir, matchParams, callback, cancellable) {
+    matchParams = Params.parse(matchParams, { nameRegex: null,
+					      fileType: null,
+					      contentType: null });
+    let queryStr = 'standard::name,standard::type,unix::mode';
+    if (matchParams.contentType)
+	queryStr += ',standard::fast-content-type';
+    walkDirInternal(dir, matchParams, callback, cancellable, queryStr);
+}
diff --git a/src/ostbuild/js/tasks/task-build.js b/src/ostbuild/js/tasks/task-build.js
index c03fd96..8b8706a 100644
--- a/src/ostbuild/js/tasks/task-build.js
+++ b/src/ostbuild/js/tasks/task-build.js
@@ -25,6 +25,7 @@ const GSystem = imports.gi.GSystem;
 const Builtin = imports.builtin;
 const Task = imports.task;
 const JsonDB = imports.jsondb;
+const FileUtil = imports.fileutil;
 const ProcUtil = imports.procutil;
 const StreamUtil = imports.streamutil;
 const JsonUtil = imports.jsonutil;
@@ -37,6 +38,11 @@ const ArgParse = imports.argparse;
 const OPT_COMMON_CFLAGS = {'i686': '-O2 -g -m32 -march=i686 -mtune=atom -fasynchronous-unwind-tables',
                            'x86_64': '-O2 -g -m64 -mtune=generic'};
 
+const DEVEL_DIRS = ['usr/include', 'usr/share/aclocal',
+		    'usr/share/pkgconfig', 'usr/lib/pkgconfig'];
+const DOC_DIRS = ['usr/share/doc', 'usr/share/gtk-doc',
+		  'usr/share/man', 'usr/share/info'];
+
 const TaskBuild = new Lang.Class({
     Name: "TaskBuild",
     Extends: Task.TaskDef,
@@ -281,6 +287,100 @@ const TaskBuild = new Lang.Class({
         return cachedata['ostree'];
     },
 
+    _installAndUnlinkRecurse: function(buildResultDir, srcFile, srcInfo, finalResultDir, cancellable) {
+	let relpath = buildResultDir.get_relative_path(srcFile);
+	let destFile;
+	if (relpath === null)
+	    destFile = finalResultDir;
+	else
+	    destFile = finalResultDir.resolve_relative_path(relpath);
+
+	GSystem.file_ensure_directory(destFile.get_parent(), true, cancellable);
+	
+	if (srcInfo.get_file_type() != Gio.FileType.SYMBOLIC_LINK) {
+	    let minimalMode = 436; // u+rw,g+rw,o+r
+	    if (srcInfo.get_file_type() == Gio.FileType.DIRECTORY)
+		minimalMode |= 64; // u+x
+	    GSystem.file_chmod(srcFile, minimalMode, cancellable);
+	}
+
+	if (srcInfo.get_file_type() == Gio.FileType.DIRECTORY) {
+	    GSystem.file_ensure_directory(destFile, true, cancellable);
+	    let e = srcFile.enumerate_children('standard::*,unix::mode', Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS, cancellable);
+	    let info;
+	    while ((info = e.next_file(cancellable)) !== null) {
+		let child = e.get_child(info);
+		this._installAndUnlinkRecurse(buildResultDir, child, info, finalResultDir, cancellable);
+	    }
+	    e.close(cancellable);
+	    srcFile.delete(cancellable);
+	} else {
+	    GSystem.file_linkcopy(srcFile, destFile, 0, cancellable);
+	    GSystem.file_unlink(srcFile, cancellable);
+	} 
+    },
+
+    _installAndUnlink: function(buildResultDir, srcFile, finalResultDir, cancellable) {
+	let srcInfo = srcFile.query_info('standard::*,unix::mode', Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS, cancellable);
+	this._installAndUnlinkRecurse(buildResultDir, srcFile, srcInfo, finalResultDir, cancellable);
+    },
+    
+    _processBuildResults: function(component, buildResultDir, finalResultDir, cancellable) {
+	let runtimePath = finalResultDir.get_child('runtime');
+	GSystem.file_ensure_directory(runtimePath, true, cancellable);
+	let develPath = finalResultDir.get_child('devel');
+	GSystem.file_ensure_directory(develPath, true, cancellable);
+	let docPath = finalResultDir.get_child('doc');
+	GSystem.file_ensure_directory(docPath, true, cancellable);
+
+	// First, remove /var from the install - components are required to
+	// auto-create these directories on demand.
+	let varPath = buildResultDir.get_child('var');
+	GSystem.shutil_rm_rf(varPath, cancellable);
+
+	// Python .co files contain timestamps
+	// .la files are generally evil
+	let DELETE_PATTERNS = [{ nameRegex: /\.(py[co])|(la)$/ },
+			       { nameRegex: /\.la$/,
+				 fileType: Gio.FileType.REGULAR }];
+			       
+	for (let i = 0; i < DELETE_PATTERNS.length; i++) {
+	    let pattern = DELETE_PATTERNS[i];
+	    FileUtil.walkDir(buildResultDir, pattern,
+			     Lang.bind(this, function(filePath, cancellable) {
+				 GSystem.file_unlink(filePath, cancellable);
+			     }), cancellable);
+	}
+
+	// Move symbolic links for shared libraries to devel
+	let libdir = buildResultDir.resolve_relative_path('usr/lib');
+	FileUtil.walkDir(libdir, { nameRegex: /\.so$/,
+				   fileType: Gio.FileType.SYMBOLIC_LINK },
+			 Lang.bind(this, function(filePath, cancellable) {
+			     this._installAndUnlink(buildResultDir, filePath, develPath, cancellable);
+			 }), cancellable);
+
+	for (let i = 0; i < DEVEL_DIRS.length; i++) {
+	    let path = DEVEL_DIRS[i];
+	    let oneDevelDir = buildResultDir.resolve_relative_path(path);
+	    
+	    if (oneDevelDir.query_exists(null)) {
+		this._installAndUnlink(buildResultDir, oneDevelDir, develPath, cancellable);
+	    }
+	}
+
+	for (let i = 0; i < DOC_DIRS.length; i++) {
+	    let path = DOC_DIRS[i];
+	    let oneDocDir = buildResultDir.resolve_relative_path(path);
+	    
+	    if (oneDocDir.query_exists(null)) {
+		this._installAndUnlink(buildResultDir, oneDocDir, docPath, cancellable);
+	    }
+	}
+
+	this._installAndUnlink(buildResultDir, buildResultDir, runtimePath, cancellable);
+    },
+
     _onBuildComplete: function(taskset, success, msg, loop) {
 	this._currentBuildSucceded = success;
 	this._currentBuildSuccessMsg = msg;
@@ -290,6 +390,7 @@ const TaskBuild = new Lang.Class({
     _buildOneComponent: function(component, architecture, cancellable) {
         let basename = component['name'];
 
+
 	let prefix = this._snapshot.data['prefix'];
         let buildname = Format.vprintf('%s/%s/%s', [prefix, basename, architecture]);
         let unixBuildname = buildname.replace(/\//g, '_');
@@ -428,7 +529,13 @@ const TaskBuild = new Lang.Class({
 	print("Started child process " + context.argv.map(GLib.shell_quote).join(' '));
 	proc.wait_sync_check(cancellable);
 
-        let recordedMetaPath = componentResultdir.get_child('_ostbuild-meta.json');
+	let finalBuildResultDir = buildWorkdir.get_child('post-results');
+	GSystem.shutil_rm_rf(finalBuildResultDir, cancellable);
+        GSystem.file_ensure_directory(finalBuildResultDir, true, cancellable);
+
+	this._processBuildResults(component, componentResultdir, finalBuildResultDir, cancellable);
+
+        let recordedMetaPath = finalBuildResultDir.get_child('_ostbuild-meta.json');
         JsonUtil.writeJsonFileAtomic(recordedMetaPath, expandedComponent, cancellable);
 
         let commitArgs = ['ostree', '--repo=' + this.repo.get_path(),
@@ -450,7 +557,7 @@ const TaskBuild = new Lang.Class({
             commitArgs.push('--statoverride=' + statoverridePath.get_path());
 	}
 
-        ProcUtil.runSync(commitArgs, cancellable, {cwd: componentResultdir,
+        ProcUtil.runSync(commitArgs, cancellable, {cwd: finalBuildResultDir,
 						   logInitiation: true});
         if (statoverridePath != null)
             GSystem.file_unlink(statoverridePath, cancellable);
diff --git a/src/ostbuild/ostree-build-compile-one b/src/ostbuild/ostree-build-compile-one
index 7651171..08aaf9b 100755
--- a/src/ostbuild/ostree-build-compile-one
+++ b/src/ostbuild/ostree-build-compile-one
@@ -88,8 +88,6 @@ _DEVEL_DIRS = ['usr/include',
                'usr/share/pkgconfig',
                'usr/lib/pkgconfig']
 
-tempfiles = []
-
 def _has_buildapi_configure_variable(name):
     var = '#buildapi-variable-%s' % (name, )
     for line in open('configure'):
@@ -218,125 +216,12 @@ def main(args):
 
     run_sync(args, cwd=builddir)
 
-    tempdir = tempfile.mkdtemp(prefix='ostbuild-destdir-%s' % (metadata['name'].replace('/', '_'), ))
-    tempfiles.append(tempdir)
-    args = ['make', 'install', 'DESTDIR=' + tempdir]
+    args = ['make', 'install', 'DESTDIR=' + ostbuild_resultdir]
     run_sync(args, cwd=builddir)
 
-    runtime_path = os.path.join(ostbuild_resultdir, 'runtime')
-    devel_path = os.path.join(ostbuild_resultdir, 'devel')
-    doc_path = os.path.join(ostbuild_resultdir, 'doc')
-    for artifact_type in ['runtime', 'devel', 'doc']:
-        resultdir = os.path.join(ostbuild_resultdir, artifact_type)
-        if os.path.isdir(resultdir):
-            shutil.rmtree(resultdir)
-        os.makedirs(resultdir)
-
-    # Remove /var from the install - components are required to
-    # auto-create these directories on demand.
-    varpath = os.path.join(tempdir, 'var')
-    if os.path.isdir(varpath):
-        shutil.rmtree(varpath)
-
-    # Delete all .la files.  See:
-    # https://bugzilla.gnome.org/show_bug.cgi?id=654013
-    libdir = os.path.join(tempdir, 'usr/lib')
-    for dirpath, subdirs, files in os.walk(libdir):
-        for filename in files:
-            path = os.path.join(dirpath, filename)
-            if filename.endswith('.la'):
-                os.unlink(path)
-
-    # Move symbolic links for shared libraries as well
-    # as static libraries into /devel.
-    if os.path.exists(libdir):
-        for filename in os.listdir(libdir):
-            path = os.path.join(libdir, filename)
-            stbuf = os.lstat(path)
-            if not ((filename.endswith('.so')
-                     and stat.S_ISLNK(stbuf.st_mode))
-                    or filename.endswith('.a')):
-                    continue
-            dest = os.path.join(devel_path, 'usr/lib', filename)
-            _install_and_unlink(path, dest)
-
-    for dirname in _DEVEL_DIRS:
-        dirpath = os.path.join(tempdir, dirname)
-        if os.path.isdir(dirpath):
-            dest = os.path.join(devel_path, dirname)
-            _install_and_unlink(dirpath, dest)
-
-    for dirname in _DOC_DIRS:
-        dirpath = os.path.join(tempdir, dirname)
-        if os.path.isdir(dirpath):
-            dest = os.path.join(doc_path, dirname)
-            _install_and_unlink(dirpath, dest)
-    
-    for filename in os.listdir(tempdir):
-        src_path = os.path.join(tempdir, filename)
-        dest_path = os.path.join(runtime_path, filename)
-        _install_and_unlink(src_path, dest_path)
-
-    for tmpname in tempfiles:
-        assert os.path.isabs(tmpname)
-        if os.path.isdir(tmpname):
-            shutil.rmtree(tmpname)
-        else:
-            try:
-                os.unlink(tmpname)
-            except OSError, e:
-                pass
-
     endtime = time.time()
 
     log("Compilation succeeded; %d seconds elapsed" % (int(endtime - starttime),))
     log("Results placed in %s" % (ostbuild_resultdir, ))
 
-def _install_and_unlink(src, dest):
-    statsrc = os.lstat(src)
-    dirname = os.path.dirname(dest)
-    if not os.path.isdir(dirname):
-        os.makedirs(dirname)
-
-    # Ensure that all installed files are at least rw-rw-r--;
-    # we don't support private/hidden files.
-    # Directories also need u+x, i.e. they're rwxrw-r--
-    if not stat.S_ISLNK(statsrc.st_mode):
-        minimal_mode = (stat.S_IRUSR | stat.S_IWUSR |
-                        stat.S_IRGRP | stat.S_IWGRP |
-                        stat.S_IROTH)
-        if stat.S_ISDIR(statsrc.st_mode):
-            minimal_mode |= stat.S_IXUSR
-        os.chmod(src, statsrc.st_mode | minimal_mode)
-
-    if stat.S_ISDIR(statsrc.st_mode):
-        if not os.path.isdir(dest):
-            os.mkdir(dest)
-        for filename in os.listdir(src):
-            src_child = os.path.join(src, filename)
-            dest_child = os.path.join(dest, filename)
-
-            _install_and_unlink(src_child, dest_child)
-        os.rmdir(src)
-    else:
-        basename = os.path.basename(src)
-        ignored = False
-        for r in _IGNORE_FILENAME_REGEXPS:
-            if r.match(basename):
-                ignored = True
-                break
-        if ignored:
-            log("Not installing %s" % (src, ))
-            os.unlink(src)
-            return
-        try:
-            os.rename(src, dest)
-        except OSError, e:
-            if stat.S_ISLNK(statsrc.st_mode):
-                linkto = os.readlink(src)
-                os.symlink(linkto, dest)
-            else:
-                shutil.copy2(src, dest)
-            os.unlink(src)
-
 main(sys.argv[1:])
[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]