[gobject-introspection] dumper.py: Use Distutils to Build Dumper Program

[Chun-wei Fan <fanchunwei src gnome org>]

commit 7ce646db1cb0842db2089df671b17081f82f2b0a
Author: Chun-wei Fan <fanchunwei src gnome org>
Date:   Mon Aug 25 11:28:41 2014 +0800

    dumper.py: Use Distutils to Build Dumper Program
    
    Add compiler() and link() functions in ccompiler.py to call
    distutil.ccompiler's compile() and link() functions, so that we can in turn
    call them from dumper.py to build the dumper program.  As distutils don't
    get along well with libtool libraries (ie .la files), we can deduce the
    libraries to link from using the file name .la file and include
    $(builddir)/.libs in the linking stage.
    
    For MSVC builds, we need to ignore mt.exe failing to find a .exe.manifest
    file as Visual Studio 2010+ do not generate such files during linking, and
    it is done by distutils as Python 2.7.x is built with Visual Studio 2008,
    which do generate such manifest files during the build.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=728313

 giscanner/ccompiler.py |  133 ++++++++++++++++++++++++++--------
 giscanner/dumper.py    |  188 ++++++++++++++++--------------------------------
 2 files changed, 164 insertions(+), 157 deletions(-)
---
diff --git a/giscanner/ccompiler.py b/giscanner/ccompiler.py
index f9105fe..da56c60 100644
--- a/giscanner/ccompiler.py
+++ b/giscanner/ccompiler.py
@@ -106,54 +106,76 @@ class CCompiler(object):
 
             self._cflags_no_deprecation_warnings = "-Wno-deprecated-declarations"
 
-    def get_internal_link_flags(self, args, libtool, libraries, libpaths):
+    def get_internal_link_flags(self, libtool, libraries, libpaths):
         # An "internal" link is where the library to be introspected
         # is being built in the current directory.
+        internal_link_args = []
 
         # Search the current directory first
-        # (This flag is not supported nor needed for Visual C++)
-        if not self.check_is_msvc():
-            args.append('-L.')
+        self.compiler.add_library_dir('.')
 
-            # https://bugzilla.gnome.org/show_bug.cgi?id=625195
+        if self.check_is_msvc():
+            for library in libraries:
+                # MSVC Builds don't use libtool, so no .la libraries,
+                # so just add the library directly.
+                self.compiler.add_library(library)
+            for libpath in libpaths:
+                self.compiler.add_library_dir(libpath)
+
+        else:
             if not libtool:
-                args.append('-Wl,-rpath=.')
-                args.append('-Wl,--no-as-needed')
+                # https://bugzilla.gnome.org/show_bug.cgi?id=625195
+                internal_link_args.append('-Wl,-rpath=.')
 
-        for library in libraries:
-            if self.check_is_msvc():
-                args.append(library + '.lib')
-            else:
+                # Ensure libraries are always linked as we are going to use ldd to work
+                # out their names later
+                internal_link_args.append('-Wl,--no-as-needed')
+
+            for library in libraries:
                 if library.endswith(".la"):  # explicitly specified libtool library
-                    args.append(library)
+                    self.compiler.add_library_dir('.libs')
+                    if os.name != 'nt':
+                        self.compiler.add_runtime_library_dir('.libs')
+                    if (library.startswith('lib')):
+                        self.compiler.add_library(library[len('lib'):library.rfind('.la')])
+                    else:
+                        self.compiler.add_library(library[:library.rfind('.la')])
                 else:
-                    args.append('-l' + library)
-
-        for library_path in libpaths:
-            # Not used/needed on Visual C++, and -Wl,-rpath options
-            # will cause grief
-            if not self.check_is_msvc():
-                args.append('-L' + library_path)
-                if os.path.isabs(library_path):
+                    self.compiler.add_library(library)
+
+            for libpath in libpaths:
+                self.compiler.add_library_dir(libpath)
+
+                # Not used/needed on Windows, add_runtime_library_dir()
+                # will cause grief on Windows
+                if os.name != 'nt':
+                    self.compiler.add_runtime_library_dir(libpath)
+                if os.path.isabs(libpath):
                     if libtool:
-                        args.append('-rpath')
-                        args.append(library_path)
+                        internal_link_args.append('-rpath')
+                        internal_link_args.append(libpath)
                     else:
-                        args.append('-Wl,-rpath=' + library_path)
+                        internal_link_args.append('-Wl,-rpath=' + libpath)
+
+        return internal_link_args
 
-    def get_external_link_flags(self, args, libraries):
+    def get_external_link_flags(self, libraries):
         # An "external" link is where the library to be introspected
         # is installed on the system; this case is used for the scanning
         # of GLib in gobject-introspection itself.
 
+        external_link_args = []
+
         for library in libraries:
-            if self.check_is_msvc():
-                args.append(library + '.lib')
-            else:
-                if library.endswith(".la"):  # explicitly specified libtool library
-                    args.append(library)
+            if library.endswith(".la"):  # explicitly specified libtool library
+                if (library.startswith('lib')):
+                    self.compiler.add_library(library[len('lib'):library.rfind('.la')])
                 else:
-                    args.append('-l' + library)
+                    self.compiler.add_library(library[:library.rfind('.la')])
+            else:
+                self.compiler.add_library(library)
+
+        return external_link_args
 
     def preprocess(self, source, output, cpp_options):
         extra_postargs = ['-C']
@@ -180,6 +202,57 @@ class CCompiler(object):
                                  include_dirs=include_dirs,
                                  extra_postargs=extra_postargs)
 
+    def compile(self, pkg_config_cflags, cpp_includes, source, init_sections, quiet):
+        extra_postargs = []
+        includes = []
+        cpp_options = None
+        source_str = ''.join(source)
+        tmpdir_idx = source_str.rfind(os.sep, 0, source_str.rfind(os.sep))
+        (include_paths, macros, extra_args) = \
+            self._set_cpp_options(pkg_config_cflags)
+
+        for include in cpp_includes:
+            includes.append(include)
+
+        # Do not add -Wall when using init code as we do not include any
+        # header of the library being introspected
+        if self.compiler_cmd == 'gcc' and not init_sections:
+            extra_postargs.append('-Wall')
+        extra_postargs.append(self._cflags_no_deprecation_warnings)
+
+        includes.extend(include_paths)
+        extra_postargs.extend(extra_args)
+
+        return self.compiler.compile(sources=source,
+                                     macros=macros,
+                                     include_dirs=includes,
+                                     extra_postargs=extra_postargs,
+                                     output_dir=source_str[tmpdir_idx + 1:
+                                                           source_str.rfind(os.sep)])
+
+    def link(self, output, objects, lib_args, libtool, quiet):
+        extra_preargs = []
+        extra_postargs = []
+        library_dirs = []
+        libraries = []
+        output_str = ''.join(output)
+        output_dir = output_str[:output_str.rfind(os.sep)]
+
+        if libtool:
+            if os.name == 'nt':
+                extra_postargs.append('-Wl,--export-all-symbols')
+            else:
+                extra_postargs.append('-export-dynamic')
+
+        for arg in lib_args:
+            extra_postargs.append(arg)
+
+        self.compiler.link(target_desc=self.compiler.EXECUTABLE,
+                           objects=objects,
+                           output_filename=output,
+                           extra_preargs=extra_preargs,
+                           extra_postargs=extra_postargs)
+
     def resolve_windows_libs(self, libraries, options):
         args = []
         libsearch = []
diff --git a/giscanner/dumper.py b/giscanner/dumper.py
index 9423177..46aa835 100644
--- a/giscanner/dumper.py
+++ b/giscanner/dumper.py
@@ -24,6 +24,7 @@ import sys
 import subprocess
 import shutil
 import tempfile
+from distutils.errors import LinkError
 
 from .gdumpparser import IntrospectionBinary
 from . import utils
@@ -92,7 +93,6 @@ class DumpCompiler(object):
         self._uninst_srcdir = os.environ.get('UNINSTALLED_INTROSPECTION_SRCDIR')
         self._packages = ['gio-2.0 gmodule-2.0']
         self._packages.extend(options.packages)
-        self._linker_cmd = os.environ.get('CC', 'cc')
 
     # Public API
 
@@ -149,29 +149,23 @@ class DumpCompiler(object):
             f.write("\n};\n")
         f.close()
 
-        # Microsoft compilers generate intermediate .obj files
-        # during compilation, unlike .o files like GCC and others
-        if self._compiler.check_is_msvc():
-            o_path = self._generate_tempfile(tmpdir, '.obj')
-        else:
-            o_path = self._generate_tempfile(tmpdir, '.o')
-
-        if os.name == 'nt':
-            ext = '.exe'
+        o_path = self._generate_tempfile(tmpdir, self._compiler.compiler.obj_extension)
+        if self._compiler.compiler.exe_extension:
+            ext = self._compiler.compiler.exe_extension
         else:
             ext = ''
 
         bin_path = self._generate_tempfile(tmpdir, ext)
 
         try:
-            self._compile(o_path, c_path)
+            introspection_obj = self._compile(o_path, c_path)
         except CompilerError as e:
             if not utils.have_debug_flag('save-temps'):
                 shutil.rmtree(tmpdir)
             raise SystemExit('compilation of temporary binary failed:' + str(e))
 
         try:
-            self._link(bin_path, o_path)
+            self._link(introspection_obj, bin_path, o_path)
         except LinkerError as e:
             if not utils.have_debug_flag('save-temps'):
                 shutil.rmtree(tmpdir)
@@ -199,128 +193,68 @@ class DumpCompiler(object):
         return proc.communicate()[0].split()
 
     def _compile(self, output, *sources):
-        # Not strictly speaking correct, but easier than parsing shell
-        args = self._compiler.compiler_cmd.split()
-        # Do not add -Wall when using init code as we do not include any
-        # header of the library being introspected
-        if self._compiler.compiler_cmd == 'gcc' and not self._options.init_sections:
-            args.append('-Wall')
-        # The Microsoft compiler uses different option flags for
-        # silencing warnings on deprecated function usage
-        if self._compiler.check_is_msvc():
-            args.append("-wd4996")
-        else:
-            args.append("-Wno-deprecated-declarations")
         pkgconfig_flags = self._run_pkgconfig('--cflags')
-        args.extend([utils.cflag_real_include_path(f) for f in pkgconfig_flags])
-        cppflags = os.environ.get('CPPFLAGS', '')
-        for cppflag in cppflags.split():
-            args.append(cppflag)
-        cflags = os.environ.get('CFLAGS', '')
-        for cflag in cflags.split():
-            args.append(cflag)
-        for include in self._options.cpp_includes:
-            args.append('-I' + include)
-        # The Microsoft compiler uses different option flags for
-        # compilation result output
-        if self._compiler.check_is_msvc():
-            args.extend(['-c', '-Fe' + output, '-Fo' + output])
-        else:
-            args.extend(['-c', '-o', output])
-        for source in sources:
-            if not os.path.exists(source):
-                raise CompilerError(
-                    "Could not find c source file: %s" % (source, ))
-        args.extend(list(sources))
-        if not self._options.quiet:
-            print "g-ir-scanner: compile: %s" % (
-                subprocess.list2cmdline(args), )
-            sys.stdout.flush()
-        try:
-            subprocess.check_call(args)
-        except subprocess.CalledProcessError as e:
-            raise CompilerError(e)
 
-    def _link(self, output, *sources):
-        args = []
+        return self._compiler.compile(pkgconfig_flags,
+                                      self._options.cpp_includes,
+                                      sources,
+                                      self._options.init_sections,
+                                      self._options.quiet)
+
+    def _add_link_internal_args(self, libtool):
+        # An "internal" link is where the library to be introspected
+        # is being built in the current directory.
+        libs = []
+        libs.extend(self._compiler.get_internal_link_flags(libtool,
+                                               self._options.libraries,
+                                               self._options.library_paths))
+
+        libs.extend(self._run_pkgconfig('--libs'))
+
+        return libs
+
+    def _add_link_external_args(self):
+        # An "external" link is where the library to be introspected
+        # is installed on the system; this case is used for the scanning
+        # of GLib in gobject-introspection itself.
+        libs = []
+
+        libs.extend(self._run_pkgconfig('--libs'))
+
+        libs.extend(self._compiler.get_external_link_flags(self._options.libraries))
+        return libs
+
+    def _link(self, introspection_obj, output, *sources):
+        link_args = []
+
         libtool = utils.get_libtool_command(self._options)
-        if libtool:
-            args.extend(libtool)
-            args.append('--mode=link')
-            args.append('--tag=CC')
-            if self._options.quiet:
-                args.append('--silent')
-
-        args.extend(self._linker_cmd.split())
-        # We can use -o for the Microsoft compiler/linker,
-        # but it is considered deprecated usage with that
-        if self._compiler.check_is_msvc():
-            args.extend(['-Fe' + output])
-        else:
-            args.extend(['-o', output])
-        if libtool:
-            if os.name == 'nt':
-                args.append('-Wl,--export-all-symbols')
-            else:
-                args.append('-export-dynamic')
-
-        cppflags = os.environ.get('CPPFLAGS', '')
-        for cppflag in cppflags.split():
-            args.append(cppflag)
-        cflags = os.environ.get('CFLAGS', '')
-        for cflag in cflags.split():
-            args.append(cflag)
-        ldflags = os.environ.get('LDFLAGS', '')
-        for ldflag in ldflags.split():
-            args.append(ldflag)
-
-        # Make sure to list the library to be introspected first since it's
-        # likely to be uninstalled yet and we want the uninstalled RPATHs have
-        # priority (or we might run with installed library that is older)
-
-        for source in sources:
-            if not os.path.exists(source):
-                raise CompilerError(
-                    "Could not find object file: %s" % (source, ))
-        args.extend(list(sources))
-
-        pkg_config_libs = self._run_pkgconfig('--libs')
 
         if not self._options.external_library:
-            self._compiler.get_internal_link_flags(args,
-                                                   libtool,
-                                                   self._options.libraries,
-                                                   self._options.library_paths)
-            args.extend(pkg_config_libs)
-
+            link_args.extend(self._add_link_internal_args(libtool))
         else:
-            args.extend(pkg_config_libs)
-            self._compiler.get_external_link_flags(args, self._options.libraries)
-
-        if not self._options.quiet:
-            print "g-ir-scanner: link: %s" % (
-                subprocess.list2cmdline(args), )
-            sys.stdout.flush()
-        msys = os.environ.get('MSYSTEM', None)
-        if msys:
-            shell = os.environ.get('SHELL', 'sh.exe')
-            # Create a temporary script file that
-            # runs the command we want
-            tf, tf_name = tempfile.mkstemp()
-            f = os.fdopen(tf, 'wb')
-            shellcontents = ' '.join([x.replace('\\', '/') for x in args])
-            fcontents = '#!/bin/sh\nunset PWD\n{}\n'.format(shellcontents)
-            f.write(fcontents)
-            f.close()
-            shell = utils.which(shell)
-            args = [shell, tf_name.replace('\\', '/')]
+            link_args.extend(self._add_link_external_args())
+
         try:
-            subprocess.check_call(args)
-        except subprocess.CalledProcessError as e:
-            raise LinkerError(e)
-        finally:
-            if msys:
-                os.remove(tf_name)
+            self._compiler.link(output,
+                                introspection_obj,
+                                link_args,
+                                libtool,
+                                self._options.quiet)
+
+        # Ignore failing to embed the manifest files, when the manifest
+        # file does not exist, especially for MSVC 2010 and later builds.
+        # If we are on Visual C++ 2005/2008, where
+        # this embedding is required, the build will fail anyway, as
+        # the dumper program will likely fail to run, and this means
+        # something went wrong with the build.
+        except LinkError, e:
+            msg = str(e)
+
+            if msg[msg.rfind('mt.exe'):] == 'mt.exe\' failed with exit status 31':
+                sys.exc_clear()
+                pass
+            else:
+                raise LinkError(e)
 
 
 def compile_introspection_binary(options, get_type_functions,