[gobject-introspection] giscanner: Use Distutils to Build Dumper Program

[Chun-wei Fan <fanchunwei src gnome org>]

commit d1f62064ff2a570d8648c622e5b59f3a561f7e9a
Author: Chun-wei Fan <fanchunwei src gnome org>
Date:   Mon Aug 31 15:50:24 2015 +0800

    giscanner: Use Distutils to Build Dumper Program
    
    Add compile() and link() functions in ccompiler.py to call
    distutils.ccompiler's compiler() and link() functions, so that we
    can in turn call them from dumper.py as needed.  Note that for
    linking the dumper program when building with libtool, we are still
    using the original method of constructing the link command line
    and running that command line, as distutils and libtool do not get
    along well with each other.  For non-libtool builds, such as MSVC
    builds, we would link the dumper program using distutils.
    
    For MSVC builds, we need to ignore mt.exe failing to find a
    .exe.manifest file as Visual Studio 2010+ do not generate such files
    during linking, and it is done by distutils as Python 2.7.x is built
    with Visual Studio 2008, which do generate such manifest files during
    the build.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=753428

 giscanner/ccompiler.py |  104 +++++++++++++++++++++------
 giscanner/dumper.py    |  187 +++++++++++++++++++++++-------------------------
 2 files changed, 172 insertions(+), 119 deletions(-)
---
diff --git a/giscanner/ccompiler.py b/giscanner/ccompiler.py
index 438f847..85ff026 100644
--- a/giscanner/ccompiler.py
+++ b/giscanner/ccompiler.py
@@ -72,7 +72,7 @@ class CCompiler(object):
             self.compiler = distutils.ccompiler.new_compiler(compiler=compiler_name)
         customize_compiler(self.compiler)
 
-        # customize_compiler from distutils only does customization
+        # customize_compiler() from distutils only does customization
         # for 'unix' compiler type.  Also, avoid linking to msvcrxx.dll
         # for MinGW builds as the dumper binary does not link to the
         # Python DLL, but link to msvcrt.dll if necessary.
@@ -110,45 +110,60 @@ class CCompiler(object):
         # An "internal" link is where the library to be introspected
         # is being built in the current directory.
 
-        # Search the current directory first
-        # (This flag is not supported nor needed for Visual C++)
-        if not self.check_is_msvc():
-            args.append('-L.')
+        if not libtool:
+            # non-libtool case: prepare distutils use
+            if self.check_is_msvc():
+                for library in libraries:
+                    # MSVC Builds don't use libtool, so no .la libraries,
+                    # so just add the library directly.
+                    self.compiler.add_library(library)
+                    for libpath in libpaths:
+                        self.compiler.add_library_dir(libpath)
+            else:
+                # Search the current directory first
+                # (This flag is not supported nor needed for Visual C++)
+                self.compiler.add_library_dir('.')
+                if os.name != 'nt':
+                    self.compiler.add_runtime_library_dir('.')
 
-            # https://bugzilla.gnome.org/show_bug.cgi?id=625195
-            if not libtool:
+                # https://bugzilla.gnome.org/show_bug.cgi?id=625195
                 args.append('-Wl,-rpath=.')
+
+                # Ensure libraries are always linked as we are going to use ldd to work
+                # out their names later
                 args.append('-Wl,--no-as-needed')
 
-        for library in libraries:
-            if self.check_is_msvc():
-                args.append(library + '.lib')
-            else:
+            for library in libraries:
+                self.compiler.add_library(library)
+            if not self.check_is_msvc():
+                for library_path in libpaths:
+                    args.append('-L' + library_path)
+                    if os.path.isabs(library_path):
+                        args.append('-Wl,-rpath=' + library_path)
+
+        else:
+            # libtool case: assemble linker command arguments, like we did before
+            args.append('-L.')
+            for library in libraries:
                 if library.endswith(".la"):  # explicitly specified libtool library
                     args.append(library)
                 else:
                     args.append('-l' + library)
 
-        for library_path in libpaths:
-            # Not used/needed on Visual C++, and -Wl,-rpath options
-            # will cause grief
-            if not self.check_is_msvc():
+            for library_path in libpaths:
                 args.append('-L' + library_path)
                 if os.path.isabs(library_path):
-                    if libtool:
-                        args.append('-rpath')
-                        args.append(library_path)
-                    else:
-                        args.append('-Wl,-rpath=' + library_path)
+                    args.append('-rpath')
+                    args.append(library_path)
 
-    def get_external_link_flags(self, args, libraries):
+    def get_external_link_flags(self, args, libtool, libraries):
         # An "external" link is where the library to be introspected
         # is installed on the system; this case is used for the scanning
         # of GLib in gobject-introspection itself.
 
         for library in libraries:
-            if self.check_is_msvc():
-                args.append(library + '.lib')
+            if not libtool:
+                self.compiler.add_library(library)
             else:
                 if library.endswith(".la"):  # explicitly specified libtool library
                     args.append(library)
@@ -180,6 +195,49 @@ class CCompiler(object):
                                  include_dirs=include_dirs,
                                  extra_postargs=extra_postargs)
 
+    def compile(self, pkg_config_cflags, cpp_includes, source, init_sections):
+        extra_postargs = []
+        includes = []
+        source_str = ''.join(source)
+        tmpdir_idx = source_str.rfind(os.sep, 0, source_str.rfind(os.sep))
+        (include_paths, macros, extra_args) = \
+            self._set_cpp_options(pkg_config_cflags)
+
+        for include in cpp_includes:
+            includes.append(include)
+
+        # Do not add -Wall when using init code as we do not include any
+        # header of the library being introspected
+        if self.compiler_cmd == 'gcc' and not init_sections:
+            extra_postargs.append('-Wall')
+        extra_postargs.append(self._cflags_no_deprecation_warnings)
+
+        includes.extend(include_paths)
+        extra_postargs.extend(extra_args)
+
+        return self.compiler.compile(sources=source,
+                                     macros=macros,
+                                     include_dirs=includes,
+                                     extra_postargs=extra_postargs,
+                                     output_dir=source_str[tmpdir_idx + 1:
+                                                           source_str.rfind(os.sep)])
+
+    def link(self, output, objects, lib_args):
+        # Note: This is used for non-libtool builds only!
+        extra_preargs = []
+        extra_postargs = []
+        library_dirs = []
+        libraries = []
+
+        for arg in lib_args:
+            extra_postargs.append(arg)
+
+        self.compiler.link(target_desc=self.compiler.EXECUTABLE,
+                           objects=objects,
+                           output_filename=output,
+                           extra_preargs=extra_preargs,
+                           extra_postargs=extra_postargs)
+
     def resolve_windows_libs(self, libraries, options):
         args = []
         libsearch = []
diff --git a/giscanner/dumper.py b/giscanner/dumper.py
index 3a7ced6..18725c8 100644
--- a/giscanner/dumper.py
+++ b/giscanner/dumper.py
@@ -24,6 +24,7 @@ import sys
 import subprocess
 import shutil
 import tempfile
+from distutils.errors import LinkError
 
 from .gdumpparser import IntrospectionBinary
 from . import utils
@@ -92,7 +93,10 @@ class DumpCompiler(object):
         self._uninst_srcdir = os.environ.get('UNINSTALLED_INTROSPECTION_SRCDIR')
         self._packages = ['gio-2.0 gmodule-2.0']
         self._packages.extend(options.packages)
-        self._linker_cmd = os.environ.get('CC', 'cc')
+        if hasattr(self._compiler.compiler, 'linker_exe'):
+            self._linker_cmd = self._compiler.compiler.linker_exe
+        else:
+            self._linker_cmd = []
 
     # Public API
 
@@ -147,29 +151,22 @@ class DumpCompiler(object):
                     f.write("  " + func)
                 f.write("\n};\n")
 
-        # Microsoft compilers generate intermediate .obj files
-        # during compilation, unlike .o files like GCC and others
-        if self._compiler.check_is_msvc():
-            o_path = self._generate_tempfile(tmpdir, '.obj')
-        else:
-            o_path = self._generate_tempfile(tmpdir, '.o')
-
-        if os.name == 'nt':
-            ext = '.exe'
+        if self._compiler.compiler.exe_extension:
+            ext = self._compiler.compiler.exe_extension
         else:
             ext = ''
 
         bin_path = self._generate_tempfile(tmpdir, ext)
 
         try:
-            self._compile(o_path, c_path)
+            introspection_obj = self._compile(c_path)
         except CompilerError as e:
             if not utils.have_debug_flag('save-temps'):
                 shutil.rmtree(tmpdir)
             raise SystemExit('compilation of temporary binary failed:' + str(e))
 
         try:
-            self._link(bin_path, o_path)
+            self._link(bin_path, introspection_obj)
         except LinkerError as e:
             if not utils.have_debug_flag('save-temps'):
                 shutil.rmtree(tmpdir)
@@ -196,91 +193,59 @@ class DumpCompiler(object):
             stdout=subprocess.PIPE)
         return proc.communicate()[0].split()
 
-    def _compile(self, output, *sources):
-        # Not strictly speaking correct, but easier than parsing shell
-        args = self._compiler.compiler_cmd.split()
-        # Do not add -Wall when using init code as we do not include any
-        # header of the library being introspected
-        if self._compiler.compiler_cmd == 'gcc' and not self._options.init_sections:
-            args.append('-Wall')
-        # The Microsoft compiler uses different option flags for
-        # silencing warnings on deprecated function usage
-        if self._compiler.check_is_msvc():
-            args.append("-wd4996")
-        else:
-            args.append("-Wno-deprecated-declarations")
+    def _compile(self, *sources):
         pkgconfig_flags = self._run_pkgconfig('--cflags')
-        args.extend([utils.cflag_real_include_path(f) for f in pkgconfig_flags])
-        cppflags = os.environ.get('CPPFLAGS', '')
-        for cppflag in cppflags.split():
-            args.append(cppflag)
-        cflags = os.environ.get('CFLAGS', '')
-        for cflag in cflags.split():
-            args.append(cflag)
-        for include in self._options.cpp_includes:
-            args.append('-I' + include)
-        # The Microsoft compiler uses different option flags for
-        # compilation result output
-        if self._compiler.check_is_msvc():
-            args.extend(['-c', '-Fe' + output, '-Fo' + output])
-        else:
-            args.extend(['-c', '-o', output])
-        for source in sources:
-            if not os.path.exists(source):
-                raise CompilerError(
-                    "Could not find c source file: %s" % (source, ))
-        args.extend(list(sources))
-        if not self._options.quiet:
-            print "g-ir-scanner: compile: %s" % (
-                subprocess.list2cmdline(args), )
-            sys.stdout.flush()
-        try:
-            subprocess.check_call(args)
-        except subprocess.CalledProcessError as e:
-            raise CompilerError(e)
+        return self._compiler.compile(pkgconfig_flags,
+                                      self._options.cpp_includes,
+                                      sources,
+                                      self._options.init_sections)
 
-    def _link(self, output, *sources):
+    def _link(self, output, sources):
         args = []
         libtool = utils.get_libtool_command(self._options)
         if libtool:
+            # Note: MSVC Builds do not use libtool!
+            # In the libtool case, put together the linker command, as we did before.
+            # We aren't using distutils to link in this case.
             args.extend(libtool)
             args.append('--mode=link')
             args.append('--tag=CC')
             if self._options.quiet:
                 args.append('--silent')
 
-        args.extend(self._linker_cmd.split())
-        # We can use -o for the Microsoft compiler/linker,
-        # but it is considered deprecated usage with that
-        if self._compiler.check_is_msvc():
-            args.extend(['-Fe' + output])
-        else:
+            args.extend(self._linker_cmd)
+
             args.extend(['-o', output])
-        if libtool:
             if os.name == 'nt':
                 args.append('-Wl,--export-all-symbols')
             else:
                 args.append('-export-dynamic')
 
-        cppflags = os.environ.get('CPPFLAGS', '')
-        for cppflag in cppflags.split():
-            args.append(cppflag)
-        cflags = os.environ.get('CFLAGS', '')
-        for cflag in cflags.split():
-            args.append(cflag)
-        ldflags = os.environ.get('LDFLAGS', '')
-        for ldflag in ldflags.split():
-            args.append(ldflag)
+        if not self._compiler.check_is_msvc():
+            # These envvars are not used for MSVC Builds!
+            # MSVC Builds use the INCLUDE, LIB envvars,
+            # which are automatically picked up during
+            # compilation and linking
+            cppflags = os.environ.get('CPPFLAGS', '')
+            for cppflag in cppflags.split():
+                args.append(cppflag)
+            cflags = os.environ.get('CFLAGS', '')
+            for cflag in cflags.split():
+                args.append(cflag)
+            ldflags = os.environ.get('LDFLAGS', '')
+            for ldflag in ldflags.split():
+                args.append(ldflag)
 
         # Make sure to list the library to be introspected first since it's
         # likely to be uninstalled yet and we want the uninstalled RPATHs have
         # priority (or we might run with installed library that is older)
-
         for source in sources:
             if not os.path.exists(source):
                 raise CompilerError(
                     "Could not find object file: %s" % (source, ))
-        args.extend(list(sources))
+
+        if libtool:
+            args.extend(sources)
 
         pkg_config_libs = self._run_pkgconfig('--libs')
 
@@ -293,31 +258,61 @@ class DumpCompiler(object):
 
         else:
             args.extend(pkg_config_libs)
-            self._compiler.get_external_link_flags(args, self._options.libraries)
-
-        if not self._options.quiet:
-            print "g-ir-scanner: link: %s" % (
-                subprocess.list2cmdline(args), )
-            sys.stdout.flush()
-        msys = os.environ.get('MSYSTEM', None)
-        if msys:
-            shell = os.environ.get('SHELL', 'sh.exe')
-            # Create a temporary script file that
-            # runs the command we want
-            tf, tf_name = tempfile.mkstemp()
-            with os.fdopen(tf, 'wb') as f:
-                shellcontents = ' '.join([x.replace('\\', '/') for x in args])
-                fcontents = '#!/bin/sh\nunset PWD\n{}\n'.format(shellcontents)
-                f.write(fcontents)
-            shell = utils.which(shell)
-            args = [shell, tf_name.replace('\\', '/')]
-        try:
-            subprocess.check_call(args)
-        except subprocess.CalledProcessError as e:
-            raise LinkerError(e)
-        finally:
+            self._compiler.get_external_link_flags(args,
+                                                   libtool,
+                                                   self._options.libraries)
+
+        if not libtool:
+            # non-libtool: prepare distutils for linking the introspection
+            # dumper program...
+            try:
+                self._compiler.link(output,
+                                    sources,
+                                    args)
+
+            # Ignore failing to embed the manifest files, when the manifest
+            # file does not exist, especially for MSVC 2010 and later builds.
+            # If we are on Visual C++ 2005/2008, where
+            # this embedding is required, the build will fail anyway, as
+            # the dumper program will likely fail to run, and this means
+            # something went wrong with the build.
+            except LinkError, e:
+                if self._compiler.check_is_msvc():
+                    msg = str(e)
+
+                    if msg[msg.rfind('mt.exe'):] == 'mt.exe\' failed with exit status 31':
+                        sys.exc_clear()
+                        pass
+                    else:
+                        raise LinkError(e)
+                else:
+                    raise LinkError(e)
+        else:
+            # libtool: Run the assembled link command, we don't use distutils
+            # for linking here.
+            if not self._options.quiet:
+                print "g-ir-scanner: link: %s" % (
+                    subprocess.list2cmdline(args), )
+                sys.stdout.flush()
+            msys = os.environ.get('MSYSTEM', None)
             if msys:
-                os.remove(tf_name)
+                shell = os.environ.get('SHELL', 'sh.exe')
+                # Create a temporary script file that
+                # runs the command we want
+                tf, tf_name = tempfile.mkstemp()
+                with os.fdopen(tf, 'wb') as f:
+                    shellcontents = ' '.join([x.replace('\\', '/') for x in args])
+                    fcontents = '#!/bin/sh\nunset PWD\n{}\n'.format(shellcontents)
+                    f.write(fcontents)
+                shell = utils.which(shell)
+                args = [shell, tf_name.replace('\\', '/')]
+            try:
+                subprocess.check_call(args)
+            except subprocess.CalledProcessError as e:
+                raise LinkerError(e)
+            finally:
+                if msys:
+                    os.remove(tf_name)
 
 
 def compile_introspection_binary(options, get_type_functions,