[gobject-introspection/wip/fanc999/dumper-distutils: 6/8] sourcescanner.py: Use Distutils for Preprocessing

[Chun-wei Fan <fanchunwei src gnome org>]

commit f0762055da8d2007a6b4881c5925b049bb17a101
Author: Chun-wei Fan <fanchunwei src gnome org>
Date:   Tue Mar 10 13:15:52 2015 +0800

    sourcescanner.py: Use Distutils for Preprocessing
    
    Add a preprocess() function in ccompiler.py so that it will call the
    preprocess() method of the distutils.ccompiler class, and make use of it
    from sourcescanner.py.
    
    As we would need to set up the options (include paths, macros, undefs) to
    pass into the preprocessor (and later for the compiler), we have a new
    private function that translates what we have from the rest of giscanner so
    that it could be passed to distutils.ccompiler in a way that it
    understands.
    
    Also, as the MSVCCompiler classes in distutils do not provide a
    preprocess() implementation, we provide our own so that we can use it when
    preprocessing, via distutils, through subclassing MSVCCompiler.
    
    https://bugzilla.gnome.org/show_bug.cgi?id=728313

 Makefile-giscanner.am      |    1 +
 giscanner/ccompiler.py     |   64 +++++++++++++++++++++++++++-
 giscanner/msvccompiler.py  |  101 ++++++++++++++++++++++++++++++++++++++++++++
 giscanner/sourcescanner.py |   46 +++-----------------
 4 files changed, 171 insertions(+), 41 deletions(-)
---
diff --git a/Makefile-giscanner.am b/Makefile-giscanner.am
index c2273cd..5bc8425 100644
--- a/Makefile-giscanner.am
+++ b/Makefile-giscanner.am
@@ -43,6 +43,7 @@ pkgpyexec_PYTHON =                    \
        giscanner/libtoolimporter.py    \
        giscanner/maintransformer.py    \
        giscanner/message.py            \
+       giscanner/msvccompiler.py       \
        giscanner/shlibs.py             \
        giscanner/scannermain.py        \
        giscanner/sectionparser.py      \
diff --git a/giscanner/ccompiler.py b/giscanner/ccompiler.py
index 884fc2c..f9105fe 100644
--- a/giscanner/ccompiler.py
+++ b/giscanner/ccompiler.py
@@ -61,7 +61,15 @@ class CCompiler(object):
             compiler_name = distutils.ccompiler.get_default_compiler()
 
         # Now, create the distutils ccompiler instance based on the info we have.
-        self.compiler = distutils.ccompiler.new_compiler(compiler=compiler_name)
+        if compiler_name == 'msvc':
+            # For MSVC, we need to create a instance of a subclass of distutil's
+            # MSVC9Compiler class, as it does not provide a preprocess()
+            # implementation
+            from . import msvccompiler
+            self.compiler = msvccompiler.get_msvc_compiler()
+
+        else:
+            self.compiler = distutils.ccompiler.new_compiler(compiler=compiler_name)
         customize_compiler(self.compiler)
 
         # customize_compiler from distutils only does customization
@@ -147,6 +155,31 @@ class CCompiler(object):
                 else:
                     args.append('-l' + library)
 
+    def preprocess(self, source, output, cpp_options):
+        extra_postargs = ['-C']
+        (include_paths, macros, postargs) = self._set_cpp_options(cpp_options)
+
+        # We always want to include the current path
+        include_dirs = ['.']
+
+        include_dirs.extend(include_paths)
+        extra_postargs.extend(postargs)
+
+        # Define these macros when using Visual C++ to silence many warnings,
+        # and prevent stepping on many Visual Studio-specific items, so that
+        # we don't have to handle them specifically in scannerlexer.l
+        if self.check_is_msvc():
+            macros.append(('_USE_DECLSPECS_FOR_SAL', None))
+            macros.append(('_CRT_SECURE_NO_WARNINGS', None))
+            macros.append(('_CRT_NONSTDC_NO_WARNINGS', None))
+            macros.append(('SAL_NO_ATTRIBUTE_DECLARATIONS', None))
+
+        self.compiler.preprocess(source=source,
+                                 output_file=output,
+                                 macros=macros,
+                                 include_dirs=include_dirs,
+                                 extra_postargs=extra_postargs)
+
     def resolve_windows_libs(self, libraries, options):
         args = []
         libsearch = []
@@ -238,3 +271,32 @@ class CCompiler(object):
             return True
         else:
             return False
+
+    # Private APIs
+    def _set_cpp_options(self, options):
+        includes = []
+        macros = []
+        other_options = []
+
+        for o in options:
+            option = utils.cflag_real_include_path(o)
+            if option.startswith('-I'):
+                includes.append(option[len('-I'):])
+            elif option.startswith('-D'):
+                macro = option[len('-D'):]
+                macro_index = macro.find('=')
+                if macro_index == -1:
+                    macro_name = macro
+                    macro_value = None
+                else:
+                    macro_name = macro[:macro_index]
+                    macro_value = macro[macro_index + 1:]
+                macros.append((macro_name, macro_value))
+            elif option.startswith('-U'):
+                macros.append((option[len('-U'):],))
+            else:
+                # We expect the preprocessor to remove macros. If debugging is turned
+                # up high enough that won't happen, so don't add those flags. Bug #720504
+                if option not in ['-g3', '-ggdb3', '-gstabs3', '-gcoff3', '-gxcoff3', '-gvms3']:
+                    other_options.append(option)
+        return (includes, macros, other_options)
diff --git a/giscanner/msvccompiler.py b/giscanner/msvccompiler.py
new file mode 100644
index 0000000..6324809
--- /dev/null
+++ b/giscanner/msvccompiler.py
@@ -0,0 +1,101 @@
+# -*- Mode: Python -*-
+# GObject-Introspection - a framework for introspecting GObject libraries
+# Copyright (C) 2014  Chun-wei Fan
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+#
+
+import os
+import distutils
+
+from distutils.errors import (DistutilsExecError, CompileError, LibError,
+                              LinkError, UnknownFileError)
+from distutils.ccompiler import CCompiler, gen_preprocess_options
+from distutils.dep_util import newer
+
+# Distutil's MSVCCompiler does not provide a preprocess()
+# Implementation, so do our own here.
+
+
+def get_msvc_compiler():
+    return MSVCCompiler()
+
+
+class MSVCCompiler(distutils.msvccompiler.MSVCCompiler):
+
+    def __init__(self, verbose=0, dry_run=0, force=0):
+        CCompiler.__init__(self, verbose, dry_run, force)
+        self.__paths = []
+        self.__arch = None  # deprecated name
+        if os.name == 'nt':
+            if isinstance(self, distutils.msvc9compiler.MSVCCompiler):
+                self.__version = distutils.msvc9compiler.VERSION
+        self.initialized = False
+        self.preprocess_options = None
+
+    def preprocess(self,
+                   source,
+                   output_file=None,
+                   macros=None,
+                   include_dirs=None,
+                   extra_preargs=None,
+                   extra_postargs=None):
+        if self.initialized is False:
+            self.initialize()
+
+        (_, macros, include_dirs) = \
+            self._fix_compile_args(None, macros, include_dirs)
+        pp_opts = gen_preprocess_options(macros, include_dirs)
+        preprocess_options = ['-E']
+        source_basename = None
+
+        if output_file is not None:
+            preprocess_options.append('-P')
+            source_basename = self._get_file_basename(source)
+        cpp_args = self.cc.split()
+        if extra_preargs is not None:
+            cpp_args[:0] = extra_preargs
+        if extra_postargs is not None:
+            preprocess_options.extend(extra_postargs)
+        cpp_args.extend(preprocess_options)
+        cpp_args.extend(pp_opts)
+        cpp_args.append(source)
+
+        # We need to preprocess: either we're being forced to, or the
+        # source file is newer than the target (or the target doesn't
+        # exist).
+        if self.force or output_file is None or newer(source, output_file):
+            try:
+                self.spawn(cpp_args)
+            except DistutilsExecError, msg:
+                print msg
+                raise CompileError
+
+        # The /P option for the MSVC preprocessor will output the results
+        # of the preprocessor to a file, as <source_without_extension>.i,
+        # so in order to output the specified filename, we need to rename
+        # that file
+        if output_file is not None:
+            if output_file != source_basename + '.i':
+                os.rename(source_basename + '.i', output_file)
+
+    def _get_file_basename(self, filename):
+        if filename is None:
+            return None
+        if filename.rfind('.') == -1:
+            return filename[filename.rfind('\\') + 1:]
+        else:
+            return filename[filename.rfind('\\') + 1:filename.rfind('.')]
diff --git a/giscanner/sourcescanner.py b/giscanner/sourcescanner.py
index 9310cff..e66ed7d 100644
--- a/giscanner/sourcescanner.py
+++ b/giscanner/sourcescanner.py
@@ -25,6 +25,7 @@ import tempfile
 
 from .libtoolimporter import LibtoolImporter
 from .message import Position
+from .ccompiler import CCompiler
 
 with LibtoolImporter(None, None):
     if 'UNINSTALLED_INTROSPECTION_SRCDIR' in os.environ:
@@ -282,6 +283,8 @@ class SourceScanner(object):
         defines = ['__GI_SCANNER__']
         undefs = []
 
+        cc = CCompiler()
+
         tmp_fd_cpp, tmp_name_cpp = tempfile.mkstemp(prefix='g-ir-cpp-', suffix='.c')
         fp_cpp = os.fdopen(tmp_fd_cpp, 'w')
         self._write_preprocess_src(fp_cpp, defines, undefs, filenames)
@@ -293,46 +296,9 @@ class SourceScanner(object):
         # so we want the name to match the output file name of the MSVC preprocessor
         tmpfile_output = tmpfile_basename + '.i'
 
-        cpp_args = os.environ.get('CC', 'cc').split()  # support CC="ccache gcc"
-
-        cpp_args += ['-E', '-C', '-I.']
-
-        # MSVC: The '-P' option makes the preprocessor output to a file, but we
-        # can't specify directly the name of the output file, so we use the
-        # MSVC-style preprocessor output file name for all builds for simplicity
-        # Define the following macros to silence many, many warnings
-        # in using the MSVC preprocessor during the parsing stage...
-        if 'cl.exe' in cpp_args or 'cl' in cpp_args:
-            cpp_args += ('-P',
-                         '-D_USE_DECLSPECS_FOR_SAL',
-                         '-D_CRT_SECURE_NO_WARNINGS',
-                         '-D_CRT_NONSTDC_NO_WARNINGS',
-                         '-DSAL_NO_ATTRIBUTE_DECLARATIONS')
-
-        cpp_args += self._cpp_options
-        cpp_args += [tmp_name_cpp]
-
-        # We expect the preprocessor to remove macros. If debugging is turned
-        # up high enough that won't happen, so strip these out. Bug #720504
-        for flag in ['-g3', '-ggdb3', '-gstabs3', '-gcoff3', '-gxcoff3', '-gvms3']:
-            try:
-                cpp_args.remove(flag)
-            except ValueError:
-                pass
-
-        proc = subprocess.Popen(cpp_args,
-                                stdin=subprocess.PIPE,
-                                stdout=subprocess.PIPE)
-
-        if 'cl.exe' not in cpp_args and'cl' not in cpp_args:
-            cpp_args += ['-o', tmpfile_output]
-
-        proc = subprocess.Popen(cpp_args)
-
-        assert proc, 'Proc was none'
-        proc.wait()
-        if proc.returncode != 0:
-            raise SystemExit('Error while processing the source.')
+        cc.preprocess(tmp_name_cpp,
+                      tmpfile_output,
+                      self._cpp_options)
 
         os.unlink(tmp_name_cpp)
         fp = open(tmpfile_output, 'r')