[jhbuild] [win32] monkeypatch subprocess.Popen on Microsoft Windows (GNOME bug 583455)

[Frederic Peters <fpeters src gnome org>]

commit 16bb6ae3761592042868f11bd4ff359667e8df71
Author: Sam Thursfield <ssssam gmail com>
Date:   Thu Jun 4 11:21:26 2009 +0200

    [win32] monkeypatch subprocess.Popen on Microsoft Windows (GNOME bug 583455)
---
 jhbuild/config.py                 |   38 ++++++++++-
 jhbuild/monkeypatch.py            |    5 ++
 jhbuild/utils/subprocess_win32.py |  136 +++++++++++++++++++++++++++++++++++++
 tests/tests.py                    |   19 +++++-
 4 files changed, 193 insertions(+), 5 deletions(-)

diff --git a/jhbuild/config.py b/jhbuild/config.py
index c286244..a3ce1de 100644
--- a/jhbuild/config.py
+++ b/jhbuild/config.py
@@ -28,6 +28,10 @@ import __builtin__
 from jhbuild.errors import UsageError, FatalError, CommandError
 from jhbuild.utils.cmds import get_output
 
+if sys.platform.startswith('win'):
+    # For munging paths for MSYS's benefit
+    import jhbuild.utils.subprocess_win32
+
 __all__ = [ 'Config' ]
 
 _defaults_file = os.path.join(os.path.dirname(__file__), 'defaults.jhbuildrc')
@@ -62,6 +66,9 @@ def addpath(envvar, path):
     '''Adds a path to an environment variable.'''
     # special case ACLOCAL_FLAGS
     if envvar in [ 'ACLOCAL_FLAGS' ]:
+        if sys.platform.startswith('win'):
+            path = jhbuild.utils.subprocess_win32.fix_path_for_msys(path)
+
         envval = os.environ.get(envvar, '-I %s' % path)
         parts = ['-I', path] + envval.split()
         i = 2
@@ -78,14 +85,36 @@ def addpath(envvar, path):
                 i += 1
         envval = ' '.join(parts)
     elif envvar in [ 'LDFLAGS', 'CFLAGS', 'CXXFLAGS' ]:
+        if sys.platform.startswith('win'):
+            path = jhbuild.utils.subprocess_win32.fix_path_for_msys(path)
+
         envval = os.environ.get(envvar)
         if envval:
             envval = path + ' ' + envval
         else:
             envval = path
     else:
+        if envvar == 'PATH':
+            # PATH is special cased on Windows to allow execution without
+            # sh.exe. The other env vars (like LD_LIBRARY_PATH) don't mean
+            # anything to native Windows so they stay in UNIX format, but
+            # PATH is kept in Windows format (; seperated, c:/ or c:\ format
+            # paths) so native Popen works.
+            pathsep = os.pathsep
+        else:
+            pathsep = ':'
+            if sys.platform.startswith('win'):
+                path = jhbuild.utils.subprocess_win32.fix_path_for_msys(path)
+
+            if sys.platform.startswith('win') and path[1]==':':
+                # Windows: Don't allow c:/ style paths in :-seperated env vars
+                # for obvious reasons. /c/ style paths are valid - if a var is
+                # seperated by : it will only be of interest to programs inside
+                # MSYS anyway.
+                path='/'+path[0]+path[2:]
+
         envval = os.environ.get(envvar, path)
-        parts = envval.split(':')
+        parts = envval.split(pathsep)
         parts.insert(0, path)
         # remove duplicate entries:
         i = 1
@@ -96,7 +125,7 @@ def addpath(envvar, path):
                 del parts[i]
             else:
                 i += 1
-        envval = ':'.join(parts)
+        envval = pathsep.join(parts)
 
     os.environ[envvar] = envval
 
@@ -275,7 +304,12 @@ class Config:
         # scripts to find modules that do not use pkg-config (such as guile
         # looking for gmp, or wireless-tools for NetworkManager)
         # (see bug #377724 and bug #545018)
+
+        # This path doesn't always get passed to addpath so we fix it here
+        if sys.platform.startswith('win'):
+            libdir = jhbuild.utils.subprocess_win32.fix_path_for_msys(libdir)
         os.environ['LDFLAGS'] = ('-L%s ' % libdir) + os.environ.get('LDFLAGS', '')
+
         includedir = os.path.join(self.prefix, 'include')
         addpath('C_INCLUDE_PATH', includedir)
         addpath('CPLUS_INCLUDE_PATH', includedir)
diff --git a/jhbuild/monkeypatch.py b/jhbuild/monkeypatch.py
index b0bded1..ddd1928 100644
--- a/jhbuild/monkeypatch.py
+++ b/jhbuild/monkeypatch.py
@@ -21,6 +21,11 @@ from __future__ import generators
 
 import sys
 
+# Windows lacks all sorts of subprocess features that we need to kludge around
+if sys.platform.startswith('win'):
+    from jhbuild.utils import subprocess_win32
+    sys.modules['subprocess'] = subprocess_win32
+
 # Python < 2.2.1 lacks  True and False constants
 import __builtin__
 if not hasattr(__builtin__, 'True'):
diff --git a/jhbuild/utils/subprocess_win32.py b/jhbuild/utils/subprocess_win32.py
new file mode 100644
index 0000000..a9bccd1
--- /dev/null
+++ b/jhbuild/utils/subprocess_win32.py
@@ -0,0 +1,136 @@
+# jhbuild - a build script for GNOME 1.x and 2.x
+# Copyright (C) 2001-2006  James Henstridge
+#
+#   subprocess_win32: monkeypatch to make jhbuild work on win32
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import os
+import sys
+import subprocess as real_subprocess
+
+PIPE = real_subprocess.PIPE
+STDOUT = real_subprocess.STDOUT
+
+def fix_path_for_msys(oldpath):
+    return oldpath.replace('\\','/')
+
+
+def cmdline2list(cmd_string):
+    """
+    Translate a command line string into a sequence of arguments, with
+    the same rules as subprocess.list2cmdline:
+
+    1) Arguments are delimited by white space, which is either a
+       space or a tab.
+
+    2) A string surrounded by double quotation marks is
+       interpreted as a single argument, regardless of white space
+       or pipe characters contained within.  A quoted string can be
+       embedded in an argument.
+
+    3) A double quotation mark preceded by a backslash is
+       interpreted as a literal double quotation mark.
+
+    4) Backslashes are interpreted literally, unless they
+       immediately precede a double quotation mark.
+
+    5) If backslashes immediately precede a double quotation mark,
+       every pair of backslashes is interpreted as a literal
+       backslash.  If the number of backslashes is odd, the last
+       backslash escapes the next double quotation mark as
+       described in rule 3.
+    """
+
+    cmd_string = cmd_string.strip()
+
+    result = []
+    current_element = ""
+    escape = False
+    in_quotes = False
+    for character in cmd_string:
+        if escape:
+            if character != '"':
+                current_element += '\\'
+            current_element += character
+            escape = False
+        else:
+            if character=='\\':
+                escape = True
+            elif character=='"':
+                if in_quotes: in_quotes = False
+                else:         in_quotes = True
+            elif (character==' ' or character==9) and not in_quotes:
+                result.append(current_element)
+                current_element = ""
+            else:
+                current_element += character
+    if escape:
+        current_element += '\\'
+    result.append(current_element)
+    return result
+
+list2cmdline = real_subprocess.list2cmdline
+
+class Popen(real_subprocess.Popen):
+    __emulate_close_fds = False
+
+    def __init__(self, command, **kws):
+        # command could be string or list - our kludges require list.
+        # subprocess converts the list back to a string using list2cmdline.
+        if not isinstance(command, list):
+            command = cmdline2list(command)
+
+        # ./ confuses windows, and these are normally shell scripts so use
+        # sh.exe
+        if command[0].startswith('./'):
+            command = ['sh', command[0].replace('./','')] + command[1:]
+        elif not command[0].endswith('.exe'):
+            # check if program has no extension or has .sh extension - it
+            # probably needs executing by sh rather than by Windows directly
+            for path in os.environ['PATH'].split(os.pathsep):
+                prog = os.path.abspath(os.path.join(path, command[0]))
+                if os.path.exists(prog) or os.path.exists(prog+".sh"):
+                    command = ['sh', '-c', ' '.join([command[0]] + command[1:])]
+                    break
+
+        # fix all backslashes to forward slashes - MSYS is smart about doing
+        # this but we're not always running things via sh.exe.
+        for i in range(0,len(command)):
+            command[i] = fix_path_for_msys(command[i])
+
+        # 'shell' flag will execute 'command' using cmd.exe, which is a waste
+        # of time.  We shall use sh.exe to execute programs which look like
+        # shell scripts.
+        if 'shell' in kws and kws['shell']:
+            kws['shell'] = False
+
+        # default Windows implementation of close_fds is useless, we have to
+        # emulate it
+        if 'close_fds' in kws and kws['close_fds']:
+            kws['close_fds'] = False
+            self.__emulate_close_fds = True
+
+        real_subprocess.Popen.__init__(self, command, **kws)
+
+    def __del__(self):
+        if self.__emulate_close_fds:
+            for f in self.stdin, self.stdout, self.stderr:
+                # This somehow tells us if we are dealing with a pipe.
+                if isinstance(f, file) and f.name == '<fdopen>':
+                    # Not that it actually matters.
+                    f.close()
+            sys.stdout.flush()
+        real_subprocess.Popen.__del__(self)
diff --git a/tests/tests.py b/tests/tests.py
index 0a8b009..52e23ed 100644
--- a/tests/tests.py
+++ b/tests/tests.py
@@ -64,6 +64,15 @@ __builtin__.__dict__['uencode'] = uencode
 
 import mock
 
+if sys.platform.startswith('win'):
+    import jhbuild.utils.subprocess_win32 as subprocess_win32
+    class WindowsTestCase(unittest.TestCase):
+        '''Tests for Windows kludges.'''
+        def testCmdline2List(self):
+            cmdline = 'test "no quotes" != \\"no\\ quotes\\"'
+            cmd_list = subprocess_win32.cmdline2list (cmdline)
+            self.assertEqual (cmd_list, ['test', 'no quotes', '!=', '"no\\ quotes"'])
+
 class ModuleOrderingTestCase(unittest.TestCase):
     '''Module Ordering'''
 
@@ -615,6 +624,7 @@ class EndToEndTest(unittest.TestCase):
                         branch_dir)
         return SimpleBranch(src_name, branch_dir)
 
+    # FIXME: broken under Win32
     def test_distutils(self):
         config = self.make_config()
         module_list = [DistutilsModule('hello',
@@ -625,7 +635,7 @@ class EndToEndTest(unittest.TestCase):
         with_stdout_hidden(build.build)
         proc = subprocess.Popen(['hello'], stdout=subprocess.PIPE)
         stdout, stderr = proc.communicate()
-        self.assertEquals(stdout, 'Hello world (distutils)\n')
+        self.assertEquals(stdout.strip(), 'Hello world (distutils)')
         self.assertEquals(proc.wait(), 0)
 
     def test_autotools(self):
@@ -638,9 +648,12 @@ class EndToEndTest(unittest.TestCase):
         with_stdout_hidden(build.build)
         proc = subprocess.Popen(['hello'], stdout=subprocess.PIPE)
         stdout, stderr = proc.communicate()
-        self.assertEquals(stdout, 'Hello world (autotools)\n')
+        self.assertEquals(stdout.strip(), 'Hello world (autotools)')
         self.assertEquals(proc.wait(), 0)
 
+    # Won't pass under stock MSYS because pkgconfig isn't installed in base
+    # path. Will work if you set ACLOCAL_FLAGS, PATH and PKG_CONFIG_PATH to
+    # a prefix where pkg-config is installed.
     def test_autotools_with_libtool(self):
         config = self.make_config()
         module_list = [
@@ -653,7 +666,7 @@ class EndToEndTest(unittest.TestCase):
         with_stdout_hidden(build.build)
         proc = subprocess.Popen(['hello'], stdout=subprocess.PIPE)
         stdout, stderr = proc.communicate()
-        self.assertEquals(stdout, 'Hello world (library test)\n')
+        self.assertEquals(stdout.strip(), 'Hello world (library test)')
         self.assertEquals(proc.wait(), 0)