[gimp/gimp-2-10] Issue #1809: update DLL directory for 32-bit plug-in run from 64-bit...

[Jehan <jehanp src gnome org>]

commit 91c139f4d0d4bba7278060896300ecc866d921ee
Author: Jehan <jehan girinstud io>
Date:   Wed Aug 15 13:52:51 2018 +0200

    Issue #1809: update DLL directory for 32-bit plug-in run from 64-bit...
    
    ... Windows installation of GIMP.
    
    Our default installer installs 32-bit version of the various DLLs in
    32/bin/ (under the installation prefix). Currently this additional
    folder is simply added in the PATH, so it works most of the time.
    Unfortunately the PATH is searched last for DLLs, and in particular, it
    is searched after system directories. So it means that if any misbehaved
    application is installing DLLs in system dirs (and in particular
    incompatible/older versions of the same DLLs a GIMP plug-in uses), it
    breaks the 32-bit plug-in.
    
    SetDllDirectoryW() bypasses this order and the set folder is searched in
    between the binary directory and the system dirs. We were already
    setting this for our main bin/ directory, which was good for 64-bit
    plug-ins, but this was not protecting 32-bit plug-ins. Now our code to
    run plug-ins check the bitness of the executable before running it, and
    updates the DLL folder accordingly.
    The alternative 32-bit folder can be overridden by the configure option
    --with-win32-32bit-dll-folder (default: 32/bin/). This option can only
    be set when building for 64-bit Windows obviously.
    
    Alternatively we could have put copies of 32-bit DLLs in a subfolder
    with each 32-bit plug-in, but this is at best a terrible workaround, as
    we would duplicate DLLs for every such case. And this would not have
    protected third-party plug-ins which wish to use some of our DLLs.
    Last alternative is to use AddDllDirectory(), but it works since Windows
    7 with a given update only. And our current official support is any
    Windows since Windows 7. So we don't want to use this right now (also
    I'm not sure it would actually be much better than current
    implementation, and it seems to have a bit more limitations than
    SetDllDirectoryW(), though I have not tested).

 app/plug-in/gimpplugin.c | 50 +++++++++++++++++++++++++++
 configure.ac             | 89 ++++++++++++++++++++++++++++++++----------------
 2 files changed, 109 insertions(+), 30 deletions(-)
---
diff --git a/app/plug-in/gimpplugin.c b/app/plug-in/gimpplugin.c
index 59f0c1e8c9..149cf35c35 100644
--- a/app/plug-in/gimpplugin.c
+++ b/app/plug-in/gimpplugin.c
@@ -108,6 +108,9 @@ static gboolean   gimp_plug_in_flush         (GIOChannel   *channel,
 static gboolean   gimp_plug_in_recv_message  (GIOChannel   *channel,
                                               GIOCondition  cond,
                                               gpointer      data);
+#if defined G_OS_WIN32 && defined WIN32_32BIT_DLL_FOLDER
+static void       gimp_plug_in_set_dll_directory (const gchar *path);
+#endif
 
 
 
@@ -360,6 +363,9 @@ gimp_plug_in_open (GimpPlugIn         *plug_in,
   /* Fork another process. We'll remember the process id so that we
    * can later use it to kill the filter if necessary.
    */
+#if defined G_OS_WIN32 && defined WIN32_32BIT_DLL_FOLDER
+  gimp_plug_in_set_dll_directory (argv[0]);
+#endif
   if (! gimp_spawn_async (argv, envp, spawn_flags, &plug_in->pid, &error))
     {
       gimp_message (plug_in->manager->gimp, NULL, GIMP_MESSAGE_ERROR,
@@ -398,6 +404,10 @@ gimp_plug_in_open (GimpPlugIn         *plug_in,
 
  cleanup:
 
+#if defined G_OS_WIN32 && defined WIN32_32BIT_DLL_FOLDER
+  gimp_plug_in_set_dll_directory (NULL);
+#endif
+
   if (debug)
     g_free (argv);
 
@@ -721,6 +731,46 @@ gimp_plug_in_flush (GIOChannel *channel,
   return TRUE;
 }
 
+#if defined G_OS_WIN32 && defined WIN32_32BIT_DLL_FOLDER
+static void
+gimp_plug_in_set_dll_directory (const gchar *path)
+{
+  const gchar *install_dir;
+  gchar       *bin_dir;
+  LPWSTR       w_bin_dir;
+  DWORD        BinaryType;
+  int          n;
+
+  w_bin_dir = NULL;
+  install_dir = gimp_installation_directory ();
+  if (path                               &&
+      GetBinaryTypeA (path, &BinaryType) &&
+      BinaryType == SCS_32BIT_BINARY)
+    bin_dir = g_build_filename (install_dir, WIN32_32BIT_DLL_FOLDER, NULL);
+  else
+    bin_dir = g_build_filename (install_dir, "bin", NULL);
+
+  n = MultiByteToWideChar (CP_UTF8, MB_ERR_INVALID_CHARS,
+                           bin_dir, -1, NULL, 0);
+  if (n == 0)
+    goto out;
+
+  w_bin_dir = g_malloc_n (n + 1, sizeof (wchar_t));
+  n = MultiByteToWideChar (CP_UTF8, MB_ERR_INVALID_CHARS,
+                           bin_dir, -1,
+                           w_bin_dir, (n + 1) * sizeof (wchar_t));
+  if (n == 0)
+    goto out;
+
+  SetDllDirectoryW (w_bin_dir);
+
+out:
+  if (w_bin_dir)
+    g_free ((void*) w_bin_dir);
+  g_free (bin_dir);
+}
+#endif
+
 GimpPlugInProcFrame *
 gimp_plug_in_get_proc_frame (GimpPlugIn *plug_in)
 {
diff --git a/configure.ac b/configure.ac
index b60efda4da..0623737dc3 100644
--- a/configure.ac
+++ b/configure.ac
@@ -935,6 +935,34 @@ if test "x$enable_win32_debug_console" = "xyes"; then
 fi
 AC_MSG_RESULT([$enable_win32_debug_console])
 
+######################################
+# Check for 32-bit DLLs (Win32 64-bit)
+######################################
+
+AC_ARG_WITH(win32-32bit-dll-folder,
+            [  --with-win32-32bit-dll-folder[=DIR]
+                          alternative folder with 32-bit versions of DLL libraries on Windows (default: 
32/bin/)],,
+            with_win32_32bit_dll_folder="32/bin")
+
+AC_MSG_CHECKING([the alternative folder containing 32-bit DLLs on Windows])
+if test "x$platform_win32" = "xyes"; then
+  case "$host_cpu" in
+    x86_64)
+    ;;
+    *)
+    with_win32_32bit_dll_folder="no"
+    ;;
+  esac
+else
+    with_win32_32bit_dll_folder="no"
+fi
+
+if test "x$with_win32_32bit_dll_folder" != "xno"; then
+  AC_DEFINE_UNQUOTED(WIN32_32BIT_DLL_FOLDER, "$with_win32_32bit_dll_folder",
+                     [Define if 32-bit versions of libraries are packaged in an alternative folder])
+fi
+AC_MSG_RESULT([$with_win32_32bit_dll_folder])
+
 #####################
 # Check for Dr. Mingw
 #####################
@@ -2895,44 +2923,45 @@ generate_po_makefile([po-windows-installer])
 # Print a summary of features enabled/disabled:
 optional_deps="
 Extra Binaries:
-  gimp-console:            $enable_gimp_console
+  gimp-console:              $enable_gimp_console
 
 Optional Features:
-  Language selection:      $have_iso_codes
-  Vector icons:            $enable_vector_icons
-  Dr. Mingw (Win32):       $enable_drmingw
-  Relocatable bundle:      $enable_relocatable_bundle
-  Default ICC directory:   $with_icc_directory
-  Debug console (Win32):   $enable_win32_debug_console
+  Language selection:        $have_iso_codes
+  Vector icons:              $enable_vector_icons
+  Dr. Mingw (Win32):         $enable_drmingw
+  Relocatable bundle:        $enable_relocatable_bundle
+  Default ICC directory:     $with_icc_directory
+  Debug console (Win32):     $enable_win32_debug_console
+  32-bit DLL folder (Win32): $with_win32_32bit_dll_folder
 
 Optional Plug-Ins:
-  Ascii Art:               $have_libaa
-  Ghostscript:             $have_gs
-  Help Browser:            $have_webkit
-  JPEG 2000:               $have_openjpeg
-  MNG:                     $have_libmng
-  OpenEXR:                 $have_openexr
-  WebP:                    $have_webp
-  Heif:                    $have_libheif
-  PDF (export):            $have_cairo_pdf
-  Print:                   $enable_print
-  Python 2:                $enable_python
-  TWAIN (Win32):           $os_win32
-  Webpage:                 $have_webkit
-  WMF:                     $have_libwmf
-  X11 Mouse Cursor:        $have_xmc
-  XPM:                     $have_libxpm
-  Email:                   $have_email
+  Ascii Art:                 $have_libaa
+  Ghostscript:               $have_gs
+  Help Browser:              $have_webkit
+  JPEG 2000:                 $have_openjpeg
+  MNG:                       $have_libmng
+  OpenEXR:                   $have_openexr
+  WebP:                      $have_webp
+  Heif:                      $have_libheif
+  PDF (export):              $have_cairo_pdf
+  Print:                     $enable_print
+  Python 2:                  $enable_python
+  TWAIN (Win32):             $os_win32
+  Webpage:                   $have_webkit
+  WMF:                       $have_libwmf
+  X11 Mouse Cursor:          $have_xmc
+  XPM:                       $have_libxpm
+  Email:                     $have_email
 
 Optional Modules:
-  ALSA (MIDI Input):       $have_alsa
-  Linux Input:             $have_linux_input (GUdev support: $have_libgudev)
-  DirectInput (Win32):     $have_dx_dinput
+  ALSA (MIDI Input):         $have_alsa
+  Linux Input:               $have_linux_input (GUdev support: $have_libgudev)
+  DirectInput (Win32):       $have_dx_dinput
 
 Tests:
-  Use xvfb-run             $have_xvfb_run
-  Test appdata             $have_appstream_util
-  Test desktop file        $have_desktop_file_validate
+  Use xvfb-run               $have_xvfb_run
+  Test appdata               $have_appstream_util
+  Test desktop file          $have_desktop_file_validate
 
 Bug report URL: $with_bug_report_url
 
$override_bug_report_url$have_recommended_gtk$warning_vector_icons_windows$warning_glib_networking$warning_gcc"