[pan: 3/15] Reimplement installer

[Dominique Dumont <ddumont src gnome org>]

commit 5c912c4a5f4a812fca064de8255f9604d1b53bf0
Author: Thomas Tanner <thosrtanner googlemail com>
Date:   Sun May 1 17:52:28 2022 +0100

    Reimplement installer

 wininstall.py | 511 +++++++++++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 398 insertions(+), 113 deletions(-)
---
diff --git a/wininstall.py b/wininstall.py
index e6f3a57..79eda56 100644
--- a/wininstall.py
+++ b/wininstall.py
@@ -1,4 +1,24 @@
 #!python3
+from __future__ import annotations
+
+""" This script provides a way of creating a standalone pan for windows.
+
+Once you have built and tested pan, run this with a target folder, and all
+needed files and quite possibly some unneeded ones will be copied there, and
+that can be run standalone or used to generate an installer.
+
+AWFUL WARNINGS:
+
+This isn't going to be very stable and needs checking every new version of pan
+or every update of a package.
+
+Unfortunately may packages come with executables and libraries, and there's no
+indication of what part of the package is a utility and what part is runtime
+that other packages may use (see gettext in particular). This means I can't
+actually use the dependencies you can get from pacman as you'd end up installing
+way too much.
+
+"""
 
 import glob
 import os
@@ -14,56 +34,285 @@ Installs built pan to specified target directory
 """)
     exit(1)
 
-def copy_executable(executable: str, target_dir: str) -> set:
-    """Copy executable and work out what dlls we need.
+class Copier:
+    """A class that copies files and logs the results."""
+
+    def __init__(self, target_dir: str):
+        """Construct Copier class
+
+        Args:
+            target_dir: place to copy files
+        """
+        self._copied_files = set()
+        self._dlls = set()
+        self._installed_packages = set()
+        self._target_dir = target_dir
+
+    def copy_and_check_dlls(
+        self,
+        source: str,
+        target_dir: str = None,
+        *,
+        verbose: bool = True
+    ):
+        """Copy file and work out what dlls we need to copy later
+
+        The required dlls are stored away for later.
+
+        Args:
+            source: File to copy
+            target_dir: Where to copy it!
+            verbose: print message if set
+        """
+        if source in self._copied_files:
+            return
+
+        if target_dir is None:
+            target_dir = self._target_dir
+
+        if verbose:
+            print(f"Copying {source} to {target_dir}")
+
+        shutil.copy2(source, target_dir)
+        self._copied_files.add(source)
+
+        if source in self._dlls:
+            # We've copied a dll someone refers to. We can remove it from
+            # the list of dlls to install
+            self._dlls.remove(source)
+            return
+
+        if os.path.splitext(source)[1] not in ( ".dll", ".exe" ):
+            return
+
+        # Get all the dlls we need in the root
+        output = subprocess.run(
+            [ "ldd", source ], capture_output=True, text=True, check=True
+        )
+        for line in output.stdout.splitlines():
+            dll = re.search(r'(/mingw64.*\.dll)', line)
+            if dll:
+                dll = self.convert_name_to_windows(dll.group())
+                if dll not in self._copied_files:
+                    self._dlls.add(dll)
+
+    @staticmethod
+    def convert_name_to_windows(file: str) -> str:
+        """Converts /mingw name to windows name.
+
+        Args:
+            file: filename starting with /mingw..
+
+        Returns:
+            filename starting with c:/....
+        """
+        return file.replace(
+            f'/{os.environ["MSYSTEM"].lower()}',
+            os.environ["MSYSTEM_PREFIX"],
+            1
+        )
 
-    Args:
-        executable: Executable file to copy
-        target_dir: Where to copy it!
+    def copy_package(
+        self,
+        name: str,
+        *,
+        library: str = None,
+        include: list(str) = None,
+        exclude: list(str) = None,
+        files: list(str) = None,
+        verbose: bool = False,
+        recopy: bool = False
+    ):
+        """Copies a package to target directory.
+
+        This gets a list of files installed with the package and copies them to
+        the target directory
+
+        Some directories are included/excluded by default. These may be
+        overridden by specifiying include and exclude
+
+        It's done like this because all of these packages seem to come with
+        loads of extraneous stuff, and it's not awfully clear what is or isn't
+        needed for a standalone executable
+
+        NB Those parameters should be any iterable or sequence?
+
+        Args:
+            name: Package name
+            library: Actual name of library if different to package name
+            include: directories to include
+            exclude: directories to exclude
+            files: files to include
+            verbose: print filenames when copied
+            recopy: allow the library to be installed a 2nd time.
+        """
+        if library is None:
+            library = name
+
+        package = f"{os.environ['MINGW_PACKAGE_PREFIX']}-{name}"
+        print(f"Installing {package} in {self._target_dir}")
+        output = subprocess.run(
+            [ "pacman", "-Q", "-l", package ],
+            capture_output=True,
+            text=True,
+            check=True
+        )
 
-    Returns:
-        Set of dlls required by executable
-    """
+        # Default directories to copy
+        # Possibly you could include anything in "lib/{package}*"?
+        dirs = set((
+            "etc",
+            f"lib/{library}",
+            f"share/{library}",
+            f"share/{name}",
+            "share/icons",
+            "share/glib-2.0/schemas",
+            "share/licenses",
+            "share/locale",
+            "share/themes",
+            "share/xml"
+        ))
+        # Extras
+        if include is not None:
+            if isinstance(include, str):
+                dirs.add(include)
+            else:
+                dirs |= set(include)
+
+        # Turn into filenames
+        dirs = set(
+            os.path.join(os.environ["MSYSTEM_PREFIX"], dir) + "/"
+                for dir in dirs
+        )
 
-    print(f"Copying {executable} to {target_dir}")
-    shutil.copy2(executable, target_dir)
+        # List of files to copy
+        if files is None:
+            files = ()
+        elif isinstance(files, str):
+            files = (files, )
+        # Turn into filenames
+        files = set(
+            os.path.join(os.environ["MSYSTEM_PREFIX"], file)
+                for file in files
+        )
 
-    dlls=set()
+        # List of directories to exclude
+        if exclude is None:
+            exclude=()
+        elif isinstance(exclude, str):
+            exclude = (exclude, )
+        # Turn into filenames
+        exclude = set(
+            os.path.join(os.environ["MSYSTEM_PREFIX"], dir) + "/"
+                for dir in exclude
+        )
 
-    if os.path.splitext(executable)[1] not in (".dll", ".exe"):
-        return dlls
-    # Get all the dlls we need in the root
-    output = subprocess.run(
-        ["ldd", executable], capture_output=True, text=True, check=True
-    )
-    for line in output.stdout.splitlines():
-        dll = re.search(r'(/mingw64.*\.dll)', line)
-        if dll:
-            dlls.add(dll.group())
+        for line in output.stdout.splitlines():
+            file = self.convert_name_to_windows(line.split()[-1])
 
-    return dlls
+            if os.path.isdir(file) or file.endswith(".a"):
+                # Ignore directories and .a files for distribution
+                continue
 
+            if file in self._dlls:
+                # This is a needed dll. This goes direct to target directory
+                self.copy_and_check_dlls(
+                    file, self._target_dir, verbose=verbose
+                )
+                continue
 
-dlls = set()
+            # FIXME This could be simplified
+            copy = False
+
+            if file in files:
+                copy = True
+            else:
+                if not any((file.startswith(dir) for dir in exclude)):
+                    copy = any((file.startswith(dir) for dir in dirs))
+
+            if copy:
+                outdir = os.path.join(
+                    self._target_dir,
+                    os.path.dirname(
+                        file[len(os.environ["MSYSTEM_PREFIX"]) + 1:]
+                    )
+                )
+                os.makedirs(outdir, exist_ok=True)
+                self.copy_and_check_dlls(file, outdir, verbose=verbose)
+            else:
+                if verbose:
+                    print("ignored " + file)
+
+        if not recopy:
+            self._installed_packages.add(name)
+
+
+    def _get_packages_containing(self, paths: list(str)) -> set(str):
+        """ Get all the packages which lay claim to specified paths.
+
+        Arguments:
+            paths: Any iterable or a string
+
+        Returns:
+            The packages laying claim to those files or directories.
+
+            Packages already installed are filtered out.
+        """
+        if isinstance(paths, str):
+            paths = (paths, )
+
+        output = subprocess.run(
+            [ "pacman", "-Q", "-o" ] + list(paths),
+            capture_output=True,
+            text=True,
+            check=True
+        )
 
-def copy_wrapper(source: str, target: str, *, follow_symlinks: bool = True):
-    got_dlls = copy_executable(source, os.path.dirname(target))
-    global dlls
-    dlls |= got_dlls
-    return target
+        packages = set()
+        for line in output.stdout.splitlines():
+            # Each line is of form
+            # <dir> is owned by <package> <version>
+            package = line.split()[-2].replace(
+                os.environ["MINGW_PACKAGE_PREFIX"] + "-", ""
+            )
+            packages.add(package)
+
+        return packages
+
+    def get_packages_containing(self, paths: list(str)) -> set(str):
+        """Get all the packages required for the dlls we need to load.
+
+        Returns:
+            Packages needed
+        """
+        packages = self._get_packages_containing(paths)
+        return set(
+            [
+                package for package in packages
+                    if not package in self._installed_packages
+            ]
+        )
+
+    def get_needed_packages(self) -> set(str):
+        """Get all the packages required for the dlls we need to load.
+
+        Returns:
+            Packages needed
 
-def copy_tree(*, target_dir: str, tree: str):
-    """Wrapper to copy an entire tree from a magic place to target folder
+        A Note: This isn't recursive, so on installing packages, it's possible
+        that it may be necessary to install other packages.
+        """
+        if len(self._dlls) == 0:
+            return set()
 
-    Args:
-        target_dir - Directory containing standalone executable
-        tree - folder in /mingw to copy to executable directory
-    """
-    shutil.copytree(os.path.join(os.environ["MSYSTEM_PREFIX"], tree),
-                    os.path.join(target_dir, tree),
-                    copy_function=copy_wrapper,
-                    ignore=shutil.ignore_patterns("*.a"),
-                    dirs_exist_ok=True)
+        packages = self._get_packages_containing(self._dlls)
+        for package in packages:
+            # Umm. Maybe we shouldn't care and reinstall at least the dll
+            # from the package.
+            if package in self._installed_packages:
+                print(f"Need {package} from one of {self._dlls} which is already installed")
+                exit(1)
+        return packages
 
 def read_configure():
     dbus = False
@@ -140,73 +389,15 @@ def main():
 # NEWS
 
     # Copy executable to target dir
-    executable = "pan/gui/pan.exe"
-    global dlls
-    dlls = copy_executable(executable, target_dir)
-
-    copy_tree(target_dir=target_dir, tree="lib/gdk-pixbuf-2.0")
-    # + share/locale/en_GB/LC_MESSAGES/gdk-pixbuf.mo?
-
-    # Deal with magically autoloaded stuff
-    #------------------------------------------------------------------------
+    copier = Copier(target_dir)
 
-    # We need this to run shells
-    dlls |= copy_executable(
-        os.path.join(os.environ["MSYSTEM_PREFIX"], "bin/gspawn-win64-helper.exe"),
-        target_dir
-    )
+    copier.copy_and_check_dlls("pan/gui/pan.exe")
 
-    # ------------ gtk2
-    if config["gtk_version"] == 2:
-        copy_tree(target_dir=target_dir, tree="lib/gtk-2.0/2.10.0/engines")
-        # + share/locale/en_GB/LC_MESSAGES/gtk20*?
-
-    # None of these appear to be necessary so why do we have them?
-    # However, the release pan version has very slightly nicer fonts, so need
-    # further check (though it might be this version or something)
-    # add etc/fonts?
-    # add etc/gtk-2.0
-    # add etc/pango
-
-    # ------------ gtk3
-    #if config["gtk_version"] == 3:
-
-    #These aren't necessary if you want a windows theme
-    #really? they don't seem gtk specific either
-    #
-    # Copy Icon themes
-    # `cp -r /mingw64/share/icons/* ./share/icons/
-    # Copy settins schemas
-    # `cp /mingw64/share/glib-2.0/schemas/* ./share/glib-2.0/schemas/`
-    # `glib-compile-schemas.exe ./share/glib-2.0/schemas/`
-
-    # See https://www.gtk.org/docs/installations/windows/ also
-
-
-    # ------------ all gtk versions
-    copy_tree(target_dir=target_dir, tree="share/themes")
-
-    if config["spellcheck"]:
-        copy_tree(target_dir=target_dir, tree="lib/enchant-2")
-        copy_tree(target_dir=target_dir, tree="share/enchant")
-        copy_tree(target_dir=target_dir, tree="share/hunspell")
-        # also share/iso-codes ?
-        # maybe I should use pacman directly for some of this?
-        copy_tree(target_dir=target_dir, tree="share/xml/iso-codes")
-        # Also possibly share/iso-codes and the whole of share/locale
-        # Maybe we should extract the pacman file directly?
-        # tar -xvf /var/cache/pacman/pkg/mingw-w64-x86_64-iso-codes-4.9.0-3-any.pkg.tar.zst
-        #   -C <target_dir> --strip-components 1
-
-    #------------------------------------------------------------------------
-
-    # Possibly copy the whole of /mingw64share/locale?
-    # May be worth while installing the packages installed to build this with the tar
-    # command above?
-
-    # Now we copy all the pan <lang>.gmo files in the po directory to the right place in
-    # <target>/locale/<dir>/LC_MESSAGES/pan.mo. This may or may not be correct for windows,
-    # as the existing install appears to set up registry keys.
+    # Now we copy all the pan <lang>.gmo files in the po directory to the right
+    # place in <target>/locale/<lang>/LC_MESSAGES/pan.mo.
+    # This may or may not be correct for windows, as the existing install
+    # appears to set up registry keys.
+    print("Copying pan locale files")
     locale_dir = os.path.join(target_dir, 'share', 'locale')
     for gmo in glob.glob("po/*.gmo"):
         name = os.path.basename(gmo)
@@ -215,17 +406,111 @@ def main():
         os.makedirs(dest_dir, exist_ok=True)
         shutil.copy2(gmo, os.path.join(dest_dir, 'pan.mo'))
 
-
-    # Now we copy all the dlls we depend on. Unfortunately, they all have
-    # unix like names, so we need to replace all of them
-    for dll in sorted(dlls):
-        dll = dll.replace(
-            f'/{os.environ["MSYSTEM"].lower()}',
-            os.environ["MSYSTEM_PREFIX"],
-            1
+    # We need this to run shells
+    copier.copy_and_check_dlls(
+        os.path.join(
+            os.environ["MSYSTEM_PREFIX"], "bin/gspawn-win64-helper.exe"
         )
-        copy_executable(dll, target_dir)
+    )
 
+    # Arguably we could look at the dlls we now have and load up all the
+    # packages they come from and the dependent packages. However, the actual
+    # packages for most of these contain a bunch of executables and libraries,
+    # which might drag in other packages and we don't really want to install
+    # those.
+    pixbuf_lib = "gdk-pixbuf-2.0"
+
+    while len(packages := copier.get_needed_packages()) != 0:
+
+        for package in sorted(packages):
+            if package == "aspell":
+                copier.copy_package(package, library="aspell-0.60")
+            elif package == "enchant":
+                copier.copy_package(package, library="enchant-2")
+                copier.copy_package(
+                    "hunspell-en", library="hunspell", include="share/doc"
+                )
+                # Should we include myspell in the above? seems to work OK
+                # without it
+                copier.copy_package("iso-codes")
+            elif package == "gdk-pixbuf2":
+                copier.copy_package(package, library=pixbuf_lib)
+                # Copy any other packages that install in the pixbuf library
+                for package in copier.get_packages_containing(
+                    os.path.join(
+                        os.environ["MSYSTEM_PREFIX"], "lib", pixbuf_lib
+                    )
+                ):
+                    copier.copy_package(
+                        package, library=pixbuf_lib, recopy=True
+                    )
+            elif package == "gettext":
+                # FIXME don't copy the gettext-tools locale files
+                # This would be a whole lot easier if gettext tools and gettext
+                # runtime were separate packages.
+                copier.copy_package(
+                    package,
+                    library="None",
+                    exclude=(f"share/{package}", "share/licenses"),
+                    files=f"share/licenses/{package}/gettext-runtime/intl/COPYING.LIB"
+                )
+            elif package == "gtk2":
+                copier.copy_package(package, library="gtk-2.0/2.10.0")
+            elif package == "gtk3":
+                copier.copy_package(package, library="gtk-3.0")
+                copier.copy_package("adwaita-icon-theme")
+            elif package == "graphite2":
+                copier.copy_package(package, exclude=f"share/{package}")
+            elif package == "icu":
+                copier.copy_package(
+                    package,
+                    library="None",
+                    exclude=f"share/{package}",
+                    files=f"share/{package}/69.1/LICENSE"
+                )
+            elif package == "libgpg-error":
+                copier.copy_package(package, exclude=f"share/{package}")
+            elif package == "libjpeg-turbo":
+                # This includes source code in some pretty odd places
+                copier.copy_package(
+                    package,
+                    exclude=f"share/licenses/{package}/simd"
+                )
+            else:
+                copier.copy_package(package)
+
+    # ---- gdk-pixbuf2 cleanup
+    # Knowing where the loader cache file exists is not ideal, but it needs to
+    # exist and I'm not entirely convinced about copying the generated one.
+    loader_path = os.path.join(target_dir, 'lib', pixbuf_lib, '2.10.0')
+    output = subprocess.run(
+        [ "gdk-pixbuf-query-loaders.exe" ],
+        env=dict(
+            os.environ,
+            GDK_PIXBUF_MODULEDIR=os.path.join(loader_path, "loaders")
+        ),
+        capture_output=True,
+        text=True,
+        check=True
+    )
+
+    # replace all the paths with windows paths relative to target
+    with open(os.path.join(loader_path, 'loaders.cache'), 'w') as cache:
+        for line in output.stdout.splitlines():
+            if target_dir in line:
+                line = line.replace(target_dir + "/", "")
+                line = line.replace("/", r"\\")
+            print(line, file=cache)
+
+    # ---- glib2 cleanup
+    # setting schemas - must be run after all packages installed.
+    output = subprocess.run(
+        [
+            "glib-compile-schemas.exe",
+            os.path.join(target_dir, 'share', 'glib-2.0', 'schemas')
+        ],
+        check=True
+    )
 
 if __name__ == "__main__":
     main()