[gimp/wip/Jehan/optimizing-win-packaging-ci] .gitlab-ci, build: avoid same DLL dependencies from previous runs.

From: Jehan <jehanp src gnome org>
To: commits-list gnome org
Cc:
Subject: [gimp/wip/Jehan/optimizing-win-packaging-ci] .gitlab-ci, build: avoid same DLL dependencies from previous runs.
Date: Mon, 21 Feb 2022 12:37:04 +0000 (UTC)

commit 38d6783299232da6c871eb634f254dbe11cbd210
Author: Jehan <jehan girinstud io>
Date:   Mon Feb 21 01:23:37 2022 +0100

    .gitlab-ci, build: avoid same DLL dependencies from previous runs.
    
    We were already avoiding re-processing a same DLL within the same run
    (this can happen when 2 dependencies have themselves a common
    dependency). But the dll_link.py script was stateless regarding previous
    runs so we might be checking again the same DLLs multiple times (even
    though we were not copying them again).
    
    Let's make the script stateful with a new parameter to give a file where
    all the previously processed DLL names are stored. I am hoping it would
    improve the efficiency of the packaging-win32-native which is suddenly
    extra slow (it always times out, even after raising the max job time;
    now we time out after 2h30! The 64-bit packaging job just takes 1h,
    which is too much already, but still much more reasonable).

 .gitlab-ci.yml                                |  2 ++
 build/windows/gitlab-ci/dll_link.py           | 27 +++++++++++++--
 build/windows/gitlab-ci/package-gimp-msys2.sh | 50 +++++++++++++++------------
 3 files changed, 53 insertions(+), 26 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 01cbdccd37..555cd1dfb6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -370,6 +370,7 @@ packaging-win64-native:
     paths:
     - gimp-w64
     - _build-w64/build/windows/installer/lang/
+    - done-dll.list
   needs: ["gimp-win64-native"]
 
 ## WINDOWS 32-bit CI (native MSYS2) ##
@@ -455,6 +456,7 @@ packaging-win32-native:
     expire_in: 1 day
     paths:
     - gimp-w32
+    - done-dll.list
   needs: ["gimp-win32-native"]
 
 ## WINDOWS 64-bit CI (cross-build crossroad) ##
diff --git a/build/windows/gitlab-ci/dll_link.py b/build/windows/gitlab-ci/dll_link.py
index 91802b6392..3b5b4975ae 100755
--- a/build/windows/gitlab-ci/dll_link.py
+++ b/build/windows/gitlab-ci/dll_link.py
@@ -27,6 +27,9 @@ import sys
 dlls = set()
 sys_dlls = set()
 
+# Previously done DLLs in previous runs
+done_dlls = set()
+
 # Common paths
 bindir = 'bin'
 
@@ -34,7 +37,15 @@ bindir = 'bin'
 # Functions
 
 # Main function
-def main(binary, srcdirs, destdir, debug):
+def main(binary, srcdirs, destdir, debug, dll_file):
+  global done_dlls
+  try:
+    if dll_file is not None:
+      with open(dll_file, 'r') as f:
+        done_dlls = { line.strip() for line in f if len(line.strip()) != 0 }
+  except FileNotFoundError:
+    pass
+
   sys.stdout.write("{} (INFO): searching for dependencies of {} in {}.\n".format(os.path.basename(__file__),
                                                                                  binary, ', '.join(srcdirs)))
   find_dependencies(os.path.abspath(binary), srcdirs)
@@ -61,10 +72,19 @@ def main(binary, srcdirs, destdir, debug):
   else:
     copy_dlls(dlls - sys_dlls, srcdirs, destdir)
 
+  if dll_file is not None:
+    with open(dll_file, 'w') as f:
+      f.write("\n".join(set.union(done_dlls, dlls, sys_dlls)))
+
 def find_dependencies(obj, srcdirs):
   '''
   List DLLs of an object file in a recursive way.
   '''
+  if obj in set.union(done_dlls, sys_dlls):
+    # Already processed, either in a previous run of the script
+    # (done_dlls) or in this one.
+    return True
+
   if not os.path.isabs(obj):
     for srcdir in srcdirs:
       abs_dll = os.path.join(srcdir, bindir, obj)
@@ -102,7 +122,7 @@ def find_dependencies(obj, srcdirs):
     # Parse lines with DLL Name instead of lib*.dll directly
     for match in re.finditer(r"DLL Name: *(\S+.dll)", out, re.MULTILINE):
       dll = match.group(1)
-      if dll not in dlls and os.path.basename(dll) not in sys_dlls:
+      if dll not in set.union(done_dlls, dlls, sys_dlls):
         dlls.add(dll)
         find_dependencies(dll, srcdirs)
 
@@ -132,9 +152,10 @@ def copy_dlls(dll_list, srcdirs, destdir):
 if __name__ == "__main__":
   parser = argparse.ArgumentParser()
   parser.add_argument('--debug', dest='debug', action = 'store_true', default = False)
+  parser.add_argument('--output-dll-list', dest='dll_file', action = 'store', default = None)
   parser.add_argument('bin')
   parser.add_argument('src', nargs='+')
   parser.add_argument('dest')
   args = parser.parse_args(sys.argv[1:])
 
-  main(args.bin, args.src, args.dest, args.debug)
+  main(args.bin, args.src, args.dest, args.debug, args.dll_file)
diff --git a/build/windows/gitlab-ci/package-gimp-msys2.sh b/build/windows/gitlab-ci/package-gimp-msys2.sh
index 4a63aa920a..e192186c18 100644
--- a/build/windows/gitlab-ci/package-gimp-msys2.sh
+++ b/build/windows/gitlab-ci/package-gimp-msys2.sh
@@ -144,72 +144,76 @@ cp -fr ${MSYS_PREFIX}/share/icons/Adwaita ${GIMP_DISTRIB}/share/icons/
 
 # XXX Why are these for exactly?
 cp -fr ${MSYS_PREFIX}/bin/gspawn*.exe ${GIMP_DISTRIB}/bin/
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gspawn-win*-helper.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX} ${GIMP_DISTRIB}
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gspawn-win*-helper-console.exe 
${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB}
+
+# We save the list of already copied DLLs to keep a state between dll_link runs.
+rm -f done-dll.list
+
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gspawn-win*-helper.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX} ${GIMP_DISTRIB} --output-dll-list done-dll.list
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gspawn-win*-helper-console.exe 
${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB} --output-dll-list done-dll.list
 
 # XXX Does not look like it's needed anymore. Check?
 cp -fr ${MSYS_PREFIX}/bin/gdk-pixbuf-query-loaders.exe ${GIMP_DISTRIB}/bin/
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gdk-pixbuf-query-loaders.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gdk-pixbuf-query-loaders.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB} --output-dll-list done-dll.list
 
 # XXX Why is bzip2.exe needed?
 cp -fr ${MSYS_PREFIX}/bin/bzip2.exe ${GIMP_DISTRIB}/bin/
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/bzip2.exe ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ 
${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/bzip2.exe ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ 
${GIMP_DISTRIB} --output-dll-list done-dll.list
 
 # Executables for supported interpreters.
 cp -fr ${MSYS_PREFIX}/bin/pythonw.exe ${GIMP_DISTRIB}/bin/
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/pythonw.exe ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ 
${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/pythonw.exe ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ 
${GIMP_DISTRIB} --output-dll-list done-dll.list
 cp -fr ${MSYS_PREFIX}/bin/python3w.exe ${GIMP_DISTRIB}/bin/
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/python3w.exe ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ 
${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/python3w.exe ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ 
${GIMP_DISTRIB} --output-dll-list done-dll.list
 cp -fr ${MSYS_PREFIX}/bin/python3.exe ${GIMP_DISTRIB}/bin/
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/python3.exe ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ 
${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/python3.exe ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ 
${GIMP_DISTRIB} --output-dll-list done-dll.list
 
 cp -fr ${MSYS_PREFIX}/bin/luajit.exe ${GIMP_DISTRIB}/bin/
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/luajit.exe ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ 
${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/luajit.exe ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ 
${GIMP_DISTRIB} --output-dll-list done-dll.list
 
 # Executable for "gegl:introspect" from graphviz package.
 cp -fr ${MSYS_PREFIX}/bin/dot.exe ${GIMP_DISTRIB}/bin/
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/dot.exe ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ 
${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/dot.exe ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ 
${GIMP_DISTRIB} --output-dll-list done-dll.list
 
 # Generate share/glib-2.0/schemas/gschemas.compiled
 glib-compile-schemas --targetdir=${GIMP_DISTRIB}/share/glib-2.0/schemas 
${GIMP_DISTRIB}/share/glib-2.0/schemas
 
 # Package needed DLLs only
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-2.99.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-2.99.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB} --output-dll-list done-dll.list
 
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-console-2.99.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-console-2.99.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB} --output-dll-list done-dll.list
 
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-debug-resume.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-debug-resume.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB} --output-dll-list done-dll.list
 
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-debug-tool-2.99.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-debug-tool-2.99.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB} --output-dll-list done-dll.list
 
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-test-clipboard-2.99.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimp-test-clipboard-2.99.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB} --output-dll-list done-dll.list
 
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimptool-2.99.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/gimptool-2.99.exe ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB} --output-dll-list done-dll.list
 
 for dll in ${GIMP_DISTRIB}/lib/babl-0.1/*.dll; do
-  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB};
+  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB} 
--output-dll-list done-dll.list;
 done
 for dll in ${GIMP_DISTRIB}/lib/gegl-0.4/*.dll; do
-  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB};
+  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB} 
--output-dll-list done-dll.list;
 done
 for dll in ${GIMP_DISTRIB}/lib/gio/modules/*.dll; do
-  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB};
+  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB} 
--output-dll-list done-dll.list;
 done
 for dll in ${GIMP_DISTRIB}/lib/gdk-pixbuf-2.0/2.10.0/loaders/*.dll; do
-  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB};
+  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB} 
--output-dll-list done-dll.list;
 done
 for dll in ${GIMP_DISTRIB}/lib/gimp/2.99/modules/*.dll; do
-  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB};
+  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB} 
--output-dll-list done-dll.list;
 done
 for dll in ${GIMP_DISTRIB}/lib/gimp/2.99/plug-ins/*/*.exe; do
-  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB};
+  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB} 
--output-dll-list done-dll.list;
 done
 
 # Libraries for GObject Introspection.
 
 cp -fr ${MSYS_PREFIX}/bin/libgirepository-1.0-1.dll ${GIMP_DISTRIB}/bin/
-python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/libgirepository-1.0-1.dll ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB}
+python3 build/windows/gitlab-ci/dll_link.py ${GIMP_DISTRIB}/bin/libgirepository-1.0-1.dll ${GIMP_PREFIX}/ 
${MSYS_PREFIX}/ ${GIMP_DISTRIB} --output-dll-list done-dll.list
 
 for dll in ${GIMP_DISTRIB}/lib/python3.9/site-packages/*/*.dll; do
-  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB};
+  python3 build/windows/gitlab-ci/dll_link.py $dll ${GIMP_PREFIX}/ ${MSYS_PREFIX}/ ${GIMP_DISTRIB} 
--output-dll-list done-dll.list;
 done
[Date Prev][Date Next] [Thread Prev][Thread Next] [Thread Index] [Date Index] [Author Index]