[glib/wip/3v1n0/support-can-fail-tests: 1/3] meson: Support tests that can fail under certain conditions

[Marco Trevisan <marcotrevi src gnome org>]

commit 9c5a9c6a87dba8bc79558aef74363b3a9c3f3889
Author: Marco Trevisan (Treviño) <mail 3v1n0 net>
Date:   Wed Oct 19 20:08:15 2022 +0200

    meson: Support tests that can fail under certain conditions
    
    We have tests that are failing under certain environments, but it's
    difficult to handle them because:
     - for some environments we just allow all the tests to fail: DANGEROUS
     - when we don't allow failures we have flacky tests: A CI pain
    
    So, to avoid this add a way to wrap tests with a script that just marks
    the test as skipped in case of failures.

 gio/tests/meson.build     | 70 ++++++++++++++++++++++++++++++++++++-----
 glib/tests/meson.build    | 70 +++++++++++++++++++++++++++++++++++++----
 gmodule/tests/meson.build | 35 ++++++++++++++++++++-
 gobject/tests/meson.build | 79 +++++++++++++++++++++++++++++++++++++++++------
 meson.build               |  4 ++-
 5 files changed, 234 insertions(+), 24 deletions(-)
---
diff --git a/gio/tests/meson.build b/gio/tests/meson.build
index 57fc7d3cc6..ba1eb757d7 100644
--- a/gio/tests/meson.build
+++ b/gio/tests/meson.build
@@ -154,9 +154,9 @@ test_extra_programs = {
   'gsubprocess-testprog' : {},
 }
 
-python_tests = [
-  'codegen.py',
-]
+python_tests = {
+  'codegen.py' : {},
+}
 
 test_env = environment(common_test_env)
 test_env.set('G_TEST_SRCDIR', meson.current_source_dir())
@@ -905,21 +905,48 @@ foreach test_name, extra_args : gio_tests
     install_tag: 'tests',
     install: install,
   )
+  args = extra_args.get('args', [])
+  depends = extra_args.get('depends', [])
 
   suite = ['gio'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
   local_test_env = test_env
 
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@s "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
   foreach var, value : extra_args.get('env', {})
     local_test_env.append(var, value)
   endforeach
 
   test(test_name, exe,
+    args: args,
     env : local_test_env,
     timeout : timeout,
     suite : suite,
     is_parallel : extra_args.get('is_parallel', true),
-    depends : extra_args.get('depends', []),
+    depends : depends,
     should_fail : extra_args.get('should_fail', false),
   )
 endforeach
@@ -937,12 +964,41 @@ foreach program_name, extra_args : test_extra_programs
   )
 endforeach
 
-foreach test_name : python_tests
+foreach test_name, extra_args : python_tests
+  exe = python
+  args = ['-B', files(test_name)]
+  timeout = test_timeout
+
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@s "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
   test(
     test_name,
-    python,
-    args: ['-B', files(test_name)],
+    exe,
+    args: args,
     env: test_env,
+    timeout: timeout,
     suite: ['gio', 'no-valgrind'],
   )
 
diff --git a/glib/tests/meson.build b/glib/tests/meson.build
index 8b5c58b8c5..ad6dd2f847 100644
--- a/glib/tests/meson.build
+++ b/glib/tests/meson.build
@@ -293,13 +293,42 @@ foreach test_name, extra_args : glib_tests
     install_tag: 'tests',
     install: install,
   )
+  args = extra_args.get('args', [])
+  depends = extra_args.get('depends', [])
 
   suite = ['glib'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
+
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@s "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
   test(test_name, exe,
+    args: args,
     env : test_env,
     timeout : timeout,
     suite : suite,
+    depends : depends,
     should_fail : extra_args.get('should_fail', false),
   )
 endforeach
@@ -312,9 +341,9 @@ if installed_tests_enabled
   )
 endif
 
-python_tests = [
-  'assert-msg-test.py',
-]
+python_tests = {
+  'assert-msg-test.py' : {},
+}
 
 executable('assert-msg-test', ['assert-msg-test.c'],
   c_args : test_cargs,
@@ -325,12 +354,41 @@ executable('assert-msg-test', ['assert-msg-test.c'],
   win_subsystem : extra_args.get('win_subsystem', 'console'),
 )
 
-foreach test_name : python_tests
+foreach test_name, extra_args : python_tests
+  exe = python
+  args = ['-B', files(test_name)]
+  timeout = test_timeout
+
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@s "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
   test(
     test_name,
-    python,
-    args: ['-B', files(test_name)],
+    exe,
+    args: args,
     env: test_env,
+    timeout: timeout,
     suite: ['glib', 'no-valgrind'],
   )
 
diff --git a/gmodule/tests/meson.build b/gmodule/tests/meson.build
index a751f3185b..ac15b2bca7 100644
--- a/gmodule/tests/meson.build
+++ b/gmodule/tests/meson.build
@@ -92,8 +92,41 @@ foreach test_name, extra_args : gmodule_tests
     install_tag: 'tests',
     install: install,
   )
+  args = extra_args.get('args', [])
+  depends = extra_args.get('depends', [])
 
   suite = ['gmodule'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
-  test(test_name, exe, env : test_env, timeout : timeout, suite : suite)
+
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@s "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
+  test(test_name, exe,
+    args: args,
+    env : test_env,
+    timeout : timeout,
+    suite : suite,
+    depends : depends,
+  )
 endforeach
diff --git a/gobject/tests/meson.build b/gobject/tests/meson.build
index 09f23e8bfb..35c8f13f16 100644
--- a/gobject/tests/meson.build
+++ b/gobject/tests/meson.build
@@ -122,11 +122,11 @@ if cc.get_id() != 'msvc'
   gobject_tests += {'autoptr' : {}}
 endif
 
-python_tests = [
-  'genmarshal.py',
-  'gobject-query.py',
-  'mkenums.py',
-]
+python_tests = {
+  'genmarshal.py' : {},
+  'gobject-query.py' : {},
+  'mkenums.py' : {},
+}
 
 test_env = environment(common_test_env)
 test_env.set('G_TEST_SRCDIR', meson.current_source_dir())
@@ -162,6 +162,8 @@ foreach test_name, extra_args : gobject_tests
     install_tag: 'tests',
     install: install,
   )
+  args = extra_args.get('args', [])
+  depends = extra_args.get('depends', [])
 
   suite = ['gobject'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
@@ -172,15 +174,74 @@ foreach test_name, extra_args : gobject_tests
     timeout = timeout * 10
   endif
 
-  test(test_name, exe, env : test_env, timeout : timeout, suite : suite)
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@s "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
+  test(test_name, exe,
+    args: args,
+    env : test_env,
+    timeout : timeout,
+    suite : suite,
+    depends : depends,
+  )
 endforeach
 
-foreach test_name : python_tests
+foreach test_name, extra_args : python_tests
+  exe = python
+  args = ['-B', files(test_name)]
+  timeout = test_timeout
+
+  if extra_args.get('can_fail', false)
+    if have_sh
+      depends += [exe]
+      args = [
+        '-c',
+        'timeout -k @0@s "@1@" "$@" || exit 77'.format(timeout, exe.full_path()),
+        '--',
+        args
+      ]
+      exe = sh_program
+      timeout = -1
+    elif cmd_exe_program.found()
+      depends += [exe]
+      args = [
+        '/c',
+        '"@0@" "@1@" || exit 77'.format(exe.full_path(), ' '.join(args)),
+      ]
+      exe = cmd_exe_program
+    else
+      warning('Test @0@ cannot be ran reliably, skipping it'.format(test_name))
+      continue
+    endif
+  endif
+
   test(
     test_name,
-    python,
-    args: ['-B', files(test_name)],
+    exe,
+    args: args,
     env: test_env,
+    timeout: timeout,
     suite: ['gobject', 'no-valgrind'],
   )
 
diff --git a/meson.build b/meson.build
index ddcdc028d4..9e597f254a 100644
--- a/meson.build
+++ b/meson.build
@@ -2243,7 +2243,9 @@ endif
 # Determine which user environment-dependent files that we want to install
 have_bash = find_program('bash', required : false).found() # For completion scripts
 bash_comp_dep = dependency('bash-completion', version: '>=2.0', required: false)
-have_sh = find_program('sh', required : false).found() # For glib-gettextize
+sh_program = find_program('sh', required : false) # For glib-gettextize
+cmd_exe_program = find_program('cmd.exe', required : false)
+have_sh = sh_program.found()
 
 # Some installed tests require a custom environment
 env_program = find_program('env', required: installed_tests_enabled)