[glib/wip/3v1n0/support-can-fail-tests: 1/4] meson: Support tests that can fail under certain conditions

[Marco Trevisan <marcotrevi src gnome org>]

commit ea7b47b7c5f810e23c815820f647d832fd426a4c
Author: Marco Trevisan (Treviño) <mail 3v1n0 net>
Date:   Wed Oct 19 20:08:15 2022 +0200

    meson: Support tests that can fail under certain conditions
    
    We have tests that are failing under certain environments, but it's
    difficult to handle them because:
     - for some environments we just allow all the tests to fail: DANGEROUS
     - when we don't allow failures we have flacky tests: A CI pain
    
    So, to avoid this add a way to wrap tests with a script that just marks
    the test as skipped in case of failures.
    
    Note that timeout for wrapped tests won't take advantage of the native
    meson tests multiplier, but we can accept that given that we'd likely to
    fail anyways.

 gio/tests/meson.build     | 36 +++++++++++++++++++++++++++++-------
 glib/tests/meson.build    | 36 ++++++++++++++++++++++++++++++------
 gmodule/tests/meson.build | 18 +++++++++++++++++-
 gobject/tests/meson.build | 45 ++++++++++++++++++++++++++++++++++++---------
 meson.build               |  2 ++
 tests/run-flaky-test.py   | 21 +++++++++++++++++++++
 6 files changed, 135 insertions(+), 23 deletions(-)
---
diff --git a/gio/tests/meson.build b/gio/tests/meson.build
index 57fc7d3cc6..7e998785ea 100644
--- a/gio/tests/meson.build
+++ b/gio/tests/meson.build
@@ -154,9 +154,9 @@ test_extra_programs = {
   'gsubprocess-testprog' : {},
 }
 
-python_tests = [
-  'codegen.py',
-]
+python_tests = {
+  'codegen.py' : {},
+}
 
 test_env = environment(common_test_env)
 test_env.set('G_TEST_SRCDIR', meson.current_source_dir())
@@ -905,21 +905,31 @@ foreach test_name, extra_args : gio_tests
     install_tag: 'tests',
     install: install,
   )
+  args = extra_args.get('args', [])
+  depends = extra_args.get('depends', [])
 
   suite = ['gio'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
   local_test_env = test_env
 
+  if extra_args.get('can_fail', false)
+    depends += [exe]
+    args = [ timeout.to_string(), exe.full_path() ] + args
+    exe = flaky_test_runner
+    timeout = -1
+  endif
+
   foreach var, value : extra_args.get('env', {})
     local_test_env.append(var, value)
   endforeach
 
   test(test_name, exe,
+    args: args,
     env : local_test_env,
     timeout : timeout,
     suite : suite,
     is_parallel : extra_args.get('is_parallel', true),
-    depends : extra_args.get('depends', []),
+    depends : depends,
     should_fail : extra_args.get('should_fail', false),
   )
 endforeach
@@ -937,12 +947,24 @@ foreach program_name, extra_args : test_extra_programs
   )
 endforeach
 
-foreach test_name : python_tests
+foreach test_name, extra_args : python_tests
+  exe = python
+  args = ['-B', files(test_name)]
+  timeout = test_timeout
+
+  if extra_args.get('can_fail', false)
+    depends += [exe]
+    args = [ timeout.to_string(), exe.full_path() ] + args
+    exe = flaky_test_runner
+    timeout = -1
+  endif
+
   test(
     test_name,
-    python,
-    args: ['-B', files(test_name)],
+    exe,
+    args: args,
     env: test_env,
+    timeout: timeout,
     suite: ['gio', 'no-valgrind'],
   )
 
diff --git a/glib/tests/meson.build b/glib/tests/meson.build
index 8b5c58b8c5..80ece56bfc 100644
--- a/glib/tests/meson.build
+++ b/glib/tests/meson.build
@@ -293,13 +293,25 @@ foreach test_name, extra_args : glib_tests
     install_tag: 'tests',
     install: install,
   )
+  args = extra_args.get('args', [])
+  depends = extra_args.get('depends', [])
 
   suite = ['glib'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
+
+  if extra_args.get('can_fail', false)
+    depends += [exe]
+    args = [ timeout.to_string(), exe.full_path() ] + args
+    exe = flaky_test_runner
+    timeout = -1
+  endif
+
   test(test_name, exe,
+    args: args,
     env : test_env,
     timeout : timeout,
     suite : suite,
+    depends : depends,
     should_fail : extra_args.get('should_fail', false),
   )
 endforeach
@@ -312,9 +324,9 @@ if installed_tests_enabled
   )
 endif
 
-python_tests = [
-  'assert-msg-test.py',
-]
+python_tests = {
+  'assert-msg-test.py' : {},
+}
 
 executable('assert-msg-test', ['assert-msg-test.c'],
   c_args : test_cargs,
@@ -325,12 +337,24 @@ executable('assert-msg-test', ['assert-msg-test.c'],
   win_subsystem : extra_args.get('win_subsystem', 'console'),
 )
 
-foreach test_name : python_tests
+foreach test_name, extra_args : python_tests
+  exe = python
+  args = ['-B', files(test_name)]
+  timeout = test_timeout
+
+  if extra_args.get('can_fail', false)
+    depends += [exe]
+    args = [ timeout.to_string(), exe.full_path() ] + args
+    exe = flaky_test_runner
+    timeout = -1
+  endif
+
   test(
     test_name,
-    python,
-    args: ['-B', files(test_name)],
+    exe,
+    args: args,
     env: test_env,
+    timeout: timeout,
     suite: ['glib', 'no-valgrind'],
   )
 
diff --git a/gmodule/tests/meson.build b/gmodule/tests/meson.build
index a751f3185b..b8ce8b762a 100644
--- a/gmodule/tests/meson.build
+++ b/gmodule/tests/meson.build
@@ -92,8 +92,24 @@ foreach test_name, extra_args : gmodule_tests
     install_tag: 'tests',
     install: install,
   )
+  args = extra_args.get('args', [])
+  depends = extra_args.get('depends', [])
 
   suite = ['gmodule'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
-  test(test_name, exe, env : test_env, timeout : timeout, suite : suite)
+
+  if extra_args.get('can_fail', false)
+    depends += [exe]
+    args = [ timeout.to_string(), exe.full_path() ] + args
+    exe = flaky_test_runner
+    timeout = -1
+  endif
+
+  test(test_name, exe,
+    args: args,
+    env : test_env,
+    timeout : timeout,
+    suite : suite,
+    depends : depends,
+  )
 endforeach
diff --git a/gobject/tests/meson.build b/gobject/tests/meson.build
index 09f23e8bfb..5c573f9e5c 100644
--- a/gobject/tests/meson.build
+++ b/gobject/tests/meson.build
@@ -122,11 +122,11 @@ if cc.get_id() != 'msvc'
   gobject_tests += {'autoptr' : {}}
 endif
 
-python_tests = [
-  'genmarshal.py',
-  'gobject-query.py',
-  'mkenums.py',
-]
+python_tests = {
+  'genmarshal.py' : {},
+  'gobject-query.py' : {},
+  'mkenums.py' : {},
+}
 
 test_env = environment(common_test_env)
 test_env.set('G_TEST_SRCDIR', meson.current_source_dir())
@@ -162,6 +162,8 @@ foreach test_name, extra_args : gobject_tests
     install_tag: 'tests',
     install: install,
   )
+  args = extra_args.get('args', [])
+  depends = extra_args.get('depends', [])
 
   suite = ['gobject'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
@@ -172,15 +174,40 @@ foreach test_name, extra_args : gobject_tests
     timeout = timeout * 10
   endif
 
-  test(test_name, exe, env : test_env, timeout : timeout, suite : suite)
+  if extra_args.get('can_fail', false)
+    depends += [exe]
+    args = [ timeout.to_string(), exe.full_path() ] + args
+    exe = flaky_test_runner
+    timeout = -1
+  endif
+
+  test(test_name, exe,
+    args: args,
+    env : test_env,
+    timeout : timeout,
+    suite : suite,
+    depends : depends,
+  )
 endforeach
 
-foreach test_name : python_tests
+foreach test_name, extra_args : python_tests
+  exe = python
+  args = ['-B', files(test_name)]
+  timeout = test_timeout
+
+  if extra_args.get('can_fail', false)
+    depends += [exe]
+    args = [ timeout.to_string(), exe.full_path() ] + args
+    exe = flaky_test_runner
+    timeout = -1
+  endif
+
   test(
     test_name,
-    python,
-    args: ['-B', files(test_name)],
+    exe,
+    args: args,
     env: test_env,
+    timeout: timeout,
     suite: ['gobject', 'no-valgrind'],
   )
 
diff --git a/meson.build b/meson.build
index ddcdc028d4..9396e291d8 100644
--- a/meson.build
+++ b/meson.build
@@ -2337,6 +2337,8 @@ common_test_env = [
 ]
 test_timeout = 60
 test_timeout_slow = 180
+flaky_test_runner = find_program('tests' / 'run-flaky-test.py',
+  required : get_option('tests'))
 
 pkg = import('pkgconfig')
 windows = import('windows')
diff --git a/tests/run-flaky-test.py b/tests/run-flaky-test.py
new file mode 100755
index 0000000000..4db60398bd
--- /dev/null
+++ b/tests/run-flaky-test.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import subprocess
+import sys
+
+if len(sys.argv) < 3:
+  print("Missing arguments <timeout> <program> [ args... ]")
+  sys.exit(1)
+
+timeout = int(sys.argv[1])
+cmd = sys.argv[2:]
+
+try:
+    p = subprocess.run(cmd, timeout=timeout)
+    p.check_returncode()
+except subprocess.TimeoutExpired:
+    print(f'Timeout of {timeout} expired for {" ".join(cmd)}')
+    sys.exit(77)
+except subprocess.CalledProcessError:
+    sys.exit(77)
+