[glib/wip/3v1n0/support-can-fail-tests: 1/9] meson, ci: Support tests that can fail under certain conditions

From: Marco Trevisan <marcotrevi src gnome org>
To: commits-list gnome org
Cc:
Subject: [glib/wip/3v1n0/support-can-fail-tests: 1/9] meson, ci: Support tests that can fail under certain conditions
Date: Thu, 20 Oct 2022 00:36:26 +0000 (UTC)

commit 198b91169b802d641c6dbad09ac276dc3d3f8d14
Author: Marco Trevisan (Treviño) <mail 3v1n0 net>
Date:   Wed Oct 19 20:08:15 2022 +0200

    meson, ci: Support tests that can fail under certain conditions
    
    We have tests that are failing in some environments, but it's
    difficult to handle them because:
     - for some environments we just allow all the tests to fail: DANGEROUS
     - when we don't allow failures we have flacky tests: A CI pain
    
    So, to avoid this and ensure that:
     - New failing tests are tracked in all platforms
     - gitlab integration on tests reports is working
     - coverage is reported also for failing tests
    
    Add support for `can_fail` keyword on tests that would mark the test as
    part of the flaky suite.
    Not adding the suite directly when defining the tests as this is
    definitely clearer and allows to define conditions more clearly (see next
    commits).
    
    Now, add a default test setup that does not run the flaky tests by
    default (not to bother distributors with testing well-known issues) and
    eventually run all the tests in CI:
     - Non-flaky tests cannot fail in all platforms
     - Flaky tests can fail
    
    In both cases we save the test reports so that gitlab integration is
    preserved.

 .gitlab-ci.yml                        | 40 ++++++++++++++++++++++++++---------
 .gitlab-ci/run-tests.sh               |  7 +++++-
 .gitlab-ci/test-msvc.bat              |  3 ++-
 .gitlab-ci/test-msys2.sh              |  4 +++-
 gio/tests/meson.build                 | 22 +++++++++++++------
 glib/tests/meson.build                | 21 +++++++++++++-----
 gmodule/tests/meson.build             |  4 ++++
 gobject/tests/meson.build             | 24 +++++++++++++++------
 gobject/tests/performance/meson.build |  4 ++++
 meson.build                           | 14 +++++++++++-
 10 files changed, 111 insertions(+), 32 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 3846535e4b..d9d1244551 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -88,7 +88,9 @@ fedora-x86_64:
     - lcov --config-file .lcovrc --directory _build --capture --output-file "_coverage/${CI_JOB_NAME}.lcov"
   artifacts:
     reports:
-      junit: "_build/meson-logs/testlog.junit.xml"
+      junit:
+        - _build/meson-logs/testlog.junit.xml
+        - _build/meson-logs/testlog-*.junit.xml
     name: "glib-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}"
     when: always
     expire_in: 1 week
@@ -130,7 +132,9 @@ debian-stable-x86_64:
     - .gitlab-ci/run-tests.sh
   artifacts:
     reports:
-      junit: "_build/meson-logs/testlog.junit.xml"
+      junit:
+        - _build/meson-logs/testlog.junit.xml
+        - _build/meson-logs/testlog-*.junit.xml
     name: "glib-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}"
     when: always
     expire_in: 1 week
@@ -196,7 +200,9 @@ G_DISABLE_ASSERT:
     - bash -x ./.gitlab-ci/run-tests.sh
   artifacts:
     reports:
-      junit: "_build/meson-logs/testlog.junit.xml"
+      junit:
+        - _build/meson-logs/testlog.junit.xml
+        - _build/meson-logs/testlog-*.junit.xml
     name: "glib-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}"
     when: always
     expire_in: 1 week
@@ -271,7 +277,9 @@ cross-mingw64:
     - ninja -C _build
   artifacts:
     reports:
-      junit: "_build/meson-logs/testlog.junit.xml"
+      junit:
+        - _build/meson-logs/testlog.junit.xml
+        - _build/meson-logs/testlog-*.junit.xml
     name: "glib-${env:CI_JOB_NAME}-${env:CI_COMMIT_REF_NAME}"
     when: always
     expire_in: 1 week
@@ -298,7 +306,9 @@ msys2-mingw32:
     - C:\msys64\usr\bin\bash -lc "bash -x ./.gitlab-ci/test-msys2.sh"
   artifacts:
     reports:
-      junit: "_build/meson-logs/testlog.junit.xml"
+      junit:
+        - _build/meson-logs/testlog.junit.xml
+        - _build/meson-logs/testlog-*.junit.xml
     name: "glib-${env:CI_JOB_NAME}-${env:CI_COMMIT_REF_NAME}"
     when: always
     expire_in: 1 week
@@ -321,7 +331,9 @@ vs2017-x64:
                                --python.purelibdir=C:\Python37\site-packages
   artifacts:
     reports:
-      junit: "_build/meson-logs/testlog.junit.xml"
+      junit:
+        - _build/meson-logs/testlog.junit.xml
+        - _build/meson-logs/testlog-*.junit.xml
     name: "glib-${env:CI_JOB_NAME}-${env:CI_COMMIT_REF_NAME}"
     when: always
     expire_in: 1 week
@@ -349,7 +361,9 @@ vs2017-x64-static:
                                --python.purelibdir=C:\Python37\site-packages
   artifacts:
     reports:
-      junit: "_build/meson-logs/testlog.junit.xml"
+      junit:
+        - _build/meson-logs/testlog.junit.xml
+        - _build/meson-logs/testlog-*.junit.xml
     name: "glib-${env:CI_JOB_NAME}-${env:CI_COMMIT_REF_NAME}"
     when: always
     expire_in: 1 week
@@ -391,7 +405,9 @@ freebsd-12-x86_64:
     - bash -x ./.gitlab-ci/run-tests.sh
   artifacts:
     reports:
-      junit: "_build/meson-logs/testlog.junit.xml"
+      junit:
+        - _build/meson-logs/testlog.junit.xml
+        - _build/meson-logs/testlog-*.junit.xml
     name: "glib-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}"
     when: always
     expire_in: 1 week
@@ -418,7 +434,9 @@ freebsd-13-x86_64:
     - bash -x ./.gitlab-ci/run-tests.sh
   artifacts:
     reports:
-      junit: "_build/meson-logs/testlog.junit.xml"
+      junit:
+        - _build/meson-logs/testlog.junit.xml
+        - _build/meson-logs/testlog-*.junit.xml
     name: "glib-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}"
     when: always
     expire_in: 1 week
@@ -458,7 +476,9 @@ macos:
     - .gitlab-ci/run-tests.sh
   artifacts:
     reports:
-      junit: "_build/meson-logs/testlog.junit.xml"
+      junit:
+        - _build/meson-logs/testlog.junit.xml
+        - _build/meson-logs/testlog-*.junit.xml
     name: "glib-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}"
     when: always
     expire_in: 1 week
diff --git a/.gitlab-ci/run-tests.sh b/.gitlab-ci/run-tests.sh
index 634f861eb8..69153c1f9a 100755
--- a/.gitlab-ci/run-tests.sh
+++ b/.gitlab-ci/run-tests.sh
@@ -5,5 +5,10 @@
 meson test \
         -C _build \
         --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
-        --no-suite flaky \
         "$@"
+
+# Run only the flaky tests, so we can log the failures but without hard failing
+meson test \
+        -C _build \
+        --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
+        "$@" --setup=flaky_tests --suite=flaky || true
diff --git a/.gitlab-ci/test-msvc.bat b/.gitlab-ci/test-msvc.bat
index 82e445e00a..6dcfebfcca 100644
--- a/.gitlab-ci/test-msvc.bat
+++ b/.gitlab-ci/test-msvc.bat
@@ -18,7 +18,8 @@ python .gitlab-ci/check-missing-install-tag.py _build || goto :error
 ninja -C _build || goto :error
 
 :: FIXME: dont ignore test errors
-meson test -C _build --timeout-multiplier %MESON_TEST_TIMEOUT_MULTIPLIER% --no-suite flaky
+meson test -C _build --timeout-multiplier %MESON_TEST_TIMEOUT_MULTIPLIER%
+meson test -C _build --timeout-multiplier %MESON_TEST_TIMEOUT_MULTIPLIER% --setup=flaky_tests --suite=flaky
 
 :: FIXME: can we get code coverage support?
 
diff --git a/.gitlab-ci/test-msys2.sh b/.gitlab-ci/test-msys2.sh
index a345fda6a7..987f6b91eb 100755
--- a/.gitlab-ci/test-msys2.sh
+++ b/.gitlab-ci/test-msys2.sh
@@ -51,7 +51,9 @@ lcov \
     --output-file "${DIR}/_coverage/${CI_JOB_NAME}-baseline.lcov"
 
 # FIXME: fix the test suite
-meson test --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" --no-suite flaky || true
+meson test --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" || true
+meson test --timeout-multiplier "${MESON_TEST_TIMEOUT_MULTIPLIER}" \
+    --setup=flaky_tests --suite flaky || true
 
 lcov \
     --quiet \
diff --git a/gio/tests/meson.build b/gio/tests/meson.build
index 57fc7d3cc6..206360ac23 100644
--- a/gio/tests/meson.build
+++ b/gio/tests/meson.build
@@ -115,7 +115,7 @@ gio_tests = {
   'vfs' : {},
   'volumemonitor' : {},
   'glistmodel' : {},
-  'testfilemonitor' : {'suite' : ['slow', 'flaky']},
+  'testfilemonitor' : {'suite' : ['slow'], can_fail: true},
   'thumbnail-verification' : {},
   'tls-certificate' : {'extra_sources' : ['gtesttlsbackend.c']},
   'tls-interaction' : {'extra_sources' : ['gtesttlsbackend.c']},
@@ -154,9 +154,9 @@ test_extra_programs = {
   'gsubprocess-testprog' : {},
 }
 
-python_tests = [
-  'codegen.py',
-]
+python_tests = {
+  'codegen.py' : {},
+}
 
 test_env = environment(common_test_env)
 test_env.set('G_TEST_SRCDIR', meson.current_source_dir())
@@ -914,6 +914,10 @@ foreach test_name, extra_args : gio_tests
     local_test_env.append(var, value)
   endforeach
 
+  if extra_args.get('can_fail', false)
+    suite += 'flaky'
+  endif
+
   test(test_name, exe,
     env : local_test_env,
     timeout : timeout,
@@ -937,13 +941,19 @@ foreach program_name, extra_args : test_extra_programs
   )
 endforeach
 
-foreach test_name : python_tests
+foreach test_name, extra_args : python_tests
+  suite = ['gio', 'no-valgrind']
+
+  if extra_args.get('can_fail', false)
+    suite += 'flaky'
+  endif
+
   test(
     test_name,
     python,
     args: ['-B', files(test_name)],
     env: test_env,
-    suite: ['gio', 'no-valgrind'],
+    suite: suite,
   )
 
   if installed_tests_enabled
diff --git a/glib/tests/meson.build b/glib/tests/meson.build
index 8b5c58b8c5..10462ba69e 100644
--- a/glib/tests/meson.build
+++ b/glib/tests/meson.build
@@ -296,6 +296,11 @@ foreach test_name, extra_args : glib_tests
 
   suite = ['glib'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
+
+  if extra_args.get('can_fail', false)
+    suite += 'flaky'
+  endif
+
   test(test_name, exe,
     env : test_env,
     timeout : timeout,
@@ -312,9 +317,9 @@ if installed_tests_enabled
   )
 endif
 
-python_tests = [
-  'assert-msg-test.py',
-]
+python_tests = {
+  'assert-msg-test.py' : {},
+}
 
 executable('assert-msg-test', ['assert-msg-test.c'],
   c_args : test_cargs,
@@ -325,13 +330,19 @@ executable('assert-msg-test', ['assert-msg-test.c'],
   win_subsystem : extra_args.get('win_subsystem', 'console'),
 )
 
-foreach test_name : python_tests
+foreach test_name, extra_args : python_tests
+  suite = ['glib', 'no-valgrind']
+
+  if extra_args.get('can_fail', false)
+    suite += 'flaky'
+  endif
+
   test(
     test_name,
     python,
     args: ['-B', files(test_name)],
     env: test_env,
-    suite: ['glib', 'no-valgrind'],
+    suite: suite,
   )
 
   if installed_tests_enabled
diff --git a/gmodule/tests/meson.build b/gmodule/tests/meson.build
index a751f3185b..55a45ad603 100644
--- a/gmodule/tests/meson.build
+++ b/gmodule/tests/meson.build
@@ -93,6 +93,10 @@ foreach test_name, extra_args : gmodule_tests
     install: install,
   )
 
+  if extra_args.get('can_fail', false)
+    suite += 'flaky'
+  endif
+
   suite = ['gmodule'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
   test(test_name, exe, env : test_env, timeout : timeout, suite : suite)
diff --git a/gobject/tests/meson.build b/gobject/tests/meson.build
index 09f23e8bfb..fba5c2e214 100644
--- a/gobject/tests/meson.build
+++ b/gobject/tests/meson.build
@@ -122,11 +122,11 @@ if cc.get_id() != 'msvc'
   gobject_tests += {'autoptr' : {}}
 endif
 
-python_tests = [
-  'genmarshal.py',
-  'gobject-query.py',
-  'mkenums.py',
-]
+python_tests = {
+  'genmarshal.py' : {},
+  'gobject-query.py' : {},
+  'mkenums.py' : {},
+}
 
 test_env = environment(common_test_env)
 test_env.set('G_TEST_SRCDIR', meson.current_source_dir())
@@ -166,6 +166,10 @@ foreach test_name, extra_args : gobject_tests
   suite = ['gobject'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
 
+  if extra_args.get('can_fail', false)
+    suite += 'flaky'
+  endif
+
   # FIXME: https://gitlab.gnome.org/GNOME/glib/issues/1316
   # aka https://bugs.debian.org/880883
   if test_name == 'closure-refcount' and ['arm', 'aarch64'].contains(host_machine.cpu_family())
@@ -175,13 +179,19 @@ foreach test_name, extra_args : gobject_tests
   test(test_name, exe, env : test_env, timeout : timeout, suite : suite)
 endforeach
 
-foreach test_name : python_tests
+foreach test_name, extra_args : python_tests
+  suite = ['gobject', 'no-valgrind']
+
+  if extra_args.get('can_fail', false)
+    suite += 'flaky'
+  endif
+
   test(
     test_name,
     python,
     args: ['-B', files(test_name)],
     env: test_env,
-    suite: ['gobject', 'no-valgrind'],
+    suite: suite,
   )
 
   if installed_tests_enabled
diff --git a/gobject/tests/performance/meson.build b/gobject/tests/performance/meson.build
index 01b6af534f..1e98a2728c 100644
--- a/gobject/tests/performance/meson.build
+++ b/gobject/tests/performance/meson.build
@@ -40,6 +40,10 @@ foreach test_name, extra_args : gobject_tests
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
   args = extra_args.get('args', [])
 
+  if extra_args.get('can_fail', false)
+    suite += 'flaky'
+  endif
+
   test(test_name, exe,
     env : test_env,
     timeout : timeout,
diff --git a/meson.build b/meson.build
index ddcdc028d4..e37d18e091 100644
--- a/meson.build
+++ b/meson.build
@@ -129,13 +129,24 @@ installed_tests_template_tap = files('tests/template-tap.test.in')
 # Don’t build the tests unless we can run them (either natively, in an exe wrapper, or by installing them 
for later use)
 build_tests = get_option('tests') and (meson.can_run_host_binaries() or installed_tests_enabled)
 
+add_test_setup('default',
+  is_default: true,
+  exclude_suites: ['flaky'],
+)
+
+add_test_setup('flaky_tests',
+  # Empty test setup, used for having different results set for flaky tests
+  # Sadly we can't use (https://github.com/mesonbuild/meson/issues/10934):
+  #suites: ['flaky']
+)
+
 # Allow the tests to be easily run under valgrind using --setup=valgrind
 valgrind = find_program('valgrind', required: false)
 if valgrind.found()
   suppression_file = files('tools' / 'glib.supp')
 
   add_test_setup('valgrind',
-    exclude_suites: [ 'no-valgrind' ],
+    exclude_suites: [ 'no-valgrind', 'flaky' ],
     exe_wrapper: [
       valgrind,
       '--tool=memcheck',
@@ -2335,6 +2346,7 @@ common_test_env = [
   'G_ENABLE_DIAGNOSTIC=1',
   'MALLOC_CHECK_=2',
 ]
+
 test_timeout = 60
 test_timeout_slow = 180
[Date Prev][Date Next] [Thread Prev][Thread Next] [Thread Index] [Date Index] [Author Index]