[glib/wip/pwithnall/macos-tests: 67/67] tests: Stop ignoring test failures on macOS

[Philip Withnall <pwithnall src gnome org>]

commit 12bf18fe16d4d46ea48d398b890dbe1cbc3aabf3
Author: Philip Withnall <pwithnall endlessos org>
Date:   Thu Feb 17 23:01:35 2022 +0000

    tests: Stop ignoring test failures on macOS
    
    Test failures were previously ignored on macOS because there are 13
    tests which consistently fail (and have not yet been fixed, because
    there are no regularly active macOS maintainers for GLib; you could help
    here!).
    
    However, this means that new test failures can’t be spotted.
    
    So, explicitly mark those 13 tests as `should_fail` on macOS, and then
    make other test failures cause failure of the CI run.
    
    We can track the process of fixing those 13 tests on #1392 and #1251.
    
    Signed-off-by: Philip Withnall <pwithnall endlessos org>
    
    Helps: #1392

 .gitlab-ci.yml         |  3 +--
 gio/tests/meson.build  | 54 ++++++++++++++++++++++++++++++++++++++++----------
 glib/tests/meson.build | 17 +++++++++++++---
 3 files changed, 59 insertions(+), 15 deletions(-)
---
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index cfd955462..81bb06581 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -449,8 +449,7 @@ macos:
             --werror
             _build
     - ninja -C _build
-    # FIXME: Multiple unit tests currently fails
-    - .gitlab-ci/run-tests.sh || true
+    - .gitlab-ci/run-tests.sh
   artifacts:
     reports:
       junit: "_build/${CI_JOB_NAME}-report.xml"
diff --git a/gio/tests/meson.build b/gio/tests/meson.build
index eab7999e6..04191e3fe 100644
--- a/gio/tests/meson.build
+++ b/gio/tests/meson.build
@@ -42,14 +42,20 @@ giotypefuncs_inc = custom_target(
 
 #  Test programs buildable on all platforms
 gio_tests = {
-  'appmonitor' : {},
+  'appmonitor' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'async-close-output-stream' : {},
   'async-splice-output-stream' : {},
   'buffered-input-stream' : {},
   'buffered-output-stream' : {},
   'cancellable' : {},
   'contexts' : {},
-  'contenttype' : {},
+  'contenttype' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392 / 
https://gitlab.gnome.org/GNOME/glib/-/issues/1251
+    'should_fail' : host_system == 'darwin',
+  },
   'converter-stream' : {},
   'credentials' : {},
   'cxx' : {
@@ -58,13 +64,26 @@ gio_tests = {
   'data-input-stream' : {},
   'data-output-stream' : {},
   'debugcontroller' : {},
-  'defaultvalue' : {'extra_sources' : [giotypefuncs_inc]},
+  'defaultvalue' : {
+    'extra_sources' : [giotypefuncs_inc],
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'fileattributematcher' : {},
   'filter-streams' : {},
-  'giomodule' : {},
-  'gsubprocess' : {},
+  'giomodule' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
+  'gsubprocess' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'g-file' : {},
-  'g-file-info' : {},
+  'g-file-info' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'g-icon' : {},
   'gdbus-addresses' : {},
   'gdbus-message' : {},
@@ -74,7 +93,11 @@ gio_tests = {
   'memory-monitor' : {},
   'memory-output-stream' : {},
   'mount-operation' : {},
-  'network-address' : {'extra_sources': ['mock-resolver.c']},
+  'network-address' : {
+    'extra_sources': ['mock-resolver.c'],
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'network-monitor' : {},
   'network-monitor-race' : {},
   'permission' : {},
@@ -85,7 +108,10 @@ gio_tests = {
   'simple-async-result' : {},
   'simple-proxy' : {},
   'sleepy-stream' : {},
-  'socket' : {},
+  'socket' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'socket-listener' : {},
   'socket-service' : {},
   'srvtarget' : {},
@@ -99,7 +125,10 @@ gio_tests = {
   'tls-interaction' : {'extra_sources' : ['gtesttlsbackend.c']},
   'tls-database' : {'extra_sources' : ['gtesttlsbackend.c']},
   'tls-bindings' : {'extra_sources' : ['gtesttlsbackend.c']},
-  'gdbus-address-get-session' : {},
+  'gdbus-address-get-session' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'win32-appinfo' : {},
 }
 
@@ -158,7 +187,9 @@ if host_machine.system() != 'windows'
     'file' : {},
     'gdbus-peer' : {
       'dependencies' : [libgdbus_example_objectmanager_dep],
-      'install_rpath' : installed_tests_execdir
+      'install_rpath' : installed_tests_execdir,
+      # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+      'should_fail' : host_system == 'darwin',
     },
     'gdbus-peer-object-manager' : {},
     'live-g-file' : {},
@@ -779,6 +810,8 @@ if not meson.is_cross_build() or meson.has_exe_wrapper()
   gio_tests += {
     'resources' : {
       'extra_sources' : resources_extra_sources,
+      # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+      'should_fail' : host_system == 'darwin',
     },
   }
 endif
@@ -832,6 +865,7 @@ foreach test_name, extra_args : gio_tests
     suite : suite,
     is_parallel : extra_args.get('is_parallel', true),
     depends : extra_args.get('depends', []),
+    should_fail : extra_args.get('should_fail', false),
   )
 endforeach
 
diff --git a/glib/tests/meson.build b/glib/tests/meson.build
index 869d5819b..f818293d6 100644
--- a/glib/tests/meson.build
+++ b/glib/tests/meson.build
@@ -19,9 +19,15 @@ glib_tests = {
     'source' : ['cxx.cpp'],
   },
   'dataset' : {},
-  'date' : {},
+  'date' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'dir' : {},
-  'environment' : {},
+  'environment' : {
+    # FIXME: https://gitlab.gnome.org/GNOME/glib/-/issues/1392
+    'should_fail' : host_system == 'darwin',
+  },
   'error' : {},
   'fileutils' : {},
   'gdatetime' : {
@@ -241,7 +247,12 @@ foreach test_name, extra_args : glib_tests
 
   suite = ['glib'] + extra_args.get('suite', [])
   timeout = suite.contains('slow') ? test_timeout_slow : test_timeout
-  test(test_name, exe, env : test_env, timeout : timeout, suite : suite)
+  test(test_name, exe,
+    env : test_env,
+    timeout : timeout,
+    suite : suite,
+    should_fail : extra_args.get('should_fail', false),
+  )
 endforeach
 
 executable('spawn-path-search-helper', 'spawn-path-search-helper.c',