[cogl/wip/cogl-1.14: 77/177] tests: Differentiate between known failures and missing requirements



commit a260bd490df0d14ff7ff2f31124735f69a2b6dbc
Author: Neil Roberts <neil linux intel com>
Date:   Fri Nov 9 16:47:01 2012 +0000

    tests: Differentiate between known failures and missing requirements
    
    Previously when make test is run it would say âfailâ in lower case
    letters for both tests that are known bugs we need to fix and for
    drivers that can't run the test. This makes it too easy to lose track
    of bugs.
    
    To fix this, the ADD_TEST macro has now been changed to take two sets
    of flags instead of just one. The first specifies the requirements for
    the test to run at all. The second specifies the set of flags required
    to run without any known failures. The table in the test report now
    says ân/aâ instead of âfailâ for tests that don't match the feature
    requirements.
    
    Reviewed-by: Robert Bragg <robert linux intel com>
    
    (cherry picked from commit 723f8d4402e7b2ef3a71f51bb29b10d1c0ec8d81)

 tests/conform/run-tests.sh        |   16 +++++-
 tests/conform/test-conform-main.c |   93 +++++++++++++++++---------------
 tests/conform/test-utils.c        |  108 +++++++++++++++++++++----------------
 tests/conform/test-utils.h        |    3 +-
 4 files changed, 126 insertions(+), 94 deletions(-)
---
diff --git a/tests/conform/run-tests.sh b/tests/conform/run-tests.sh
index 266f1c6..11e740a 100755
--- a/tests/conform/run-tests.sh
+++ b/tests/conform/run-tests.sh
@@ -10,7 +10,8 @@ trap "" SIGFPE
 trap "" SIGSEGV
 
 EXIT=0
-WARNING="WARNING: Missing required feature";
+MISSING_FEATURE="WARNING: Missing required feature";
+KNOWN_FAILURE="WARNING: Test is known to fail";
 
 if test -f ./test-conformance; then
   TEST_CONFORMANCE=./test-conformance
@@ -20,6 +21,7 @@ fi
 
 echo "Key:"
 echo "ok = Test passed"
+echo "n/a = Driver is missing a feature required for the test"
 echo "FAIL = Unexpected failure"
 echo "fail = Test failed, but it was an expected failure"
 echo "PASS! = Unexpected pass"
@@ -38,6 +40,10 @@ get_status()
       400)
       echo -n 'PASS!';;
 
+      # Special value to indicate the test is missing a required feature
+      500)
+      echo -n "n/a";;
+
       0)
       echo -n "ok";;
 
@@ -52,7 +58,13 @@ run_test()
   TMP=$?
   var_name=$2_result
   eval $var_name=$TMP
-  if grep -q "$WARNING" .log; then
+  if grep -q "$MISSING_FEATURE" .log; then
+    if test $TMP -ne 0; then
+      eval $var_name=500
+    else
+      eval $var_name=400
+    fi
+  elif grep -q "$KNOWN_FAILURE" .log; then
     if test $TMP -ne 0; then
       eval $var_name=300
     else
diff --git a/tests/conform/test-conform-main.c b/tests/conform/test-conform-main.c
index 4e2c8bb..b510543 100644
--- a/tests/conform/test-conform-main.c
+++ b/tests/conform/test-conform-main.c
@@ -10,16 +10,17 @@
 #include "test-utils.h"
 
 /* A bit of sugar for adding new conformance tests */
-#define ADD_TEST(FUNC, REQUIREMENTS)  G_STMT_START {      \
-  extern void FUNC (void);                                \
-  if (strcmp (#FUNC, argv[1]) == 0)                       \
-    {                                                     \
-      test_utils_init (REQUIREMENTS);                     \
-      FUNC ();                                            \
-      test_utils_fini ();                                 \
-      exit (0);                                           \
-    }                                                     \
-} G_STMT_END
+#define ADD_TEST(FUNC, REQUIREMENTS, KNOWN_FAIL_REQUIREMENTS)           \
+  G_STMT_START {                                                        \
+    extern void FUNC (void);                                            \
+    if (strcmp (#FUNC, argv[1]) == 0)                                   \
+      {                                                                 \
+        test_utils_init (REQUIREMENTS, KNOWN_FAIL_REQUIREMENTS);        \
+        FUNC ();                                                        \
+        test_utils_fini ();                                             \
+        exit (0);                                                       \
+      }                                                                 \
+  } G_STMT_END
 
 #define UNPORTED_TEST(FUNC)
 
@@ -51,66 +52,70 @@ main (int argc, char **argv)
   UNPORTED_TEST (test_object);
   UNPORTED_TEST (test_fixed);
   UNPORTED_TEST (test_materials);
-  ADD_TEST (test_pipeline_user_matrix, 0);
-  ADD_TEST (test_blend_strings, 0);
-  ADD_TEST (test_premult, 0);
+  ADD_TEST (test_pipeline_user_matrix, 0, 0);
+  ADD_TEST (test_blend_strings, 0, 0);
+  ADD_TEST (test_premult, 0, 0);
   UNPORTED_TEST (test_readpixels);
-  ADD_TEST (test_path, 0);
-  ADD_TEST (test_depth_test, 0);
-  ADD_TEST (test_color_mask, 0);
-  ADD_TEST (test_backface_culling, TEST_REQUIREMENT_NPOT);
-  ADD_TEST (test_layer_remove, 0);
+  ADD_TEST (test_path, 0, 0);
+  ADD_TEST (test_depth_test, 0, 0);
+  ADD_TEST (test_color_mask, 0, 0);
+  ADD_TEST (test_backface_culling, 0, TEST_REQUIREMENT_NPOT);
+  ADD_TEST (test_layer_remove, 0, 0);
 
-  ADD_TEST (test_sparse_pipeline, 0);
+  ADD_TEST (test_sparse_pipeline, 0, 0);
 
-  ADD_TEST (test_npot_texture, TEST_REQUIREMENT_NPOT);
+  ADD_TEST (test_npot_texture, 0, TEST_REQUIREMENT_NPOT);
   UNPORTED_TEST (test_multitexture);
   UNPORTED_TEST (test_texture_mipmaps);
-  ADD_TEST (test_sub_texture, 0);
-  ADD_TEST (test_pixel_buffer, 0);
+  ADD_TEST (test_sub_texture, 0, 0);
+  ADD_TEST (test_pixel_buffer, 0, 0);
   UNPORTED_TEST (test_texture_rectangle);
-  ADD_TEST (test_texture_3d, TEST_REQUIREMENT_TEXTURE_3D);
-  ADD_TEST (test_wrap_modes, 0);
+  ADD_TEST (test_texture_3d, TEST_REQUIREMENT_TEXTURE_3D, 0);
+  ADD_TEST (test_wrap_modes, 0, 0);
   UNPORTED_TEST (test_texture_pixmap_x11);
   UNPORTED_TEST (test_texture_get_set_data);
-  ADD_TEST (test_atlas_migration, 0);
-  ADD_TEST (test_read_texture_formats, 0);
-  ADD_TEST (test_write_texture_formats, 0);
+  ADD_TEST (test_atlas_migration, 0, 0);
+  ADD_TEST (test_read_texture_formats, 0, 0);
+  ADD_TEST (test_write_texture_formats, 0, 0);
 
   UNPORTED_TEST (test_vertex_buffer_contiguous);
   UNPORTED_TEST (test_vertex_buffer_interleved);
   UNPORTED_TEST (test_vertex_buffer_mutability);
 
-  ADD_TEST (test_primitive, 0);
+  ADD_TEST (test_primitive, 0, 0);
 
-  ADD_TEST (test_just_vertex_shader, TEST_REQUIREMENT_GLSL);
-  ADD_TEST (test_pipeline_uniforms, TEST_REQUIREMENT_GLSL);
-  ADD_TEST (test_snippets, TEST_REQUIREMENT_GLSL);
-  ADD_TEST (test_custom_attributes, TEST_REQUIREMENT_GLSL);
+  ADD_TEST (test_just_vertex_shader, TEST_REQUIREMENT_GLSL, 0);
+  ADD_TEST (test_pipeline_uniforms, TEST_REQUIREMENT_GLSL, 0);
+  ADD_TEST (test_snippets, TEST_REQUIREMENT_GLSL, 0);
+  ADD_TEST (test_custom_attributes, TEST_REQUIREMENT_GLSL, 0);
 
-  ADD_TEST (test_bitmask, 0);
+  ADD_TEST (test_bitmask, 0, 0);
 
-  ADD_TEST (test_offscreen, 0);
+  ADD_TEST (test_offscreen, 0, 0);
 
-  ADD_TEST (test_point_size, 0);
+  ADD_TEST (test_point_size, 0, 0);
   ADD_TEST (test_point_sprite,
-            TEST_REQUIREMENT_POINT_SPRITE);
+            TEST_REQUIREMENT_POINT_SPRITE,
+            0);
   ADD_TEST (test_point_sprite_orientation,
-            TEST_KNOWN_FAILURE | TEST_REQUIREMENT_POINT_SPRITE);
+            TEST_REQUIREMENT_POINT_SPRITE,
+            TEST_KNOWN_FAILURE);
 
-  ADD_TEST (test_version, 0);
+  ADD_TEST (test_version, 0, 0);
 
-  ADD_TEST (test_alpha_test, 0);
+  ADD_TEST (test_alpha_test, 0, 0);
 
-  ADD_TEST (test_map_buffer_range, TEST_REQUIREMENT_MAP_WRITE);
+  ADD_TEST (test_map_buffer_range, TEST_REQUIREMENT_MAP_WRITE, 0);
 
   UNPORTED_TEST (test_viewport);
 
-  ADD_TEST (test_gles2_context, TEST_REQUIREMENT_GLES2_CONTEXT);
-  ADD_TEST (test_gles2_context_fbo, TEST_REQUIREMENT_GLES2_CONTEXT);
-  ADD_TEST (test_gles2_context_copy_tex_image, TEST_REQUIREMENT_GLES2_CONTEXT);
+  ADD_TEST (test_gles2_context, TEST_REQUIREMENT_GLES2_CONTEXT, 0);
+  ADD_TEST (test_gles2_context_fbo, TEST_REQUIREMENT_GLES2_CONTEXT, 0);
+  ADD_TEST (test_gles2_context_copy_tex_image,
+            TEST_REQUIREMENT_GLES2_CONTEXT,
+            0);
 
-  ADD_TEST (test_euler_quaternion, 0);
+  ADD_TEST (test_euler_quaternion, 0, 0);
 
   g_printerr ("Unknown test name \"%s\"\n", argv[1]);
 
diff --git a/tests/conform/test-utils.c b/tests/conform/test-utils.c
index 661926b..684327d 100644
--- a/tests/conform/test-utils.c
+++ b/tests/conform/test-utils.c
@@ -12,94 +12,106 @@ static CoglBool cogl_test_is_verbose;
 CoglContext *test_ctx;
 CoglFramebuffer *test_fb;
 
-void
-test_utils_init (TestFlags flags)
+static CoglBool
+check_flags (TestFlags flags,
+             CoglRenderer *renderer)
 {
-  static int counter = 0;
-  CoglError *error = NULL;
-  CoglOnscreen *onscreen = NULL;
-  CoglDisplay *display;
-  CoglRenderer *renderer;
-  CoglBool missing_requirement = FALSE;
-
-  if (counter != 0)
-    g_critical ("We don't support running more than one test at a time\n"
-                "in a single test run due to the state leakage that can\n"
-                "cause subsequent tests to fail.\n"
-                "\n"
-                "If you want to run all the tests you should run\n"
-                "$ make test-report");
-  counter++;
-
-  if (g_getenv ("COGL_TEST_VERBOSE") || g_getenv ("V"))
-    cogl_test_is_verbose = TRUE;
-
-  if (g_getenv ("G_DEBUG"))
-    {
-      char *debug = g_strconcat (g_getenv ("G_DEBUG"), ",fatal-warnings", NULL);
-      g_setenv ("G_DEBUG", debug, TRUE);
-      g_free (debug);
-    }
-  else
-    g_setenv ("G_DEBUG", "fatal-warnings", TRUE);
-
-  g_setenv ("COGL_X11_SYNC", "1", 0);
-
-  test_ctx = cogl_context_new (NULL, &error);
-  if (!test_ctx)
-    g_critical ("Failed to create a CoglContext: %s", error->message);
-
-  display = cogl_context_get_display (test_ctx);
-  renderer = cogl_display_get_renderer (display);
-
   if (flags & TEST_REQUIREMENT_GL &&
       cogl_renderer_get_driver (renderer) != COGL_DRIVER_GL &&
       cogl_renderer_get_driver (renderer) != COGL_DRIVER_GL3)
     {
-      missing_requirement = TRUE;
+      return FALSE;
     }
 
   if (flags & TEST_REQUIREMENT_NPOT &&
       !cogl_has_feature (test_ctx, COGL_FEATURE_ID_TEXTURE_NPOT))
     {
-      missing_requirement = TRUE;
+      return FALSE;
     }
 
   if (flags & TEST_REQUIREMENT_TEXTURE_3D &&
       !cogl_has_feature (test_ctx, COGL_FEATURE_ID_TEXTURE_3D))
     {
-      missing_requirement = TRUE;
+      return FALSE;
     }
 
   if (flags & TEST_REQUIREMENT_POINT_SPRITE &&
       !cogl_has_feature (test_ctx, COGL_FEATURE_ID_POINT_SPRITE))
     {
-      missing_requirement = TRUE;
+      return FALSE;
     }
 
   if (flags & TEST_REQUIREMENT_GLES2_CONTEXT &&
       !cogl_has_feature (test_ctx, COGL_FEATURE_ID_GLES2_CONTEXT))
     {
-      missing_requirement = TRUE;
+      return FALSE;
     }
 
   if (flags & TEST_REQUIREMENT_MAP_WRITE &&
       !cogl_has_feature (test_ctx, COGL_FEATURE_ID_MAP_BUFFER_FOR_WRITE))
     {
-      missing_requirement = TRUE;
+      return FALSE;
     }
 
   if (flags & TEST_REQUIREMENT_GLSL &&
       !cogl_has_feature (test_ctx, COGL_FEATURE_ID_GLSL))
     {
-      missing_requirement = TRUE;
+      return FALSE;
     }
 
   if (flags & TEST_KNOWN_FAILURE)
     {
-      missing_requirement = TRUE;
+      return FALSE;
     }
 
+  return TRUE;
+}
+
+void
+test_utils_init (TestFlags requirement_flags,
+                 TestFlags known_failure_flags)
+{
+  static int counter = 0;
+  CoglError *error = NULL;
+  CoglOnscreen *onscreen = NULL;
+  CoglDisplay *display;
+  CoglRenderer *renderer;
+  CoglBool missing_requirement;
+  CoglBool known_failure;
+
+  if (counter != 0)
+    g_critical ("We don't support running more than one test at a time\n"
+                "in a single test run due to the state leakage that can\n"
+                "cause subsequent tests to fail.\n"
+                "\n"
+                "If you want to run all the tests you should run\n"
+                "$ make test-report");
+  counter++;
+
+  if (g_getenv ("COGL_TEST_VERBOSE") || g_getenv ("V"))
+    cogl_test_is_verbose = TRUE;
+
+  if (g_getenv ("G_DEBUG"))
+    {
+      char *debug = g_strconcat (g_getenv ("G_DEBUG"), ",fatal-warnings", NULL);
+      g_setenv ("G_DEBUG", debug, TRUE);
+      g_free (debug);
+    }
+  else
+    g_setenv ("G_DEBUG", "fatal-warnings", TRUE);
+
+  g_setenv ("COGL_X11_SYNC", "1", 0);
+
+  test_ctx = cogl_context_new (NULL, &error);
+  if (!test_ctx)
+    g_critical ("Failed to create a CoglContext: %s", error->message);
+
+  display = cogl_context_get_display (test_ctx);
+  renderer = cogl_display_get_renderer (display);
+
+  missing_requirement = !check_flags (requirement_flags, renderer);
+  known_failure = !check_flags (known_failure_flags, renderer);
+
   if (getenv  ("COGL_TEST_ONSCREEN"))
     {
       onscreen = cogl_onscreen_new (test_ctx, 640, 480);
@@ -133,6 +145,8 @@ test_utils_init (TestFlags flags)
 
   if (missing_requirement)
     g_print ("WARNING: Missing required feature[s] for this test\n");
+  else if (known_failure)
+    g_print ("WARNING: Test is known to fail\n");
 }
 
 void
diff --git a/tests/conform/test-utils.h b/tests/conform/test-utils.h
index 93cce27..02af5f4 100644
--- a/tests/conform/test-utils.h
+++ b/tests/conform/test-utils.h
@@ -25,7 +25,8 @@ extern CoglContext *test_ctx;
 extern CoglFramebuffer *test_fb;
 
 void
-test_utils_init (TestFlags flags);
+test_utils_init (TestFlags requirement_flags,
+                 TestFlags known_failure_flags);
 
 void
 test_utils_fini (void);



[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]