brl/mutter
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tests: Differentiate between known failures and missing requirements

Browse Source 
Previously when make test is run it would say ‘fail’ in lower case
letters for both tests that are known bugs we need to fix and for
drivers that can't run the test. This makes it too easy to lose track
of bugs.

To fix this, the ADD_TEST macro has now been changed to take two sets
of flags instead of just one. The first specifies the requirements for
the test to run at all. The second specifies the set of flags required
to run without any known failures. The table in the test report now
says ‘n/a’ instead of ‘fail’ for tests that don't match the feature
requirements.

Reviewed-by: Robert Bragg <robert@linux.intel.com>

(cherry picked from commit 723f8d4402e7b2ef3a71f51bb29b10d1c0ec8d81)
This commit is contained in:
Neil Roberts 2012-11-09 16:47:01 +00:00 committed by Robert Bragg
parent 0b8780f94c
commit 996260ceab
4 changed files with 128 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tests/conform/run-tests.sh
View File

@ -10,7 +10,8 @@ trap "" SIGFPE
trap "" SIGSEGV trap "" SIGSEGV
EXIT=0 EXIT=0
WARNING="WARNING: Missing required feature"; MISSING_FEATURE="WARNING: Missing required feature";
KNOWN_FAILURE="WARNING: Test is known to fail";
if test -f ./test-conformance; then if test -f ./test-conformance; then
TEST_CONFORMANCE=./test-conformance TEST_CONFORMANCE=./test-conformance
@ -20,6 +21,7 @@ fi
echo "Key:" echo "Key:"
echo "ok = Test passed" echo "ok = Test passed"
echo "n/a = Driver is missing a feature required for the test"
echo "FAIL = Unexpected failure" echo "FAIL = Unexpected failure"
echo "fail = Test failed, but it was an expected failure" echo "fail = Test failed, but it was an expected failure"
echo "PASS! = Unexpected pass" echo "PASS! = Unexpected pass"
@ -38,6 +40,10 @@ get_status()
400) 400)
echo -n 'PASS!';; echo -n 'PASS!';;
# Special value to indicate the test is missing a required feature
500)
echo -n "n/a";;
0) 0)
echo -n "ok";; echo -n "ok";;
@ -52,7 +58,13 @@ run_test()
TMP=$? TMP=$?
var_name=$2_result var_name=$2_result
eval $var_name=$TMP eval $var_name=$TMP
if grep -q "$WARNING" .log; then if grep -q "$MISSING_FEATURE" .log; then
if test $TMP -ne 0; then
eval $var_name=500
else
eval $var_name=400
fi
elif grep -q "$KNOWN_FAILURE" .log; then
if test $TMP -ne 0; then if test $TMP -ne 0; then
eval $var_name=300 eval $var_name=300
else else

93
tests/conform/test-conform-main.c
View File

@ -10,16 +10,17 @@
#include "test-utils.h" #include "test-utils.h"
/* A bit of sugar for adding new conformance tests */ /* A bit of sugar for adding new conformance tests */
#define ADD_TEST(FUNC, REQUIREMENTS) G_STMT_START { \ #define ADD_TEST(FUNC, REQUIREMENTS, KNOWN_FAIL_REQUIREMENTS) \
extern void FUNC (void); \ G_STMT_START { \
if (strcmp (#FUNC, argv[1]) == 0) \ extern void FUNC (void); \
{ \ if (strcmp (#FUNC, argv[1]) == 0) \
test_utils_init (REQUIREMENTS); \ { \
FUNC (); \ test_utils_init (REQUIREMENTS, KNOWN_FAIL_REQUIREMENTS); \
test_utils_fini (); \ FUNC (); \
exit (0); \ test_utils_fini (); \
} \ exit (0); \
} G_STMT_END } \
} G_STMT_END
#define UNPORTED_TEST(FUNC) #define UNPORTED_TEST(FUNC)
@ -51,66 +52,70 @@ main (int argc, char **argv)
UNPORTED_TEST (test_object); UNPORTED_TEST (test_object);
UNPORTED_TEST (test_fixed); UNPORTED_TEST (test_fixed);
UNPORTED_TEST (test_materials); UNPORTED_TEST (test_materials);
ADD_TEST (test_pipeline_user_matrix, 0); ADD_TEST (test_pipeline_user_matrix, 0, 0);
ADD_TEST (test_blend_strings, 0); ADD_TEST (test_blend_strings, 0, 0);
ADD_TEST (test_premult, 0); ADD_TEST (test_premult, 0, 0);
UNPORTED_TEST (test_readpixels); UNPORTED_TEST (test_readpixels);
ADD_TEST (test_path, 0); ADD_TEST (test_path, 0, 0);
ADD_TEST (test_depth_test, 0); ADD_TEST (test_depth_test, 0, 0);
ADD_TEST (test_color_mask, 0); ADD_TEST (test_color_mask, 0, 0);
ADD_TEST (test_backface_culling, TEST_REQUIREMENT_NPOT); ADD_TEST (test_backface_culling, 0, TEST_REQUIREMENT_NPOT);
ADD_TEST (test_layer_remove, 0); ADD_TEST (test_layer_remove, 0, 0);
ADD_TEST (test_sparse_pipeline, 0); ADD_TEST (test_sparse_pipeline, 0, 0);
ADD_TEST (test_npot_texture, TEST_REQUIREMENT_NPOT); ADD_TEST (test_npot_texture, 0, TEST_REQUIREMENT_NPOT);
UNPORTED_TEST (test_multitexture); UNPORTED_TEST (test_multitexture);
UNPORTED_TEST (test_texture_mipmaps); UNPORTED_TEST (test_texture_mipmaps);
ADD_TEST (test_sub_texture, 0); ADD_TEST (test_sub_texture, 0, 0);
ADD_TEST (test_pixel_buffer, 0); ADD_TEST (test_pixel_buffer, 0, 0);
UNPORTED_TEST (test_texture_rectangle); UNPORTED_TEST (test_texture_rectangle);
ADD_TEST (test_texture_3d, TEST_REQUIREMENT_TEXTURE_3D); ADD_TEST (test_texture_3d, TEST_REQUIREMENT_TEXTURE_3D, 0);
ADD_TEST (test_wrap_modes, 0); ADD_TEST (test_wrap_modes, 0, 0);
UNPORTED_TEST (test_texture_pixmap_x11); UNPORTED_TEST (test_texture_pixmap_x11);
UNPORTED_TEST (test_texture_get_set_data); UNPORTED_TEST (test_texture_get_set_data);
ADD_TEST (test_atlas_migration, 0); ADD_TEST (test_atlas_migration, 0, 0);
ADD_TEST (test_read_texture_formats, 0); ADD_TEST (test_read_texture_formats, 0, 0);
ADD_TEST (test_write_texture_formats, 0); ADD_TEST (test_write_texture_formats, 0, 0);
UNPORTED_TEST (test_vertex_buffer_contiguous); UNPORTED_TEST (test_vertex_buffer_contiguous);
UNPORTED_TEST (test_vertex_buffer_interleved); UNPORTED_TEST (test_vertex_buffer_interleved);
UNPORTED_TEST (test_vertex_buffer_mutability); UNPORTED_TEST (test_vertex_buffer_mutability);
ADD_TEST (test_primitive, 0); ADD_TEST (test_primitive, 0, 0);
ADD_TEST (test_just_vertex_shader, TEST_REQUIREMENT_GLSL); ADD_TEST (test_just_vertex_shader, TEST_REQUIREMENT_GLSL, 0);
ADD_TEST (test_pipeline_uniforms, TEST_REQUIREMENT_GLSL); ADD_TEST (test_pipeline_uniforms, TEST_REQUIREMENT_GLSL, 0);
ADD_TEST (test_snippets, TEST_REQUIREMENT_GLSL); ADD_TEST (test_snippets, TEST_REQUIREMENT_GLSL, 0);
ADD_TEST (test_custom_attributes, TEST_REQUIREMENT_GLSL); ADD_TEST (test_custom_attributes, TEST_REQUIREMENT_GLSL, 0);
ADD_TEST (test_bitmask, 0); ADD_TEST (test_bitmask, 0, 0);
ADD_TEST (test_offscreen, 0); ADD_TEST (test_offscreen, 0, 0);
ADD_TEST (test_point_size, 0); ADD_TEST (test_point_size, 0, 0);
ADD_TEST (test_point_sprite, ADD_TEST (test_point_sprite,
TEST_REQUIREMENT_POINT_SPRITE); TEST_REQUIREMENT_POINT_SPRITE,
0);
ADD_TEST (test_point_sprite_orientation, ADD_TEST (test_point_sprite_orientation,
TEST_KNOWN_FAILURE | TEST_REQUIREMENT_POINT_SPRITE); TEST_REQUIREMENT_POINT_SPRITE,
TEST_KNOWN_FAILURE);
ADD_TEST (test_version, 0); ADD_TEST (test_version, 0, 0);
ADD_TEST (test_alpha_test, 0); ADD_TEST (test_alpha_test, 0, 0);
ADD_TEST (test_map_buffer_range, TEST_REQUIREMENT_MAP_WRITE); ADD_TEST (test_map_buffer_range, TEST_REQUIREMENT_MAP_WRITE, 0);
UNPORTED_TEST (test_viewport); UNPORTED_TEST (test_viewport);
ADD_TEST (test_gles2_context, TEST_REQUIREMENT_GLES2_CONTEXT); ADD_TEST (test_gles2_context, TEST_REQUIREMENT_GLES2_CONTEXT, 0);
ADD_TEST (test_gles2_context_fbo, TEST_REQUIREMENT_GLES2_CONTEXT); ADD_TEST (test_gles2_context_fbo, TEST_REQUIREMENT_GLES2_CONTEXT, 0);
ADD_TEST (test_gles2_context_copy_tex_image, TEST_REQUIREMENT_GLES2_CONTEXT); ADD_TEST (test_gles2_context_copy_tex_image,
TEST_REQUIREMENT_GLES2_CONTEXT,
0);
ADD_TEST (test_euler_quaternion, 0); ADD_TEST (test_euler_quaternion, 0, 0);
g_printerr ("Unknown test name \"%s\"\n", argv[1]); g_printerr ("Unknown test name \"%s\"\n", argv[1]);

112
tests/conform/test-utils.c
View File

@ -12,15 +12,72 @@ static CoglBool cogl_test_is_verbose;
CoglContext *test_ctx; CoglContext *test_ctx;
CoglFramebuffer *test_fb; CoglFramebuffer *test_fb;
static CoglBool
check_flags (TestFlags flags,
CoglRenderer *renderer)
{
if (flags & TEST_REQUIREMENT_GL &&
cogl_renderer_get_driver (renderer) != COGL_DRIVER_GL &&
cogl_renderer_get_driver (renderer) != COGL_DRIVER_GL3)
{
return FALSE;
}
if (flags & TEST_REQUIREMENT_NPOT &&
!cogl_has_feature (test_ctx, COGL_FEATURE_ID_TEXTURE_NPOT))
{
return FALSE;
}
if (flags & TEST_REQUIREMENT_TEXTURE_3D &&
!cogl_has_feature (test_ctx, COGL_FEATURE_ID_TEXTURE_3D))
{
return FALSE;
}
if (flags & TEST_REQUIREMENT_POINT_SPRITE &&
!cogl_has_feature (test_ctx, COGL_FEATURE_ID_POINT_SPRITE))
{
return FALSE;
}
if (flags & TEST_REQUIREMENT_GLES2_CONTEXT &&
!cogl_has_feature (test_ctx, COGL_FEATURE_ID_GLES2_CONTEXT))
{
return FALSE;
}
if (flags & TEST_REQUIREMENT_MAP_WRITE &&
!cogl_has_feature (test_ctx, COGL_FEATURE_ID_MAP_BUFFER_FOR_WRITE))
{
return FALSE;
}
if (flags & TEST_REQUIREMENT_GLSL &&
!cogl_has_feature (test_ctx, COGL_FEATURE_ID_GLSL))
{
return FALSE;
}
if (flags & TEST_KNOWN_FAILURE)
{
return FALSE;
}
return TRUE;
}
void void
test_utils_init (TestFlags flags) test_utils_init (TestFlags requirement_flags,
TestFlags known_failure_flags)
{ {
static int counter = 0; static int counter = 0;
CoglError *error = NULL; CoglError *error = NULL;
CoglOnscreen *onscreen = NULL; CoglOnscreen *onscreen = NULL;
CoglDisplay *display; CoglDisplay *display;
CoglRenderer *renderer; CoglRenderer *renderer;
CoglBool missing_requirement = FALSE; CoglBool missing_requirement;
CoglBool known_failure;
if (counter != 0) if (counter != 0)
g_critical ("We don't support running more than one test at a time\n" g_critical ("We don't support running more than one test at a time\n"
@ -52,53 +109,8 @@ test_utils_init (TestFlags flags)
display = cogl_context_get_display (test_ctx); display = cogl_context_get_display (test_ctx);
renderer = cogl_display_get_renderer (display); renderer = cogl_display_get_renderer (display);
if (flags & TEST_REQUIREMENT_GL && missing_requirement = !check_flags (requirement_flags, renderer);
cogl_renderer_get_driver (renderer) != COGL_DRIVER_GL && known_failure = !check_flags (known_failure_flags, renderer);
cogl_renderer_get_driver (renderer) != COGL_DRIVER_GL3)
{
missing_requirement = TRUE;
}
if (flags & TEST_REQUIREMENT_NPOT &&
!cogl_has_feature (test_ctx, COGL_FEATURE_ID_TEXTURE_NPOT))
{
missing_requirement = TRUE;
}
if (flags & TEST_REQUIREMENT_TEXTURE_3D &&
!cogl_has_feature (test_ctx, COGL_FEATURE_ID_TEXTURE_3D))
{
missing_requirement = TRUE;
}
if (flags & TEST_REQUIREMENT_POINT_SPRITE &&
!cogl_has_feature (test_ctx, COGL_FEATURE_ID_POINT_SPRITE))
{
missing_requirement = TRUE;
}
if (flags & TEST_REQUIREMENT_GLES2_CONTEXT &&
!cogl_has_feature (test_ctx, COGL_FEATURE_ID_GLES2_CONTEXT))
{
missing_requirement = TRUE;
}
if (flags & TEST_REQUIREMENT_MAP_WRITE &&
!cogl_has_feature (test_ctx, COGL_FEATURE_ID_MAP_BUFFER_FOR_WRITE))
{
missing_requirement = TRUE;
}
if (flags & TEST_REQUIREMENT_GLSL &&
!cogl_has_feature (test_ctx, COGL_FEATURE_ID_GLSL))
{
missing_requirement = TRUE;
}
if (flags & TEST_KNOWN_FAILURE)
{
missing_requirement = TRUE;
}
if (getenv ("COGL_TEST_ONSCREEN")) if (getenv ("COGL_TEST_ONSCREEN"))
{ {
@ -133,6 +145,8 @@ test_utils_init (TestFlags flags)
if (missing_requirement) if (missing_requirement)
g_print ("WARNING: Missing required feature[s] for this test\n"); g_print ("WARNING: Missing required feature[s] for this test\n");
else if (known_failure)
g_print ("WARNING: Test is known to fail\n");
} }
void void

3
tests/conform/test-utils.h
View File

@ -25,7 +25,8 @@ extern CoglContext *test_ctx;
extern CoglFramebuffer *test_fb; extern CoglFramebuffer *test_fb;
void void
test_utils_init (TestFlags flags); test_utils_init (TestFlags requirement_flags,
TestFlags known_failure_flags);
void void
test_utils_fini (void); test_utils_fini (void);