mutter/tests/conform/run-tests.sh
Neil Roberts 996260ceab tests: Differentiate between known failures and missing requirements
Previously when make test is run it would say ‘fail’ in lower case
letters for both tests that are known bugs we need to fix and for
drivers that can't run the test. This makes it too easy to lose track
of bugs.

To fix this, the ADD_TEST macro has now been changed to take two sets
of flags instead of just one. The first specifies the requirements for
the test to run at all. The second specifies the set of flags required
to run without any known failures. The table in the test report now
says ‘n/a’ instead of ‘fail’ for tests that don't match the feature
requirements.

Reviewed-by: Robert Bragg <robert@linux.intel.com>

(cherry picked from commit 723f8d4402e7b2ef3a71f51bb29b10d1c0ec8d81)
2013-01-22 17:48:06 +00:00

150 lines
3.2 KiB
Bash
Executable File

#!/bin/bash
. $1
set +m
trap "" ERR
trap "" SIGABRT
trap "" SIGFPE
trap "" SIGSEGV
EXIT=0
MISSING_FEATURE="WARNING: Missing required feature";
KNOWN_FAILURE="WARNING: Test is known to fail";
if test -f ./test-conformance; then
TEST_CONFORMANCE=./test-conformance
elif test -f ./test-conformance.exe; then
TEST_CONFORMANCE=./test-conformance.exe
fi
echo "Key:"
echo "ok = Test passed"
echo "n/a = Driver is missing a feature required for the test"
echo "FAIL = Unexpected failure"
echo "fail = Test failed, but it was an expected failure"
echo "PASS! = Unexpected pass"
echo ""
get_status()
{
case $1 in
# Special value we use to indicate that the test failed
# but it was an expected failure so don't fail the
# overall test run as a result...
300)
echo -n "fail";;
# Special value we use to indicate that the test passed
# but we weren't expecting it to pass‽
400)
echo -n 'PASS!';;
# Special value to indicate the test is missing a required feature
500)
echo -n "n/a";;
0)
echo -n "ok";;
*)
echo -n "FAIL";;
esac
}
run_test()
{
$($TEST_CONFORMANCE $1 &>.log)
TMP=$?
var_name=$2_result
eval $var_name=$TMP
if grep -q "$MISSING_FEATURE" .log; then
if test $TMP -ne 0; then
eval $var_name=500
else
eval $var_name=400
fi
elif grep -q "$KNOWN_FAILURE" .log; then
if test $TMP -ne 0; then
eval $var_name=300
else
eval $var_name=400
fi
else
if test $TMP -ne 0; then EXIT=$TMP; fi
fi
}
TITLE_FORMAT="%35s"
printf $TITLE_FORMAT "Test"
if test $HAVE_GL -eq 1; then
GL_FORMAT=" %6s %8s %7s %6s %6s"
printf "$GL_FORMAT" "GL+FF" "GL+ARBFP" "GL+GLSL" "GL-NPT" "GL3"
fi
if test $HAVE_GLES2 -eq 1; then
GLES2_FORMAT=" %6s %7s"
printf "$GLES2_FORMAT" "ES2" "ES2-NPT"
fi
echo ""
echo ""
for test in `cat unit-tests`
do
export COGL_DEBUG=
if test $HAVE_GL -eq 1; then
export COGL_DRIVER=gl
export COGL_DEBUG=disable-glsl,disable-arbfp
run_test $test gl_ff
export COGL_DRIVER=gl
# NB: we can't explicitly disable fixed + glsl in this case since
# the arbfp code only supports fragment processing so we need either
# the fixed or glsl vertends
export COGL_DEBUG=
run_test $test gl_arbfp
export COGL_DRIVER=gl
export COGL_DEBUG=disable-fixed,disable-arbfp
run_test $test gl_glsl
export COGL_DRIVER=gl
export COGL_DEBUG=disable-npot-textures
run_test $test gl_npot
export COGL_DRIVER=gl3
export COGL_DEBUG=
run_test $test gl3
fi
if test $HAVE_GLES2 -eq 1; then
export COGL_DRIVER=gles2
export COGL_DEBUG=
run_test $test gles2
export COGL_DRIVER=gles2
export COGL_DEBUG=disable-npot-textures
run_test $test gles2_npot
fi
printf $TITLE_FORMAT "$test:"
if test $HAVE_GL -eq 1; then
printf "$GL_FORMAT" \
"`get_status $gl_ff_result`" \
"`get_status $gl_arbfp_result`" \
"`get_status $gl_glsl_result`" \
"`get_status $gl_npot_result`" \
"`get_status $gl3_result`"
fi
if test $HAVE_GLES2 -eq 1; then
printf "$GLES2_FORMAT" \
"`get_status $gles2_result`" \
"`get_status $gles2_npot_result`"
fi
echo ""
done
exit $EXIT