tests: Improve thoroughness and reporting of make test

This patch reworks our conformance testing framework because it seems
that glib's gtesting framework isn't really well suited to our use case.
For example we weren't able to test windows builds given the way we
were using it and also for each test we'd like to repeat the test
with several different environments so we can test important driver and
feature combinations.

This patch instead switches away to a simplified but custom approach for
running our unit tests. We hope that having a more bespoke setup will
enable us to easily extend it to focus on the details important to us.

Notable changes with this new approach are:

We can now run 'make test' for our mingw windows builds.

We've got rid of all the test-*report* make rules and we're just left
with 'make test'

'make test' now runs each test several times with different driver and
feature combinations checking the result for each run. 'make test' will
then output a concise table of all of the results.

The combinations tested are:
- OpenGL Fixed Function
- OpenGL ARBfp
- OpenGL GLSL
- OpenGL No NPOT texture support
- OpenGLES 2.0
- OpenGLES 2.0 No NPOT texture support

Reviewed-by: Neil Roberts <neil@linux.intel.com>

configure.ac

@@ -432,6 +432,7 @@ GL_LIBRARY_DIRECTLY_LINKED=no
 
 enabled_drivers=""
 
+HAVE_GLES1=0
 AC_ARG_ENABLE(
   [gles1],
   [AC_HELP_STRING([--enable-gles1=@<:@no/yes@:>@], [Enable support for OpenGL-ES 1.1 @<:@default=no@:>@])],
@@ -450,6 +451,7 @@ AS_IF([test "x$enable_gles1" = "xyes"],
         AC_DEFINE([HAVE_COGL_GLES], 1, [Have GLES 1.1 for rendering])
         COGL_DEFINES_SYMBOLS="$COGL_DEFINES_SYMBOLS COGL_HAS_GLES CLUTTER_COGL_HAS_GLES"
         COGL_DEFINES_SYMBOLS="$COGL_DEFINES_SYMBOLS COGL_HAS_GLES1"
+        HAVE_GLES1=1
 
         PKG_CHECK_EXISTS([glesv1_cm],
           [COGL_PKG_REQUIRES_GL="$COGL_PKG_REQUIRES_GL glesv1_cm"
@@ -511,6 +513,7 @@ AS_IF([test "x$enable_gles1" = "xyes"],
           ])
       ])
 
+HAVE_GLES2=0
 AC_ARG_ENABLE(
   [gles2],
   [AC_HELP_STRING([--enable-gles2=@<:@no/yes@:>@], [Enable support for OpenGL-ES 2.0 @<:@default=no@:>@])],
@@ -528,6 +531,7 @@ AS_IF([test "x$enable_gles2" = "xyes"],
         AC_DEFINE([HAVE_COGL_GLES2], 1, [Have GLES 2.0 for rendering])
         COGL_DEFINES_SYMBOLS="$COGL_DEFINES_SYMBOLS COGL_HAS_GLES CLUTTER_COGL_HAS_GLES"
         COGL_DEFINES_SYMBOLS="$COGL_DEFINES_SYMBOLS COGL_HAS_GLES2"
+        HAVE_GLES2=1
 
         PKG_CHECK_EXISTS([glesv2],
           [COGL_PKG_REQUIRES_GL="$COGL_PKG_REQUIRES_GL glesv2"
@@ -551,6 +555,7 @@ AS_IF([test "x$enable_gles2" = "xyes"],
         NEED_EGL=yes
       ])
 
+HAVE_GL=0
 AC_ARG_ENABLE(
   [gl],
   [AC_HELP_STRING([--enable-gl=@<:@no/yes@:>@], [Enable support for OpenGL @<:@default=yes@:>@])],
@@ -602,6 +607,7 @@ AS_IF([test "x$enable_gl" = "xyes"],
 
         COGL_DEFINES_SYMBOLS="$COGL_DEFINES_SYMBOLS COGL_HAS_GL"
         COGL_DEFINES_SYMBOLS="$COGL_DEFINES_SYMBOLS CLUTTER_COGL_HAS_GL"
+        HAVE_GL=1
       ])
 
 AM_CONDITIONAL([COGL_DRIVER_GL_SUPPORTED], [test "x$enable_gl" = "xyes"])
@@ -623,8 +629,11 @@ AC_ARG_WITH([gles2-libname],
             [COGL_GLES2_LIBNAME="$withval"])
 
 AC_SUBST([COGL_GL_LIBNAME])
+AC_SUBST([HAVE_GL])
 AC_SUBST([COGL_GLES1_LIBNAME])
+AC_SUBST([HAVE_GLES1])
 AC_SUBST([COGL_GLES2_LIBNAME])
+AC_SUBST([HAVE_GLES2])
 
 if test "x$GL_LIBRARY_DIRECTLY_LINKED" = "xyes"; then
    AC_DEFINE([HAVE_DIRECTLY_LINKED_GL_LIBRARY], [1],
@@ -1126,6 +1135,7 @@ doc/reference/cogl-2.0-experimental/cogl-2.0-experimental-docs.xml
 examples/Makefile
 tests/Makefile
 tests/conform/Makefile
+tests/conform/config.env
 tests/conform/test-launcher.sh
 tests/data/Makefile
 po/Makefile.in

tests/Makefile.am

@@ -7,10 +7,7 @@ EXTRA_DIST = README
 test conform:
 	( cd ./conform && $(MAKE) $(AM_MAKEFLAGS) $@ ) || exit $$?
 
-test-report full-report:
+.PHONY: test conform
-	( cd ./conform && $(MAKE) $(AM_MAKEFLAGS) $@ ) || exit $$?
-
-.PHONY: test conform test-report full-report
 
 # run make test as part of make check
 check-local: test

tests/conform/Makefile.am

@@ -61,39 +61,25 @@ endif
 # For convenience, this provides a way to easily run individual unit tests:
 .PHONY: wrappers clean-wrappers
 
-#UNIT_TESTS = `./test-conformance -l -m thorough | $(GREP) '^/'`
-
 wrappers: stamp-test-conformance
 	@true
 stamp-test-conformance: Makefile $(srcdir)/test-conform-main.c
 	@mkdir -p wrappers
-	@sed -n \
+	@sed -n -e 's/^ \{1,\}ADD_TEST *( *\([a-zA-Z0-9_]\{1,\}\).*/\1/p' $(srcdir)/test-conform-main.c > unit-tests
-		-e 's/^ \{1,\}ADD_TEST *(.*"\([^",]\{1,\}\)", *\([a-zA-Z0-9_]\{1,\}\).*/\/conform\1\/\2/p' \
-		-e 's/^ \{1,\}ADD_CONDITIONAL_TEST *(.*"\([^",]\{1,\}\)", *\([a-zA-Z0-9_]\{1,\}\).*/\/conform\1\/\2/p' \
-		-e 's/^ \{1,\}ADD_TODO_TEST *(.*"\([^",]\{1,\}\)", *\([a-zA-Z0-9_]\{1,\}\).*/\/conform\1\/\2/p' \
-	$(srcdir)/test-conform-main.c > unit-tests
 	@chmod +x test-launcher.sh
 	@( echo "/stamp-test-conformance" ; \
-	   echo "/test-conformance" ; \
+	   echo "/test-conformance$(EXEEXT)" ; \
 	   echo "*.o" ; \
-	   echo "*.xml" ; \
-	   echo "*.html" ; \
 	   echo ".gitignore" ; \
-	   echo "unit-tests" ; \
+	   echo "unit-tests" ; ) > .gitignore
-	   echo "/wrappers/" ) > .gitignore
 	@for i in `cat unit-tests`; \
 	do \
 		unit=`basename $$i | sed -e s/_/-/g`; \
 		echo "  GEN    $$unit"; \
 		( echo "#!/bin/sh" ; echo "$(abs_builddir)/test-launcher.sh '$$i' \"\$$@\"" ) > $$unit$(SHEXT) ; \
-		( echo "#!/bin/sh" ; echo "exec $(abs_builddir)/test-conformance$(EXEEXT) -p $$i \"\$$@\"" ) > wrappers/$$unit$(SHEXT) ; \
-		( echo "test-conformance-cogl$(EXEEXT) -p $$i" ) > $$unit.bat ; \
-		( echo "test-conformance-cogl$(EXEEXT) -p $$i" ) >> test-conformance-cogl.bat ; \
 		chmod +x $$unit$(SHEXT); \
-		chmod +x wrappers/$$unit$(SHEXT); \
 		echo "/$$unit$(SHEXT)" >> .gitignore; \
 	done \
-	&& cp *.bat $(top_builddir)/build/win32/ \
 	&& echo timestamp > $(@F)
 
 clean-wrappers:
@@ -102,11 +88,8 @@ clean-wrappers:
 		unit=`basename $$i | sed -e s/_/-/g`; \
 		echo "  RM     $$unit"; \
 		rm -f $$unit$(SHEXT) ; \
-		rm -f wrappers/$$unit$(SHEXT) ; \
 	done \
 	&& rm -f unit-tests \
-	&& rm -f $(top_builddir)/build/win32/*.bat \
-	&& rm -f *.bat \
 	&& rm -f stamp-test-conformance
 
 # NB: BUILT_SOURCES here a misnomer. We aren't building source, just inserting
@@ -131,117 +114,13 @@ test_conformance_LDADD = $(COGL_DEP_LIBS) $(top_builddir)/cogl/libcogl.la
 test_conformance_LDFLAGS = -export-dynamic
 
 test: wrappers
-	@$(top_srcdir)/tests/conform/run-tests.sh \
+	@$(top_srcdir)/tests/conform/run-tests.sh $(abs_builddir)/config.env
-	  ./test-conformance$(EXEEXT) -o test-report.xml
-
-test-verbose: wrappers
-	@$(top_srcdir)/tests/conform/run-tests.sh \
-	  ./test-conformance$(EXEEXT) -o test-report.xml --verbose
-
-GTESTER = gtester
-GTESTER_REPORT = gtester-report
 
 # XXX: we could prevent the conformance test suite from running
 #      by simply defining this variable conditionally
 TEST_PROGS = test-conformance
 
 .PHONY: test
-.PHONY: test-report perf-report full-report
-.PHONY: test-report-npot perf-report-npot full-report-npot
-
-# test-report: run tests and generate report
-# perf-report: run tests with -m perf and generate report
-# full-report: like test-report: with -m perf and -m slow
-test-report perf-report full-report:	${TEST_PROGS}
-	@test -z "${TEST_PROGS}" || { \
-	  export GTESTER_LOGDIR=`mktemp -d "$(srcdir)/.testlogs-XXXXXX"` ; \
-	  if test -d "$(top_srcdir)/.git"; then \
-	    export REVISION="`git describe`" ;  \
-	  else \
-	    export REVISION="$(VERSION) $(CLUTTER_RELEASE_STATUS)" ; \
-	  fi ; \
-	  export TIMESTAMP=`date +%Y-%m-%dT%H:%M:%S%z` ; \
-	  case $@ in \
-	  test-report) test_options="-k";; \
-	  perf-report) test_options="-k -m=perf";; \
-	  full-report) test_options="-k -m=perf -m=slow";; \
-	  esac ; \
-	  $(top_srcdir)/tests/conform/run-tests.sh \
-	    ./test-conformance$(EXEEXT) \
-	    --verbose \
-	    $$test_options \
-	    -o `mktemp "$$GTESTER_LOGDIR/log-XXXXXX"` ; \
-	  echo '<?xml version="1.0"?>'              > $@.xml ; \
-	  echo '<report-collection>'               >> $@.xml ; \
-	  echo '<info>'                            >> $@.xml ; \
-	  echo '  <package>$(PACKAGE)</package>'   >> $@.xml ; \
-	  echo '  <version>$(VERSION)</version>'   >> $@.xml ; \
-	  echo "  <revision>$$REVISION</revision>" >> $@.xml ; \
-	  echo "  <date>$$TIMESTAMP</date>"        >> $@.xml ; \
-	  echo '</info>'                           >> $@.xml ; \
-	  for lf in `ls -L "$$GTESTER_LOGDIR"/.` ; do \
-	    sed '1,1s/^<?xml\b[^>?]*?>//' <"$$GTESTER_LOGDIR"/"$$lf" >> $@.xml ; \
-	  done ; \
-	  echo >> $@.xml ; \
-	  echo '</report-collection>' >> $@.xml ; \
-	  ${GTESTER_REPORT} --version 2>/dev/null 1>&2 ; test "$$?" != 0 || ${GTESTER_REPORT} $@.xml >$@.html ; \
-	  rm -rf "$$GTESTER_LOGDIR" ; \
-	}
-
-# same as above, but with a wrapper that forcibly disables non-power of
-# two textures
-test-report-npot perf-report-npot full-report-npot:	${TEST_PROGS}
-	@test -z "${TEST_PROGS}" || { \
-	  export COGL_DEBUG="$COGL_DEBUG,disable-npot-textures"; \
-	  export GTESTER_LOGDIR=`mktemp -d "$(srcdir)/.testlogs-XXXXXX"` ; \
-	  if test -d "$(top_srcdir)/.git"; then \
-	    export REVISION="`git describe`" ;  \
-	  else \
-	    export REVISION="$(VERSION) $(CLUTTER_RELEASE_STATUS)" ; \
-	  fi ; \
-	  export TIMESTAMP=`date +%Y-%m-%dT%H:%M:%S%z` ; \
-	  case $@ in \
-	  test-report-npot) test_options="-k";; \
-	  perf-report-npot) test_options="-k -m=perf";; \
-	  full-report-npot) test_options="-k -m=perf -m=slow";; \
-	  esac ; \
-	  $(top_srcdir)/tests/conform/run-tests.sh \
-	    ./test-conformance$(EXEEXT) \
-	    --verbose \
-	    $$test_options \
-	    -o `mktemp "$$GTESTER_LOGDIR/log-XXXXXX"` ; \
-	  echo '<?xml version="1.0"?>'              > $@.xml ; \
-	  echo '<report-collection>'               >> $@.xml ; \
-	  echo '<info>'                            >> $@.xml ; \
-	  echo '  <package>$(PACKAGE)</package>'   >> $@.xml ; \
-	  echo '  <version>$(VERSION)</version>'   >> $@.xml ; \
-	  echo "  <revision>$$REVISION</revision>" >> $@.xml ; \
-	  echo "  <date>$$TIMESTAMP</date>"        >> $@.xml ; \
-	  echo '</info>'                           >> $@.xml ; \
-	  for lf in `ls -L "$$GTESTER_LOGDIR"/.` ; do \
-	    sed '1,1s/^<?xml\b[^>?]*?>//' <"$$GTESTER_LOGDIR"/"$$lf" >> $@.xml ; \
-	  done ; \
-	  echo >> $@.xml ; \
-	  echo '</report-collection>' >> $@.xml ; \
-	  ${GTESTER_REPORT} --version 2>/dev/null 1>&2 ; test "$$?" != 0 || ${GTESTER_REPORT} $@.xml >$@.html ; \
-	  rm -rf "$$GTESTER_LOGDIR" ; \
-	}
-
-XML_REPORTS = \
-	test-report.xml 	\
-	perf-report.xml 	\
-	full-report.xml 	\
-	test-report-npot.xml 	\
-	perf-report-npot.xml 	\
-	full-report-npot.xml
-
-HTML_REPORTS = \
-	test-report.html 	\
-	perf-report.html 	\
-	full-report.html 	\
-	test-report-npot.html 	\
-	perf-report-npot.html 	\
-	full-report-npot.html
 
 EXTRA_DIST = test-launcher.sh.in run-tests.sh
 DISTCLEANFILES = test-launcher.sh .gitignore
@@ -286,4 +165,3 @@ DISTCLEANFILES += ../../build/win32/vs9/test-conformance-cogl.vcproj	\
 # we override the clean-generic target to clean up the wrappers so
 # we cannot use CLEANFILES
 clean-generic: clean-wrappers
-	$(QUIET_RM)rm -f $(XML_REPORTS) $(HTML_REPORTS)

tests/conform/config.env.in

@@ -0,0 +1,3 @@
+HAVE_GL=@HAVE_GL@
+HAVE_GLES1=@HAVE_GLES1@
+HAVE_GLES2=@HAVE_GLES2@

tests/conform/run-tests.sh

@@ -1,12 +1,132 @@
-#!/bin/sh
+#!/bin/bash
 
-BINARY=$1
+. $1
-shift
 
-TMP=`./$BINARY -l -m thorough | grep '^/' | sed -e s/_/-/g`
+set +m
-for i in $TMP
+
+trap "" ERR
+trap "" SIGABRT
+trap "" SIGFPE
+trap "" SIGSEGV
+
+EXIT=0
+WARNING="WARNING: Missing required feature";
+
+if test -f ./test-conformance; then
+  TEST_CONFORMANCE=./test-conformance
+elif test -f ./test-conformance.exe; then
+  TEST_CONFORMANCE=./test-conformance.exe
+fi
+
+echo "Key:"
+echo "ok = Test passed"
+echo "FAIL = Unexpected failure"
+echo "fail = Test failed, but it was an expected failure"
+echo "PASS! = Unexpected pass"
+echo ""
+
+get_status()
+{
+  case $1 in
+      # Special value we use to indicate that the test failed
+      # but it was an expected failure so don't fail the
+      # overall test run as a result...
+      300)
+      echo -n "fail";;
+      # Special value we use to indicate that the test passed
+      # but we weren't expecting it to pass‽
+      400)
+      echo -n 'PASS!';;
+
+      0)
+      echo -n "ok";;
+
+      *)
+      echo -n "FAIL";;
+  esac
+}
+
+run_test()
+{
+  $($TEST_CONFORMANCE $1 &>.log)
+  TMP=$?
+  var_name=$2_result
+  eval $var_name=$TMP
+  if grep -q "$WARNING" .log; then
+    if test $TMP -ne 0; then
+      eval $var_name=300
+    else
+      eval $var_name=400
+    fi
+  else
+    if test $TMP -ne 0; then EXIT=$TMP; fi
+  fi
+}
+
+TITLE_FORMAT="%35s"
+printf $TITLE_FORMAT "Test"
+
+if test $HAVE_GL -eq 1; then
+  GL_FORMAT=" %6s %8s %7s %6s"
+  printf "$GL_FORMAT" "GL+FF" "GL+ARBFP" "GL+GLSL" "GL-NPT"
+fi
+if test $HAVE_GLES2 -eq 1; then
+  GLES2_FORMAT=" %6s %7s"
+  printf "$GLES2_FORMAT" "ES2" "ES2-NPT"
+fi
+
+echo ""
+echo ""
+
+for test in `cat unit-tests`
 do
-  TESTS="$TESTS wrappers/`basename $i`"
+  export COGL_DEBUG=
+
+  if test $HAVE_GL -eq 1; then
+    export COGL_DRIVER=gl
+    export COGL_DEBUG=disable-glsl,disable-arbfp
+    run_test $test gl_ff
+
+    export COGL_DRIVER=gl
+    # NB: we can't explicitly disable fixed + glsl in this case since
+    # the arbfp code only supports fragment processing so we need either
+    # the fixed or glsl vertends
+    export COGL_DEBUG=
+    run_test $test gl_arbfp
+
+    export COGL_DRIVER=gl
+    export COGL_DEBUG=disable-fixed,disable-arbfp
+    run_test $test gl_glsl
+
+    export COGL_DRIVER=gl
+    export COGL_DEBUG=disable-npot-textures
+    run_test $test gl_npot
+  fi
+
+  if test $HAVE_GLES2 -eq 1; then
+    export COGL_DRIVER=gles2
+    export COGL_DEBUG=
+    run_test $test gles2
+
+    export COGL_DRIVER=gles2
+    export COGL_DEBUG=disable-npot-textures
+    run_test $test gles2_npot
+  fi
+
+  printf $TITLE_FORMAT "$test:"
+  if test $HAVE_GL -eq 1; then
+    printf "$GL_FORMAT" \
+      "`get_status $gl_ff_result`" \
+      "`get_status $gl_arbfp_result`" \
+      "`get_status $gl_glsl_result`" \
+      "`get_status $gl_npot_result`"
+  fi
+  if test $HAVE_GLES2 -eq 1; then
+    printf "$GLES2_FORMAT" \
+      "`get_status $gles2_result`" \
+      "`get_status $gles2_npot_result`"
+  fi
+  echo ""
 done
 
-exec gtester "$@" $TESTS
+exit $EXIT

tests/conform/test-atlas-migration.c

@@ -128,6 +128,6 @@ test_cogl_atlas_migration (TestUtilsGTestFixture *fixture,
   for (tex_num = 0; tex_num < N_TEXTURES; tex_num++)
     cogl_object_unref (textures[tex_num]);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }

tests/conform/test-backface-culling.c

@@ -307,7 +307,7 @@ test_cogl_backface_culling (TestUtilsGTestFixture *fixture,
   cogl_handle_unref (state.offscreen_tex);
   cogl_handle_unref (state.texture);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-bitmask.c

@@ -181,6 +181,6 @@ test_cogl_bitmask (TestUtilsGTestFixture *fixture,
   for (i = 0; i < 1024; i++)
     g_assert_cmpint (_cogl_bitmask_get (&bitmask, i), ==, (i < 500));
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }

tests/conform/test-blend-strings.c

@@ -82,7 +82,7 @@ test_blend (TestState *state,
     {
       /* It's not strictly a test failure; you need a more capable GPU or
        * driver to test this blend string. */
-      if (g_test_verbose ())
+      if (cogl_test_verbose ())
 	{
 	  g_debug ("Failed to test blend string %s: %s",
 		   blend_string, error->message);
@@ -106,7 +106,7 @@ test_blend (TestState *state,
   y_off = y * QUAD_WIDTH + (QUAD_WIDTH / 2);
   x_off = x * QUAD_WIDTH + (QUAD_WIDTH / 2);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     {
       g_print ("test_blend (%d, %d):\n%s\n", x, y, blend_string);
       g_print ("  src color = %02x, %02x, %02x, %02x\n", Sr, Sg, Sb, Sa);
@@ -276,7 +276,7 @@ test_tex_combine (TestState *state,
   y_off = y * QUAD_WIDTH + (QUAD_WIDTH / 2);
   x_off = x * QUAD_WIDTH + (QUAD_WIDTH / 2);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     {
       g_print ("test_tex_combine (%d, %d):\n%s\n", x, y, combine_string);
       g_print ("  texture 0 color = 0x%08lX\n", (unsigned long)tex0_color);
@@ -422,7 +422,7 @@ test_cogl_blend_strings (TestUtilsGTestFixture *fixture,
 
   paint (&state);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-color-mask.c

@@ -103,7 +103,7 @@ test_cogl_color_mask (TestUtilsGTestFixture *fixture,
 
   paint (&state);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-conform-main.c

@@ -5,116 +5,45 @@
 #include <glib.h>
 #include <locale.h>
 #include <stdlib.h>
+#include <string.h>
 
 #include "test-utils.h"
 
-#if 0
-void
-skip_init (TestUtilsGTestFixture *fixture,
-           const void *data)
-{
-  /* void */
-}
-
-static void
-skip_test (TestUtilsGTestFixture *fixture,
-           const void *data)
-{
-  /* void */
-}
-
-void
-skip_fini (TestUtilsGTestFixture *fixture,
-           const void *data)
-{
-  /* void */
-}
-#endif
-
-static void
-run_todo_test (TestUtilsGTestFixture *fixture,
-               void *data)
-{
-#ifdef G_OS_UNIX
-  TestUtilsSharedState *state = data;
-
-  if (g_test_trap_fork (0, G_TEST_TRAP_SILENCE_STDOUT | G_TEST_TRAP_SILENCE_STDERR))
-    {
-      state->todo_func (fixture, data);
-      exit (0);
-    }
-
-  g_test_trap_assert_failed ();
-#endif
-}
-
-void
-verify_failure (TestUtilsGTestFixture *fixture,
-                void *data)
-{
-  g_assert (FALSE);
-}
-
 static TestUtilsSharedState *shared_state = NULL;
 
-/* This is a bit of sugar for adding new conformance tests:
+/* A bit of sugar for adding new conformance tests */
- *
+#define ADD_TEST(FUNC, REQUIREMENTS)  G_STMT_START {      \
- * - It adds an extern function definition just to save maintaining a header
- *   that lists test entry points.
- * - It sets up callbacks for a fixture, which lets us share initialization
- *   code between tests. (see test-utils.c)
- * - It passes in a shared data pointer that is initialised once in main(),
- *   that gets passed to the fixture setup and test functions. (See the
- *   definition in test-utils.h)
- */
-#define ADD_TEST(NAMESPACE, FUNC)            G_STMT_START {             \
   extern void FUNC (TestUtilsGTestFixture *, void *);     \
-  g_test_add ("/conform" NAMESPACE "/" #FUNC,                           \
+  if (strcmp (#FUNC, argv[1]) == 0)                       \
-	      TestUtilsGTestFixture,                                    \
+    {                                                     \
-	      shared_state, /* data argument for test */                \
+      test_utils_init (shared_state, REQUIREMENTS);       \
-	      test_utils_init,                                          \
+      FUNC (NULL, shared_state);                          \
-	      (void *)(FUNC),                                           \
+      test_utils_fini (shared_state);                     \
-	      test_utils_fini);    } G_STMT_END
+      exit (0);                                           \
+    }                                                     \
+} G_STMT_END
 
-/* this is a macro that conditionally executes a test if CONDITION
+#define UNPORTED_TEST(FUNC)
- * evaluates to TRUE; otherwise, it will put the test under the
- * "/skip" namespace and execute a dummy function that will always
- * pass.
- */
-#define ADD_CONDITIONAL_TEST(CONDITION, NAMESPACE, FUNC)   G_STMT_START {   \
-  if (!(CONDITION)) {                                                       \
-    g_test_add ("/skipped" NAMESPACE "/" #FUNC,                             \
-                TestUtilsGTestFixture,                                      \
-                shared_state, /* data argument for test */                  \
-                skip_init,                                                  \
-                skip_test,                                                  \
-                skip_fini);                                                 \
-  } else { ADD_TEST (NAMESPACE, FUNC); }     } G_STMT_END
-
-#define ADD_TODO_TEST(NAMESPACE, FUNC)              G_STMT_START {          \
-   extern void FUNC (TestUtilsGTestFixture *, void *);                      \
-   shared_state->todo_func = FUNC;                                          \
-   g_test_add ("/todo" NAMESPACE "/" #FUNC,                                 \
-              TestUtilsGTestFixture,                                        \
-              shared_state,                                                 \
-              test_utils_init,                                              \
-              (void *)(run_todo_test),                                      \
-              test_utils_fini);    } G_STMT_END
-
-#define UNPORTED_TEST(NAMESPACE, FUNC)
-
-gchar *
-clutter_test_get_data_file (const gchar *filename)
-{
-  return g_build_filename (TESTS_DATADIR, filename, NULL);
-}
 
 int
 main (int argc, char **argv)
 {
-  g_test_init (&argc, &argv, NULL);
+  int i;
 
-  g_test_bug_base ("http://bugzilla.gnome.org/show_bug.cgi?id=%s");
+  if (argc != 2)
+    {
+      g_printerr ("usage %s UNIT_TEST\n", argv[0]);
+      exit (1);
+    }
+
+  /* Just for convenience in case people try passing the wrapper
+   * filenames for the UNIT_TEST argument we normalize '-' characters
+   * to '_' characters... */
+  for (i = 0; argv[1][i]; i++)
+    {
+      if (argv[1][i] == '-')
+        argv[1][i] = '_';
+    }
 
   /* Initialise the state you need to share with everything.
    */
@@ -124,56 +53,53 @@ main (int argc, char **argv)
 
   /* This file is run through a sed script during the make step so the
    * lines containing the tests need to be formatted on a single line
-   * each. To comment out a test use the SKIP or TODO macros. Using
+   * each.
-   * #if 0 would break the script. */
+   */
 
-  /* sanity check for the test suite itself */
+  UNPORTED_TEST (test_cogl_object);
-  ADD_TODO_TEST ("/suite", verify_failure);
+  UNPORTED_TEST (test_cogl_fixed);
+  UNPORTED_TEST (test_cogl_materials);
+  ADD_TEST (test_cogl_pipeline_user_matrix, 0);
+  ADD_TEST (test_cogl_blend_strings, 0);
+  UNPORTED_TEST (test_cogl_premult);
+  UNPORTED_TEST (test_cogl_readpixels);
+  ADD_TEST (test_cogl_path, 0);
+  ADD_TEST (test_cogl_depth_test, 0);
+  ADD_TEST (test_cogl_color_mask, 0);
+  ADD_TEST (test_cogl_backface_culling, TEST_REQUIREMENT_NPOT);
 
-  UNPORTED_TEST ("/cogl", test_cogl_object);
+  ADD_TEST (test_cogl_sparse_pipeline, 0);
-  UNPORTED_TEST ("/cogl", test_cogl_fixed);
-  UNPORTED_TEST ("/cogl", test_cogl_materials);
-  ADD_TEST ("/cogl", test_cogl_pipeline_user_matrix);
-  ADD_TEST ("/cogl", test_cogl_blend_strings);
-  UNPORTED_TEST ("/cogl", test_cogl_premult);
-  UNPORTED_TEST ("/cogl", test_cogl_readpixels);
-  ADD_TEST ("/cogl", test_cogl_path);
-  ADD_TEST ("/cogl", test_cogl_depth_test);
-  ADD_TEST ("/cogl", test_cogl_color_mask);
-  ADD_TEST ("/cogl", test_cogl_backface_culling);
 
-  ADD_TEST ("/cogl/pipeline", test_cogl_sparse_pipeline);
+  UNPORTED_TEST (test_cogl_npot_texture);
+  UNPORTED_TEST (test_cogl_multitexture);
+  UNPORTED_TEST (test_cogl_texture_mipmaps);
+  ADD_TEST (test_cogl_sub_texture, TEST_REQUIREMENT_GL);
+  UNPORTED_TEST (test_cogl_pixel_array);
+  UNPORTED_TEST (test_cogl_texture_rectangle);
+  ADD_TEST (test_cogl_texture_3d, 0);
+  ADD_TEST (test_cogl_wrap_modes, 0);
+  UNPORTED_TEST (test_cogl_texture_pixmap_x11);
+  UNPORTED_TEST (test_cogl_texture_get_set_data);
+  UNPORTED_TEST (test_cogl_atlas_migration);
 
-  UNPORTED_TEST ("/cogl/texture", test_cogl_npot_texture);
+  UNPORTED_TEST (test_cogl_vertex_buffer_contiguous);
-  UNPORTED_TEST ("/cogl/texture", test_cogl_multitexture);
+  UNPORTED_TEST (test_cogl_vertex_buffer_interleved);
-  UNPORTED_TEST ("/cogl/texture", test_cogl_texture_mipmaps);
+  UNPORTED_TEST (test_cogl_vertex_buffer_mutability);
-  ADD_TEST ("/cogl/texture", test_cogl_sub_texture);
-  UNPORTED_TEST ("/cogl/texture", test_cogl_pixel_array);
-  UNPORTED_TEST ("/cogl/texture", test_cogl_texture_rectangle);
-  ADD_TEST ("/cogl/texture", test_cogl_texture_3d);
-  ADD_TEST ("/cogl/texture", test_cogl_wrap_modes);
-  UNPORTED_TEST ("/cogl/texture", test_cogl_texture_pixmap_x11);
-  UNPORTED_TEST ("/cogl/texture", test_cogl_texture_get_set_data);
-  UNPORTED_TEST ("/cogl/texture", test_cogl_atlas_migration);
 
-  UNPORTED_TEST ("/cogl/vertex-buffer", test_cogl_vertex_buffer_contiguous);
+  ADD_TEST (test_cogl_primitive, 0);
-  UNPORTED_TEST ("/cogl/vertex-buffer", test_cogl_vertex_buffer_interleved);
-  UNPORTED_TEST ("/cogl/vertex-buffer", test_cogl_vertex_buffer_mutability);
 
-  ADD_TEST ("/cogl/vertex-array", test_cogl_primitive);
+  ADD_TEST (test_cogl_just_vertex_shader, 0);
+  ADD_TEST (test_cogl_pipeline_uniforms, 0);
+  ADD_TEST (test_cogl_snippets, 0);
+  ADD_TEST (test_cogl_custom_attributes, 0);
 
-  ADD_TEST ("/cogl/shaders", test_cogl_just_vertex_shader);
+  ADD_TEST (test_cogl_bitmask, 0);
-  ADD_TEST ("/cogl/shaders", test_cogl_pipeline_uniforms);
-  ADD_TEST ("/cogl/shaders", test_cogl_snippets);
-  ADD_TEST ("/cogl/shaders", test_cogl_custom_attributes);
 
-  ADD_TEST ("/cogl/internal/bitmask", test_cogl_bitmask);
+  ADD_TEST (test_cogl_offscreen, 0);
 
-  ADD_TEST ("/cogl", test_cogl_offscreen);
+  UNPORTED_TEST (test_cogl_viewport);
 
-  /* left to the end because they aren't currently very orthogonal and tend to
+  g_printerr ("Unknown test name \"%s\"\n", argv[1]);
-   * break subsequent tests! */
-  UNPORTED_TEST ("/cogl", test_cogl_viewport);
 
-  return g_test_run ();
+  return 1;
 }

tests/conform/test-custom-attributes.c

@@ -299,9 +299,9 @@ test_cogl_custom_attributes (TestUtilsGTestFixture *fixture,
       cogl_object_unref (state.pipeline);
       cogl_object_unref (snippet);
 
-      if (g_test_verbose ())
+      if (cogl_test_verbose ())
         g_print ("OK\n");
     }
-  else if (g_test_verbose ())
+  else if (cogl_test_verbose ())
     g_print ("Skipping\n");
 }

tests/conform/test-depth-test.c

@@ -243,7 +243,7 @@ test_cogl_depth_test (TestUtilsGTestFixture *fixture,
 
   paint (&state);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-just-vertex-shader.c

@@ -196,10 +196,10 @@ test_cogl_just_vertex_shader (TestUtilsGTestFixture *fixture,
       paint (&state);
       validate_result ();
 
-      if (g_test_verbose ())
+      if (cogl_test_verbose ())
         g_print ("OK\n");
     }
-  else if (g_test_verbose ())
+  else if (cogl_test_verbose ())
     g_print ("Skipping\n");
 }
 

tests/conform/test-launcher.sh.in

@@ -1,28 +1,33 @@
 #!/bin/sh
 
-UNIT_TEST_PATH=$1
+UNIT_TEST=$1
 shift
 
-test -z ${UNIT_TEST_PATH} && {
+test -z ${UNIT_TEST} && {
-        echo "Usage: $0 /path/to/unit_test"
+        echo "Usage: $0 UNIT_TEST"
         exit 1
 }
 
-UNIT_TEST=`basename ${UNIT_TEST_PATH}`
+UNIT_TEST=`echo $UNIT_TEST|sed 's/-/_/g'`
 
-echo "Running: ./test-conformance -p ${UNIT_TEST_PATH} $@"
+echo "Running: ./test-conformance ${UNIT_TEST} $@"
 echo ""
-@abs_builddir@/test-conformance -p ${UNIT_TEST_PATH} "$@"
+if test -f @abs_builddir@/test-conformance; then
+  TEST_CONFORMANCE=@abs_builddir@/test-conformance
+elif test -f @abs_builddir@/test-conformance.exe; then
+  TEST_CONFORMANCE=@abs_builddir@/test-conformance.exe
+fi
+COGL_TEST_VERBOSE=1 $TEST_CONFORMANCE ${UNIT_TEST} "$@"
 exit_val=$?
 
 echo ""
 echo "NOTE: For debugging purposes, you can run this single test as follows:"
 echo "$ libtool --mode=execute \\"
 echo "          gdb --eval-command=\"b ${UNIT_TEST}\" \\"
-echo "          --args ./test-conformance -p ${UNIT_TEST_PATH}"
+echo "          --args ./test-conformance ${UNIT_TEST}"
 echo "or:"
 echo "$ env G_SLICE=always-malloc \\"
 echo "  libtool --mode=execute \\"
-echo "          valgrind ./test-conformance -p ${UNIT_TEST_PATH}"
+echo "          valgrind ./test-conformance ${UNIT_TEST}"
 
 exit $exit_val

tests/conform/test-materials.c

@@ -247,7 +247,7 @@ test_cogl_materials (TestUtilsGTestFixture *fixture,
 
   g_source_remove (idle_source);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-multitexture.c

@@ -201,6 +201,6 @@ test_cogl_multitexture (TestUtilsGTestFixture *fixture,
 
   g_source_remove (idle_source);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }

tests/conform/test-npot-texture.c

@@ -171,7 +171,7 @@ make_texture (void)
 
   g_free (tex_data);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     {
       if (cogl_texture_is_sliced (tex))
         g_print ("Texture is sliced\n");
@@ -196,7 +196,7 @@ test_cogl_npot_texture (TestUtilsGTestFixture *fixture,
   ClutterActor *group;
   unsigned int idle_source;
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     {
       if (cogl_features_available (COGL_FEATURE_TEXTURE_NPOT))
         g_print ("NPOT textures are supported\n");
@@ -230,7 +230,7 @@ test_cogl_npot_texture (TestUtilsGTestFixture *fixture,
 
   cogl_handle_unref (state.texture);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-object.c

@@ -80,7 +80,7 @@ test_cogl_object (TestUtilsGTestFixture *fixture,
   g_assert_cmpint (destroy1_count, ==, 2);
   g_assert_cmpint (destroy2_count, ==, 1);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-offscreen.c

@@ -194,6 +194,6 @@ test_cogl_offscreen (TestUtilsGTestFixture *fixture,
   test_paint (&state);
   test_flush (&state);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }

tests/conform/test-path.c

@@ -201,7 +201,7 @@ test_cogl_path (TestUtilsGTestFixture *fixture,
   paint (&state);
   validate_result ();
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-pipeline-uniforms.c

@@ -424,9 +424,9 @@ test_cogl_pipeline_uniforms (TestUtilsGTestFixture *fixture,
 
       destroy_state (&state);
 
-      if (g_test_verbose ())
+      if (cogl_test_verbose ())
         g_print ("OK\n");
     }
-  else if (g_test_verbose ())
+  else if (cogl_test_verbose ())
     g_print ("Skipping\n");
 }

tests/conform/test-pipeline-user-matrix.c

@@ -138,6 +138,6 @@ test_cogl_pipeline_user_matrix (TestUtilsGTestFixture *fixture,
   paint (&state);
   validate_result (&state);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }

tests/conform/test-pixel-buffer.c

@@ -324,7 +324,7 @@ test_cogl_pixel_array (TestUtilsGTestFixture *fixture,
                              (ClutterCallback) clutter_actor_destroy,
                              NULL);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-premult.c

@@ -98,7 +98,7 @@ on_paint (ClutterActor *actor, TestState *state)
 
   /* If the user explicitly specifies an unmultiplied internal format then
    * Cogl shouldn't automatically premultiply the given texture data... */
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("make_texture (0xff00ff80, "
                             "src = RGBA_8888, internal = RGBA_8888)\n");
   tex = make_texture (0xff00ff80,
@@ -111,7 +111,7 @@ on_paint (ClutterActor *actor, TestState *state)
   /* If the user explicitly requests a premultiplied internal format and
    * gives unmultiplied src data then Cogl should always premultiply that
    * for us */
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("make_texture (0xff00ff80, "
                             "src = RGBA_8888, internal = RGBA_8888_PRE)\n");
   tex = make_texture (0xff00ff80,
@@ -125,7 +125,7 @@ on_paint (ClutterActor *actor, TestState *state)
    * by default Cogl should premultiply the given texture data...
    * (In the future there will be additional Cogl API to control this
    *  behaviour) */
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("make_texture (0xff00ff80, "
                             "src = RGBA_8888, internal = ANY)\n");
   tex = make_texture (0xff00ff80,
@@ -138,7 +138,7 @@ on_paint (ClutterActor *actor, TestState *state)
   /* If the user requests a premultiplied internal texture format and supplies
    * premultiplied source data, Cogl should never modify that source data...
    */
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("make_texture (0x80008080, "
                             "src = RGBA_8888_PRE, "
                             "internal = RGBA_8888_PRE)\n");
@@ -152,7 +152,7 @@ on_paint (ClutterActor *actor, TestState *state)
   /* If the user requests an unmultiplied internal texture format, but
    * supplies premultiplied source data, then Cogl should always
    * un-premultiply the source data... */
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("make_texture (0x80008080, "
                             "src = RGBA_8888_PRE, internal = RGBA_8888)\n");
   tex = make_texture (0x80008080,
@@ -166,7 +166,7 @@ on_paint (ClutterActor *actor, TestState *state)
    * source data then by default Cogl shouldn't modify the source data...
    * (In the future there will be additional Cogl API to control this
    *  behaviour) */
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("make_texture (0x80008080, "
                             "src = RGBA_8888_PRE, internal = ANY)\n");
   tex = make_texture (0x80008080,
@@ -180,13 +180,13 @@ on_paint (ClutterActor *actor, TestState *state)
    * Test cogl_texture_set_region() ....
    */
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("make_texture (0xDEADBEEF, "
                             "src = RGBA_8888, internal = RGBA_8888)\n");
   tex = make_texture (0xDEADBEEF,
                       COGL_PIXEL_FORMAT_RGBA_8888, /* src format */
                       COGL_PIXEL_FORMAT_RGBA_8888); /* internal format */
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("set_region (0xff00ff80, RGBA_8888)\n");
   tex_data = gen_tex_data (0xff00ff80);
   cogl_texture_set_region (tex,
@@ -204,13 +204,13 @@ on_paint (ClutterActor *actor, TestState *state)
   /* Updating a texture region for an unmultiplied texture using premultiplied
    * region data should result in Cogl unmultiplying the given region data...
    */
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("make_texture (0xDEADBEEF, "
                             "src = RGBA_8888, internal = RGBA_8888)\n");
   tex = make_texture (0xDEADBEEF,
                       COGL_PIXEL_FORMAT_RGBA_8888, /* src format */
                       COGL_PIXEL_FORMAT_RGBA_8888); /* internal format */
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("set_region (0x80008080, RGBA_8888_PRE)\n");
   tex_data = gen_tex_data (0x80008080);
   cogl_texture_set_region (tex,
@@ -226,14 +226,14 @@ on_paint (ClutterActor *actor, TestState *state)
                  0xff00ff80); /* expected */
 
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("make_texture (0xDEADBEEF, "
                             "src = RGBA_8888_PRE, "
                             "internal = RGBA_8888_PRE)\n");
   tex = make_texture (0xDEADBEEF,
                       COGL_PIXEL_FORMAT_RGBA_8888_PRE, /* src format */
                       COGL_PIXEL_FORMAT_RGBA_8888_PRE); /* internal format */
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("set_region (0x80008080, RGBA_8888_PRE)\n");
   tex_data = gen_tex_data (0x80008080);
   cogl_texture_set_region (tex,
@@ -252,14 +252,14 @@ on_paint (ClutterActor *actor, TestState *state)
   /* Updating a texture region for a premultiplied texture using unmultiplied
    * region data should result in Cogl premultiplying the given region data...
    */
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("make_texture (0xDEADBEEF, "
                             "src = RGBA_8888_PRE, "
                             "internal = RGBA_8888_PRE)\n");
   tex = make_texture (0xDEADBEEF,
                       COGL_PIXEL_FORMAT_RGBA_8888_PRE, /* src format */
                       COGL_PIXEL_FORMAT_RGBA_8888_PRE); /* internal format */
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("set_region (0xff00ff80, RGBA_8888)\n");
   tex_data = gen_tex_data (0xff00ff80);
   cogl_texture_set_region (tex,
@@ -322,7 +322,7 @@ test_cogl_premult (TestUtilsGTestFixture *fixture,
 
   g_source_remove (idle_source);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-primitive.c

@@ -332,6 +332,6 @@ test_cogl_primitive (TestUtilsGTestFixture *fixture,
   test_paint (&state);
   test_copy (&state);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }

tests/conform/test-readpixels.c

@@ -172,7 +172,7 @@ test_cogl_readpixels (TestUtilsGTestFixture *fixture,
                              (ClutterCallback) clutter_actor_destroy,
                              NULL);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-snippets.c

@@ -734,9 +734,9 @@ test_cogl_snippets (TestUtilsGTestFixture *fixture,
 
       run_tests (&state);
 
-      if (g_test_verbose ())
+      if (cogl_test_verbose ())
         g_print ("OK\n");
     }
-  else if (g_test_verbose ())
+  else if (cogl_test_verbose ())
     g_print ("Skipping\n");
 }

tests/conform/test-sparse-pipeline.c

@@ -64,7 +64,7 @@ test_cogl_sparse_pipeline (TestUtilsGTestFixture *fixture,
      whether using an attribute with sparse texture coordinates will
      work */
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-sub-texture.c

@@ -317,7 +317,7 @@ test_cogl_sub_texture (TestUtilsGTestFixture *fixture,
 
   cogl_object_unref (state.tex);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-texture-3d.c

@@ -286,10 +286,10 @@ test_cogl_texture_3d (TestUtilsGTestFixture *fixture,
 
       test_multi_texture (&state);
 
-      if (g_test_verbose ())
+      if (cogl_test_verbose ())
         g_print ("OK\n");
     }
-  else if (g_test_verbose ())
+  else if (cogl_test_verbose ())
     g_print ("Skipping\n");
 }
 

tests/conform/test-texture-get-set-data.c

@@ -161,6 +161,6 @@ test_cogl_texture_get_set_data (TestUtilsGTestFixture *fixture,
 
   g_signal_handler_disconnect (stage, paint_handler);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }

tests/conform/test-texture-mipmaps.c

@@ -131,6 +131,6 @@ test_cogl_texture_mipmaps (TestUtilsGTestFixture *fixture,
 
   g_source_remove (idle_source);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }

tests/conform/test-texture-pixmap-x11.c

@@ -232,12 +232,12 @@ test_cogl_texture_pixmap_x11 (TestUtilsGTestFixture *fixture,
 
   XFreePixmap (state.display, state.pixmap);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 
 #else /* COGL_HAS_XLIB */
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
    g_print ("Skipping\n");
 
 #endif /* COGL_HAS_XLIB */

tests/conform/test-texture-rectangle.c

@@ -267,10 +267,10 @@ test_cogl_texture_rectangle (TestUtilsGTestFixture *fixture,
       g_source_remove (idle_source);
       g_signal_handler_disconnect (state.stage, paint_handler);
 
-      if (g_test_verbose ())
+      if (cogl_test_verbose ())
         g_print ("OK\n");
     }
-  else if (g_test_verbose ())
+  else if (cogl_test_verbose ())
     g_print ("Skipping\n");
 }
 

tests/conform/test-utils.c

@@ -1,3 +1,4 @@
+#define COGL_ENABLE_EXPERIMENTAL_2_0_API
 #include <cogl/cogl.h>
 #include <stdlib.h>
 
@@ -6,14 +7,18 @@
 #define FB_WIDTH 512
 #define FB_HEIGHT 512
 
+static gboolean cogl_test_is_verbose;
+
 void
-test_utils_init (TestUtilsGTestFixture *fixture,
+test_utils_init (TestUtilsSharedState *state,
-                 const void *data)
+                 TestRequirement requirements)
 {
-  TestUtilsSharedState *state = (TestUtilsSharedState *)data;
   static int counter = 0;
   GError *error = NULL;
   CoglOnscreen *onscreen = NULL;
+  CoglDisplay *display;
+  CoglRenderer *renderer;
+  gboolean missing_requirement = FALSE;
 
   if (counter != 0)
     g_critical ("We don't support running more than one test at a time\n"
@@ -24,12 +29,45 @@ test_utils_init (TestUtilsGTestFixture *fixture,
                 "$ make test-report");
   counter++;
 
+  if (g_getenv ("COGL_TEST_VERBOSE") || g_getenv ("V"))
+    cogl_test_is_verbose = TRUE;
+
+  if (g_getenv ("G_DEBUG"))
+    {
+      char *debug = g_strconcat (g_getenv ("G_DEBUG"), ",fatal-warnings", NULL);
+      g_setenv ("G_DEBUG", debug, TRUE);
+      g_free (debug);
+    }
+  else
+    g_setenv ("G_DEBUG", "fatal-warnings", TRUE);
+
   g_setenv ("COGL_X11_SYNC", "1", 0);
 
   state->ctx = cogl_context_new (NULL, &error);
   if (!state->ctx)
     g_critical ("Failed to create a CoglContext: %s", error->message);
 
+  display = cogl_context_get_display (state->ctx);
+  renderer = cogl_display_get_renderer (display);
+
+  if (requirements & TEST_REQUIREMENT_GL &&
+      cogl_renderer_get_driver (renderer) != COGL_DRIVER_GL)
+    {
+      missing_requirement = TRUE;
+    }
+
+  if (requirements & TEST_REQUIREMENT_NPOT &&
+      !cogl_has_feature (state->ctx, COGL_FEATURE_ID_TEXTURE_NPOT))
+    {
+      missing_requirement = TRUE;
+    }
+
+  if (requirements & TEST_REQUIREMENT_TEXTURE_3D &&
+      !cogl_has_feature (state->ctx, COGL_FEATURE_ID_TEXTURE_3D))
+    {
+      missing_requirement = TRUE;
+    }
+
   if (getenv  ("COGL_TEST_ONSCREEN"))
     {
       onscreen = cogl_onscreen_new (state->ctx, 640, 480);
@@ -62,14 +100,14 @@ test_utils_init (TestUtilsGTestFixture *fixture,
                             0, 0, 0, 1);
 
   cogl_push_framebuffer (state->fb);
+
+  if (missing_requirement)
+    g_print ("WARNING: Missing required feature[s] for this test\n");
 }
 
 void
-test_utils_fini (TestUtilsGTestFixture *fixture,
+test_utils_fini (TestUtilsSharedState *state)
-                 const void *data)
 {
-  const TestUtilsSharedState *state = (TestUtilsSharedState *)data;
-
   cogl_pop_framebuffer ();
 
   if (state->fb)
@@ -169,3 +207,9 @@ test_utils_create_color_texture (CoglContext *context,
 
   return COGL_TEXTURE (tex_2d);
 }
+
+gboolean
+cogl_test_verbose (void)
+{
+  return cogl_test_is_verbose;
+}

tests/conform/test-utils.h

@@ -1,20 +1,17 @@
 #ifndef _TEST_UTILS_H_
 #define _TEST_UTILS_H_
 
-/* This fixture structure is allocated by glib, and before running
+typedef enum _TestRequirement
- * each test we get a callback to initialize it.
- *
- * Actually we don't use this currently, we instead manage our own
- * TestUtilsSharedState structure which also gets passed as a private
- * data argument to the same initialization callback. The advantage of
- * allocating our own shared state structure is that we can put data
- * in it before we start running anything.
- */
-typedef struct _TestUtilsGTestFixture
 {
-  /**/
+  TEST_REQUIREMENT_GL         = 1<<0,
-  int dummy;
+  TEST_REQUIREMENT_NPOT       = 1<<2,
-} TestUtilsGTestFixture;
+  TEST_REQUIREMENT_TEXTURE_3D = 1<<3
+} TestRequirement;
+
+/* For compatability since we used to use the glib gtester
+ * infrastructure and all our unit tests have an entry
+ * point with a first argument of this type... */
+typedef struct _TestUtilsGTestFixture TestUtilsGTestFixture;
 
 /* Stuff you put in here is setup once in main() and gets passed around to
  * all test functions and fixture setup/teardown functions in the data
@@ -24,19 +21,16 @@ typedef struct _TestUtilsSharedState
   int    *argc_addr;
   char ***argv_addr;
 
-  void (* todo_func) (TestUtilsGTestFixture *, void *data);
-
   CoglContext *ctx;
   CoglFramebuffer *fb;
 } TestUtilsSharedState;
 
 void
-test_utils_init (TestUtilsGTestFixture *fixture,
+test_utils_init (TestUtilsSharedState *state,
-                 const void *data);
+                 TestRequirement requirements);
 
 void
-test_utils_fini (TestUtilsGTestFixture *fixture,
+test_utils_fini (TestUtilsSharedState *state);
-                 const void *data);
 
 /*
  * test_utils_check_pixel:
@@ -112,4 +106,11 @@ CoglTexture *
 test_utils_create_color_texture (CoglContext *context,
                                  guint32 color);
 
+/* cogl_test_verbose:
+ *
+ * Queries if the user asked for verbose output or not.
+ */
+gboolean
+cogl_test_verbose (void);
+
 #endif /* _TEST_UTILS_H_ */

tests/conform/test-vertex-buffer-contiguous.c

@@ -29,7 +29,7 @@ validate_result (TestState *state)
   GLubyte pixel[4];
   GLint y_off = 90;
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("y_off = %d\n", y_off);
 
   /* NB: We ignore the alpha, since we don't know if our render target is
@@ -44,7 +44,7 @@ validate_result (TestState *state)
                     COGL_READ_PIXELS_COLOR_BUFFER,
                     COGL_PIXEL_FORMAT_RGBA_8888_PRE,
                     pixel);
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("pixel 0 = %x, %x, %x\n", pixel[RED], pixel[GREEN], pixel[BLUE]);
   g_assert (pixel[RED] == 0 && pixel[GREEN] == 0 && pixel[BLUE] != 0);
 
@@ -53,7 +53,7 @@ validate_result (TestState *state)
                     COGL_READ_PIXELS_COLOR_BUFFER,
                     COGL_PIXEL_FORMAT_RGBA_8888_PRE,
                     pixel);
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("pixel 1 = %x, %x, %x\n", pixel[RED], pixel[GREEN], pixel[BLUE]);
   g_assert (pixel[RED] != 0 && pixel[GREEN] == 0 && pixel[BLUE] == 0);
 
@@ -62,7 +62,7 @@ validate_result (TestState *state)
                     COGL_READ_PIXELS_COLOR_BUFFER,
                     COGL_PIXEL_FORMAT_RGBA_8888_PRE,
                     pixel);
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("pixel 2 = %x, %x, %x\n", pixel[RED], pixel[GREEN], pixel[BLUE]);
   g_assert (pixel[RED] == 0 && pixel[GREEN] == 0 && pixel[BLUE] != 0);
 
@@ -71,7 +71,7 @@ validate_result (TestState *state)
                     COGL_READ_PIXELS_COLOR_BUFFER,
                     COGL_PIXEL_FORMAT_RGBA_8888_PRE,
                     pixel);
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("pixel 3 = %x, %x, %x\n", pixel[RED], pixel[GREEN], pixel[BLUE]);
   g_assert (pixel[GREEN] > pixel[RED] && pixel[GREEN] > pixel[BLUE]);
 
@@ -80,7 +80,7 @@ validate_result (TestState *state)
                     COGL_READ_PIXELS_COLOR_BUFFER,
                     COGL_PIXEL_FORMAT_RGBA_8888_PRE,
                     pixel);
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("pixel 4 = %x, %x, %x\n", pixel[RED], pixel[GREEN], pixel[BLUE]);
   g_assert (pixel[RED] > pixel[GREEN] && pixel[RED] > pixel[BLUE]);
 
@@ -251,7 +251,7 @@ test_cogl_vertex_buffer_contiguous (TestUtilsGTestFixture *fixture,
 
   g_source_remove (idle_source);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-vertex-buffer-interleved.c

@@ -48,7 +48,7 @@ validate_result (TestState *state)
                     COGL_READ_PIXELS_COLOR_BUFFER,
                     COGL_PIXEL_FORMAT_RGBA_8888_PRE,
                     pixel);
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("pixel 0 = %x, %x, %x\n", pixel[RED], pixel[GREEN], pixel[BLUE]);
   g_assert (pixel[RED] == 0 && pixel[GREEN] == 0 && pixel[BLUE] != 0);
 
@@ -156,7 +156,7 @@ test_cogl_vertex_buffer_interleved (TestUtilsGTestFixture *fixture,
 
   g_source_remove (idle_source);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-vertex-buffer-mutability.c

@@ -35,7 +35,7 @@ validate_result (TestState *state)
                     COGL_READ_PIXELS_COLOR_BUFFER,
                     COGL_PIXEL_FORMAT_RGBA_8888_PRE,
                     pixel);
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("pixel 0 = %x, %x, %x\n", pixel[RED], pixel[GREEN], pixel[BLUE]);
   g_assert (pixel[RED] != 0 && pixel[GREEN] == 0 && pixel[BLUE] == 0);
 
@@ -44,7 +44,7 @@ validate_result (TestState *state)
                     COGL_READ_PIXELS_COLOR_BUFFER,
                     COGL_PIXEL_FORMAT_RGBA_8888_PRE,
                     pixel);
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("pixel 1 = %x, %x, %x\n", pixel[RED], pixel[GREEN], pixel[BLUE]);
   g_assert (pixel[RED] == 0 && pixel[GREEN] != 0 && pixel[BLUE] == 0);
 
@@ -192,7 +192,7 @@ test_cogl_vertex_buffer_mutability (TestUtilsGTestFixture *fixture,
 
   g_source_remove (idle_source);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-viewport.c

@@ -410,7 +410,7 @@ test_cogl_viewport (TestUtilsGTestFixture *fixture,
                              (ClutterCallback) clutter_actor_destroy,
                              NULL);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }
 

tests/conform/test-wrap-modes.c

@@ -284,6 +284,6 @@ test_cogl_wrap_modes (TestUtilsGTestFixture *fixture,
 
   paint (&state);
 
-  if (g_test_verbose ())
+  if (cogl_test_verbose ())
     g_print ("OK\n");
 }