mutter/tests/conform/Makefile.am
Robert Bragg bbcbece6c9 tests: Improve thoroughness and reporting of make test
This patch reworks our conformance testing framework because it seems
that glib's gtesting framework isn't really well suited to our use case.
For example we weren't able to test windows builds given the way we
were using it and also for each test we'd like to repeat the test
with several different environments so we can test important driver and
feature combinations.

This patch instead switches away to a simplified but custom approach for
running our unit tests. We hope that having a more bespoke setup will
enable us to easily extend it to focus on the details important to us.

Notable changes with this new approach are:

We can now run 'make test' for our mingw windows builds.

We've got rid of all the test-*report* make rules and we're just left
with 'make test'

'make test' now runs each test several times with different driver and
feature combinations checking the result for each run. 'make test' will
then output a concise table of all of the results.

The combinations tested are:
- OpenGL Fixed Function
- OpenGL ARBfp
- OpenGL GLSL
- OpenGL No NPOT texture support
- OpenGLES 2.0
- OpenGLES 2.0 No NPOT texture support

Reviewed-by: Neil Roberts <neil@linux.intel.com>
2012-02-24 14:42:31 +00:00

168 lines
5.2 KiB
Makefile

include $(top_srcdir)/build/autotools/Makefile.am.silent
NULL =
noinst_PROGRAMS = test-conformance
common_sources = \
test-utils.h \
test-utils.c \
test-conform-main.c \
$(NULL)
unported_test_sources = \
test-fixed.c \
test-materials.c \
test-viewport.c \
test-multitexture.c \
test-npot-texture.c \
test-object.c \
test-pixel-buffer.c \
test-premult.c \
test-readpixels.c \
test-texture-get-set-data.c \
test-texture-mipmaps.c \
test-texture-pixmap-x11.c \
test-texture-rectangle.c \
test-atlas-migration.c \
test-vertex-buffer-contiguous.c \
test-vertex-buffer-interleved.c \
test-vertex-buffer-mutability.c \
$(NULL)
test_sources = \
test-bitmask.c \
test-blend-strings.c \
test-depth-test.c \
test-color-mask.c \
test-backface-culling.c \
test-just-vertex-shader.c \
test-path.c \
test-pipeline-user-matrix.c \
test-pipeline-uniforms.c \
test-snippets.c \
test-wrap-modes.c \
test-sub-texture.c \
test-custom-attributes.c \
test-offscreen.c \
test-primitive.c \
test-texture-3d.c \
test-sparse-pipeline.c \
$(NULL)
test_conformance_SOURCES = $(common_sources) $(test_sources)
if OS_WIN32
SHEXT =
else
SHEXT = $(EXEEXT)
endif
# For convenience, this provides a way to easily run individual unit tests:
.PHONY: wrappers clean-wrappers
wrappers: stamp-test-conformance
@true
stamp-test-conformance: Makefile $(srcdir)/test-conform-main.c
@mkdir -p wrappers
@sed -n -e 's/^ \{1,\}ADD_TEST *( *\([a-zA-Z0-9_]\{1,\}\).*/\1/p' $(srcdir)/test-conform-main.c > unit-tests
@chmod +x test-launcher.sh
@( echo "/stamp-test-conformance" ; \
echo "/test-conformance$(EXEEXT)" ; \
echo "*.o" ; \
echo ".gitignore" ; \
echo "unit-tests" ; ) > .gitignore
@for i in `cat unit-tests`; \
do \
unit=`basename $$i | sed -e s/_/-/g`; \
echo " GEN $$unit"; \
( echo "#!/bin/sh" ; echo "$(abs_builddir)/test-launcher.sh '$$i' \"\$$@\"" ) > $$unit$(SHEXT) ; \
chmod +x $$unit$(SHEXT); \
echo "/$$unit$(SHEXT)" >> .gitignore; \
done \
&& echo timestamp > $(@F)
clean-wrappers:
@for i in `cat unit-tests`; \
do \
unit=`basename $$i | sed -e s/_/-/g`; \
echo " RM $$unit"; \
rm -f $$unit$(SHEXT) ; \
done \
&& rm -f unit-tests \
&& rm -f stamp-test-conformance
# NB: BUILT_SOURCES here a misnomer. We aren't building source, just inserting
# a phony rule that will generate symlink scripts for running individual tests
BUILT_SOURCES = wrappers
# The include of the $(buildir)/cogl directory here is to make it so
# that tests that directly include Cogl source code for whitebox
# testing (such as test-bitmask) will still compile
INCLUDES = \
-I$(top_srcdir) \
-I$(top_builddir)/cogl
test_conformance_CPPFLAGS = \
-DCOGL_ENABLE_EXPERIMENTAL_API \
-DCOGL_DISABLE_DEPRECATED \
-DTESTS_DATADIR=\""$(top_srcdir)/tests/data"\" \
-DCLUTTER_COMPILATION
test_conformance_CFLAGS = -g3 -O0 $(COGL_DEP_CFLAGS) $(COGL_EXTRA_CFLAGS)
test_conformance_LDADD = $(COGL_DEP_LIBS) $(top_builddir)/cogl/libcogl.la
test_conformance_LDFLAGS = -export-dynamic
test: wrappers
@$(top_srcdir)/tests/conform/run-tests.sh $(abs_builddir)/config.env
# XXX: we could prevent the conformance test suite from running
# by simply defining this variable conditionally
TEST_PROGS = test-conformance
.PHONY: test
EXTRA_DIST = test-launcher.sh.in run-tests.sh
DISTCLEANFILES = test-launcher.sh .gitignore
dist-hook: ../../build/win32/vs9/test-conformance-cogl.vcproj ../../build/win32/vs10/test-conformance-cogl.vcxproj ../../build/win32/vs10/test-conformance-cogl.vcxproj.filters
../../build/win32/vs9/test-conformance-cogl.vcproj: $(top_srcdir)/build/win32/vs9/test-conformance-cogl.vcprojin
for F in $(test_conformance_SOURCES); do \
case $$F in \
*.c) echo ' <File RelativePath="..\..\..\tests\conform\'$$F'" />' \
;; \
esac; \
done >testconformance.sourcefiles
$(CPP) -P - <$(top_srcdir)/build/win32/vs9/test-conformance-cogl.vcprojin >$@
rm testconformance.sourcefiles
../../build/win32/vs10/test-conformance-cogl.vcxproj: $(top_srcdir)/build/win32/vs10/test-conformance-cogl.vcxprojin
for F in $(test_conformance_SOURCES); do \
case $$F in \
*.c) echo ' <ClCompile Include="..\..\..\tests\conform\'$$F'" />' \
;; \
esac; \
done >testconformance.vs10.sourcefiles
$(CPP) -P - <$(top_srcdir)/build/win32/vs10/test-conformance-cogl.vcxprojin >$@
rm testconformance.vs10.sourcefiles
../../build/win32/vs10/test-conformance-cogl.vcxproj.filters: $(top_srcdir)/build/win32/vs10/test-conformance-cogl.vcxproj.filtersin
for F in $(test_conformance_SOURCES); do \
case $$F in \
*.c) echo ' <ClCompile Include="..\..\..\tests\conform\'$$F'"><Filter>Sources</Filter></ClCompile>' \
;; \
esac; \
done >testconformance.vs10.sourcefiles.filters
$(CPP) -P - <$(top_srcdir)/build/win32/vs10/test-conformance-cogl.vcxproj.filtersin >$@
rm testconformance.vs10.sourcefiles.filters
# Let the VS9/VS10 Project files be cleared out before they are re-expanded...
DISTCLEANFILES += ../../build/win32/vs9/test-conformance-cogl.vcproj \
../../build/win32/vs10/test-conformance-cogl.vcxproj \
../../build/win32/vs10/test-conformance-cogl.vcxproj.filters
# we override the clean-generic target to clean up the wrappers so
# we cannot use CLEANFILES
clean-generic: clean-wrappers