conform: Make gtester run a list of wrappers

Instead of asking gtester to run ./test-conformance directly we now tell
it to run a list of wrapper scripts. This results in each test being
spawned in a separate process avoiding leakage of state between tests
which has been a big problem with the conformance tests for quite a
while now.

tests/conform/Makefile.am

@@ -60,12 +60,15 @@ UNIT_TESTS = `./test-conformance -l -m thorough | $(GREP) '^/'`
 wrappers: stamp-test-conformance
 	@true
 stamp-test-conformance: test-conformance$(EXEEXT)
+	@mkdir -p wrappers
 	@for i in $(UNIT_TESTS); \
 	do \
 		unit=`basename $$i | sed -e s/_/-/g`; \
 		echo "  GEN    $$unit"; \
-		( echo "#!/bin/sh" ; echo "$(top_srcdir)/tests/conform/test-launcher.sh '$$i'" ) > $$unit$(EXEEXT) ; \
+		( echo "#!/bin/sh" ; echo "$(top_srcdir)/tests/conform/test-launcher.sh '$$i' \"\$$@\"" ) > $$unit$(EXEEXT) ; \
+		( echo "#!/bin/sh" ; echo "exec ./test-conformance$(EXEEXT) -p $$i \"\$$@\"" ) > wrappers/$$unit$(EXEEXT) ; \
 		chmod +x $$unit$(EXEEXT); \
+		chmod +x wrappers/$$unit$(EXEEXT); \
 	done \
 	&& echo timestamp > $(@F)
 
@@ -75,6 +78,7 @@ clean-wrappers:
 		unit=`basename $$i | sed -e s/_/-/g`; \
 		echo "  RM     $$unit"; \
 		rm -f $$unit$(EXEEXT) ; \
+		rm -f wrappers/$$unit$(EXEEXT) ; \
 	done \
 	&& rm -f stamp-test-conformance
 
@@ -107,22 +111,26 @@ test_conformance_LDFLAGS = -rdynamic
 .PHONY: full-report full-report-normal full-report-disable-npots
 .PHONY: full-report-generate
 
-test:
+test: wrappers
-	@gtester -o=test-conformance-results.xml ./test-conformance
+	@$(top_srcdir)/tests/conform/run-tests.sh \
+	  ./test-conformance$(EXEEXT) test-conformance-results.xml
 
-test-verbose:
+test-verbose: wrappers
-	@gtester --verbose -o=test-conformance-result.xml ./test-conformance
+	@$(top_srcdir)/tests/conform/run-tests.sh \
+	  ./test-conformance$(EXEEXT) test-conformance-results.xml --verbose
 
-test-report-normal:
+test-report-normal: wrappers
-	@gtester -o=test-conformance-results.xml -k ./test-conformance \
+	@$(top_srcdir)/tests/conform/run-tests.sh \
+	  ./test-conformance$(EXEEXT) test-conformance-results.xml -k \
 	  && ( gtester-report test-conformance-results.xml \
 	      | sed 's/>GTester Unit Test Report</>GTester Unit Test Report (normal)</' \
 	      > test-conformance-results.html ) \
 	  && gnome-open ./test-conformance-results.html
 
-test-report-disable-npots:
+test-report-disable-npots: wrappers
 	@../tools/disable-npots.sh \
-	  gtester -o=test-conformance-results-dn.xml -k ./test-conformance \
+	  $(top_srcdir)/tests/conform/run-tests.sh \
+	    ./test-conformance$(EXEEXT) test-conformance-results-dn.xml -k \
 	  && ( gtester-report test-conformance-results-dn.xml \
 	      | sed 's/>GTester Unit Test Report</>GTester Unit Test Report (no NPOTs)</' \
 	      > test-conformance-results-dn.html ) \
@@ -130,15 +138,17 @@ test-report-disable-npots:
 
 test-report: test-report-normal
 
-full-report-normal:
+full-report-normal: wrappers
-	@gtester -o=test-conformance-results.xml -k -m=slow ./test-conformance \
+	@$(top_srcdir)/tests/conform/run-tests.sh \
+	  ./test-conformance$(EXEEXT) test-conformance-results.xml -k -m=slow \
 	  && ( gtester-report test-conformance-results.xml \
 	      | sed 's/>GTester Unit Test Report</>GTester Unit Test Report (normal)</' \
 	      > test-conformance-results.html )
 
-full-report-disable-npots:
+full-report-disable-npots: wrappers
 	@../tools/disable-npots.sh \
-	  gtester -o=test-conformance-results-dn.xml -k -m=slow ./test-conformance \
+	  $(top_srcdir)/tests/conform/run-tests.sh \
+	    ./test-conformance$(EXEEXT) test-conformance-results-dn.xml -k -m=slow \
 	  && ( gtester-report test-conformance-results-dn.xml \
 	      | sed 's/>GTester Unit Test Report</>GTester Unit Test Report (no NPOTs)</' \
 	      > test-conformance-results-dn.html )
@@ -161,7 +171,7 @@ full-report: full-report-generate
 		gnome-open "$$x"; \
 	done
 
-EXTRA_DIST = ADDING_NEW_TESTS test-launcher.sh
+EXTRA_DIST = ADDING_NEW_TESTS test-launcher.sh run-tests.sh
 
 # we override the clean-generic target to clean up the wrappers so
 # we cannot use CLEANFILES

tests/conform/run-tests.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+BINARY=$1
+shift
+XML=$1
+shift
+
+TMP=`./$BINARY -l -m thorough | grep '^/' | sed -e s/_/-/g`
+for i in $TMP
+do
+  TESTS="$TESTS wrappers/`basename $i`"
+done
+
+exec gtester -o=$XML "$@" $TESTS

tests/conform/test-launcher.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 
 UNIT_TEST_PATH=$1
+shift
 
 test -z ${UNIT_TEST_PATH} && {
         echo "Usage: $0 /path/to/unit_test"
@@ -9,9 +10,9 @@ test -z ${UNIT_TEST_PATH} && {
 
 UNIT_TEST=`basename ${UNIT_TEST_PATH}`
 
-echo "Running: gtester -p ${UNIT_TEST_PATH} ./test-conformance"
+echo "Running: ./test-conformance -p ${UNIT_TEST_PATH} $@"
 echo ""
-gtester -p ${UNIT_TEST_PATH} -m slow --verbose ./test-conformance
+./test-conformance -p ${UNIT_TEST_PATH} "$@"
 
 echo ""
 echo "NOTE: For debugging purposes, you can run this single test as follows:"