conform: Implement TEST_CONFORM_TODO() correctly

The TODO() macro for adding new tests to the test suite has always meant
to be implemented like the TODO block in Test::More, i.e. a test that is
assumed to fail, and which warns if it unexpectedly succeeds.

Since GTest lacks the expressivity of Test::More, the implementation
just verifies that the tests marked as TODO actually fail, and will fail
if they happen to succeed - at which point the developer will have to
change the macro to SIMPLE or SKIP.

tests/conform/ADDING_NEW_TESTS

@@ -1,31 +1,65 @@
+How to add new units to the Clutter Conformance test suite
+-------------------------------------------------------------------------------
 
-How:
-----
+If the new unit is not logically part of an existing file, you should create a
+new source file and add it to the build. The built files are listed in the
+Makefile.am template. Please, respect the sections already available there.
 
-You should including the following at the top of your new unit test:
-#include "test-conform-common.h"
+When creating a new file, you should include "test-conform-common.h", as well
+as <clutter/clutter.h> and <glib.h>.
 
-Instead of a main () function add an entry point with a prototype as follows:
+Each new test unit should be contained in a function named following this
+pattern:
 
-void
-test_blah (TestConformSimpleFixture *fixture,
-           gconstpointer data)
-{
+  void
+  <section>_<unit> (void)
+  {
+  }
 
-}
+For instance:
 
-Add a TEST_CONFORM_SIMPLE() entry in test-conform-main.c
+  void
+  actor_allocation_changes (void)
+  {
+    /* test allocation changes */
+  }
 
+  void
+  rectangle_border (void)
+  {
+    /* test ClutterRectangle's border property */
+  }
+
+After adding the test unit, edit test-conform-main.c and add the unit to the
+list of tests, by using one of these three macros:
+
+  • TEST_CONFORM_SIMPLE (path, function_name);
+
+    @path is the unit path in the suite, and it's used to generate the
+          executable wrapper for running the unit as a stand-alone binary
+          and for the HTML report.
+    @function_name is the name of the function containing the newly added
+          test unit.
+
+    This is the simple form of test unit, and it will always be run.
+
+  • TEST_CONFORM_SKIP (condition, path, function_name);
+
+    @condition is used to decide whether to run the unit or not.
+
+    This macro will check @condition on start, and if it evaluates to TRUE
+    then the @function_name will be called and the unit executed; otherwise
+    the test will automatically succeed.
+
+  • TEST_CONFORM_TODO (path, function_name);
+
+    This macro will execute @function_name and will succeed if the unit
+    fails. This macro should be used for tests that are known to fail.
 
 Notes:
-------
 
-NB: A test fails if it exits. (regardless of the exit status)
-
-Don't call clutter_init since that is handled in test-conform-common.c
-
-Make sure you clean up *everything* you create. Noteable things you might miss
-include timelines, behaviours, and all actors you add to the stage. This is important because otherwise you can cause cascading failures in other tests.
-
-Be aware that to help avoid tests causing cascading side effects for other tests all children of the default stage are destroyed between tests.
+  • Do not call clutter_init() in your units; initialization is handled
+    by the conformance test suite itself.
 
+  • All units are invoked in a new process, to prevent clobbering the
+    state.

tests/conform/Makefile.am

@@ -91,8 +91,10 @@ wrappers: stamp-test-conformance
 	@true
 stamp-test-conformance: Makefile $(srcdir)/test-conform-main.c
 	@mkdir -p wrappers
-	@sed -n -e 's/^ \{1,\}TEST_CONFORM_SIMPLE *(.*"\([^",]\{1,\}\)", *\([a-zA-Z0-9_]\{1,\}\).*/\/conform\1\/\2/p' \
-	-e 's/^ \{1,\}TEST_CONFORM_SKIP *(.*"\([^",]\{1,\}\)", *\([a-zA-Z0-9_]\{1,\}\).*/\/conform\1\/\2/p' \
+	@sed -n \
+		-e 's/^ \{1,\}TEST_CONFORM_SIMPLE *(.*"\([^",]\{1,\}\)", *\([a-zA-Z0-9_]\{1,\}\).*/\/conform\1\/\2/p' \
+		-e 's/^ \{1,\}TEST_CONFORM_SKIP *(.*"\([^",]\{1,\}\)", *\([a-zA-Z0-9_]\{1,\}\).*/\/conform\1\/\2/p' \
+		-e 's/^ \{1,\}TEST_CONFORM_TODO *(.*"\([^",]\{1,\}\)", *\([a-zA-Z0-9_]\{1,\}\).*/\/conform\1\/\2/p' \
 	$(srcdir)/test-conform-main.c > unit-tests
 	@chmod +x test-launcher.sh
 	@( echo "/stamp-test-conformance" ; \

tests/conform/test-actor-invariants.c

@@ -5,6 +5,13 @@
 
 #include "test-conform-common.h"
 
+void
+verify_failure (TestConformSimpleFixture *fixture,
+                gconstpointer data)
+{
+  g_assert (FALSE);
+}
+
 void
 test_initial_state (TestConformSimpleFixture *fixture,
                     gconstpointer             data)

tests/conform/test-conform-common.h

@@ -19,6 +19,12 @@ typedef struct _TestConformSimpleFixture
   int dummy;
 } TestConformSimpleFixture;
 
+typedef struct _TestConformTodo
+{
+  gchar *name;
+  void (* func) (TestConformSimpleFixture *, gconstpointer);
+} TestConformTodo;
+
 void test_conform_simple_fixture_setup (TestConformSimpleFixture *fixture,
 					gconstpointer data);
 void test_conform_simple_fixture_teardown (TestConformSimpleFixture *fixture,

tests/conform/test-conform-main.c

@@ -15,6 +15,23 @@ test_conform_skip_test (TestConformSimpleFixture *fixture,
   /* void */
 }
 
+static void
+test_conform_todo_test (TestConformSimpleFixture *fixture,
+                        gconstpointer             data)
+{
+#ifdef G_OS_UNIX
+  const TestConformTodo *todo = data;
+
+  if (g_test_trap_fork (0, G_TEST_TRAP_SILENCE_STDOUT | G_TEST_TRAP_SILENCE_STDERR))
+    {
+      todo->func (fixture, NULL);
+      exit (0);
+    }
+
+  g_test_trap_assert_failed ();
+#endif
+}
+
 static TestConformSharedState *shared_state = NULL;
 
 /* This is a bit of sugar for adding new conformance tests:
@@ -52,11 +69,15 @@ static TestConformSharedState *shared_state = NULL;
   } else { TEST_CONFORM_SIMPLE (NAMESPACE, FUNC); }     } G_STMT_END
 
 #define TEST_CONFORM_TODO(NAMESPACE, FUNC)              G_STMT_START {  \
-  g_test_add ("/todo" NAMESPACE "/" #FUNC,                              \
+   extern void FUNC (TestConformSimpleFixture *, gconstpointer);        \
+   TestConformTodo *_clos = g_new0 (TestConformTodo, 1);                \
+   _clos->name = g_strdup ( #FUNC );                                    \
+   _clos->func = FUNC;                                                  \
+   g_test_add ("/todo" NAMESPACE "/" #FUNC,                             \
               TestConformSimpleFixture,                                 \
-              shared_state,                                             \
+              _clos,                                                    \
               test_conform_simple_fixture_setup,                        \
-              test_conform_skip_test,                                   \
+              test_conform_todo_test,                                   \
               test_conform_simple_fixture_teardown);    } G_STMT_END
 
 gchar *
@@ -97,6 +118,9 @@ main (int argc, char **argv)
      each. To comment out a test use the SKIP or TODO macros. Using
      #if 0 would break the script. */
 
+  /* this is a sanity check for the test suite itself */
+  TEST_CONFORM_TODO ("/suite", verify_failure);
+
   TEST_CONFORM_SIMPLE ("/timeline", test_timeline);
   TEST_CONFORM_SKIP (!g_test_slow (), "/timeline", test_timeline_interpolate);
   TEST_CONFORM_SKIP (!g_test_slow (), "/timeline", test_timeline_rewind);