monitor-transform: Fix transform() and drop relative_transform()

The following implicit definition for `transform()` did not
correctly apply:
```
a * b = c
c * invert(b) = a
```

Crucially the following did not apply for `FLIPPED-90`
and `FLIPPED-270`:
```
a * invert(a) = identity
```

Fix this by applying the operations, first the flip, then the
rotation, in this order and add tests to ensure correct results
for the requirement above.

Also drop `relative_transform()` as it only had a single user and
can be replaced by `transform()`:
```
invert(a) * b = c
a * c = b
```

As this is not very intuitive, ensure in tests as well.

Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/2459>

src/backends/meta-monitor-transform.c

@@ -58,47 +58,56 @@ meta_monitor_transform_invert (MetaMonitorTransform transform)
   return 0;
 }
 
+static MetaMonitorTransform
+meta_monitor_transform_flip (MetaMonitorTransform transform)
+{
+  switch (transform)
+    {
+    case META_MONITOR_TRANSFORM_NORMAL:
+      return META_MONITOR_TRANSFORM_FLIPPED;
+    case META_MONITOR_TRANSFORM_90:
+      return META_MONITOR_TRANSFORM_FLIPPED_270;
+    case META_MONITOR_TRANSFORM_180:
+      return META_MONITOR_TRANSFORM_FLIPPED_180;
+    case META_MONITOR_TRANSFORM_270:
+      return META_MONITOR_TRANSFORM_FLIPPED_90;
+    case META_MONITOR_TRANSFORM_FLIPPED:
+      return META_MONITOR_TRANSFORM_NORMAL;
+    case META_MONITOR_TRANSFORM_FLIPPED_90:
+      return META_MONITOR_TRANSFORM_270;
+    case META_MONITOR_TRANSFORM_FLIPPED_180:
+      return META_MONITOR_TRANSFORM_180;
+    case META_MONITOR_TRANSFORM_FLIPPED_270:
+      return META_MONITOR_TRANSFORM_90;
+    }
+  g_assert_not_reached ();
+  return 0;
+}
+
 MetaMonitorTransform
 meta_monitor_transform_transform (MetaMonitorTransform transform,
                                   MetaMonitorTransform other)
 {
   MetaMonitorTransform new_transform;
+  gboolean needs_flip = FALSE;
 
-  new_transform = (transform + other) % META_MONITOR_TRANSFORM_FLIPPED;
+  if (meta_monitor_transform_is_flipped (other))
-  if (meta_monitor_transform_is_flipped (transform) !=
+    new_transform = meta_monitor_transform_flip (transform);
-      meta_monitor_transform_is_flipped (other))
+  else
+    new_transform = transform;
+
+  if (meta_monitor_transform_is_flipped (new_transform))
+    needs_flip = TRUE;
+
+  new_transform += other;
+  new_transform %= META_MONITOR_TRANSFORM_FLIPPED;
+
+  if (needs_flip)
     new_transform += META_MONITOR_TRANSFORM_FLIPPED;
 
   return new_transform;
 }
 
-/**
- * meta_monitor_transform_relative_transform:
- * @transform: The transform to start from
- * @other: The transform to go to
- *
- * Return value: a transform to get from @transform to @other
- */
-MetaMonitorTransform
-meta_monitor_transform_relative_transform (MetaMonitorTransform transform,
-                                           MetaMonitorTransform other)
-{
-  MetaMonitorTransform relative_transform;
-
-  relative_transform = ((other % META_MONITOR_TRANSFORM_FLIPPED -
-                         transform % META_MONITOR_TRANSFORM_FLIPPED) %
-                        META_MONITOR_TRANSFORM_FLIPPED);
-
-  if (meta_monitor_transform_is_flipped (transform) !=
-      meta_monitor_transform_is_flipped (other))
-    {
-      relative_transform = (meta_monitor_transform_invert (relative_transform) +
-                            META_MONITOR_TRANSFORM_FLIPPED);
-    }
-
-  return relative_transform;
-}
-
 void
 meta_monitor_transform_transform_point (MetaMonitorTransform  transform,
                                         int                   area_width,

src/backends/meta-monitor-transform.h

@@ -64,9 +64,6 @@ META_EXPORT_TEST
 MetaMonitorTransform meta_monitor_transform_transform (MetaMonitorTransform transform,
                                                        MetaMonitorTransform other);
 
-MetaMonitorTransform meta_monitor_transform_relative_transform (MetaMonitorTransform transform,
-                                                                MetaMonitorTransform other);
-
 void meta_monitor_transform_transform_point (MetaMonitorTransform  transform,
                                              int                   area_width,
                                              int                   area_height,

src/backends/native/meta-cursor-renderer-native.c

@@ -816,8 +816,9 @@ get_common_crtc_sprite_transform_for_logical_monitors (MetaCursorRenderer   *ren
         {
           MetaMonitor *monitor = l_mon->data;
 
-          tmp_transform = meta_monitor_transform_relative_transform (
+          tmp_transform = meta_monitor_transform_transform (
-            meta_cursor_sprite_get_texture_transform (cursor_sprite),
+            meta_monitor_transform_invert (
+              meta_cursor_sprite_get_texture_transform (cursor_sprite)),
             meta_monitor_logical_to_crtc_transform (monitor, logical_transform));
 
           if (has_visible_crtc_sprite && transform != tmp_transform)

src/tests/monitor-transform-tests.c

@@ -83,6 +83,7 @@ test_transform (void)
     },
   };
   int i;
+  MetaMonitorTransform transform;
 
   for (i = 0; i < G_N_ELEMENTS (tests); i++)
     {
@@ -92,6 +93,34 @@ test_transform (void)
                                                  tests[i].other);
       g_assert_cmpint (result, ==, tests[i].expect);
     }
+
+  for (transform = 0; transform <= META_MONITOR_TRANSFORM_FLIPPED_270; transform++)
+    {
+      MetaMonitorTransform other;
+      MetaMonitorTransform result1;
+
+      result1 =
+        meta_monitor_transform_transform (transform,
+                                          meta_monitor_transform_invert (transform));
+      g_assert_cmpint (result1, ==, META_MONITOR_TRANSFORM_NORMAL);
+
+      for (other = 0; other <= META_MONITOR_TRANSFORM_FLIPPED_270; other++)
+        {
+          MetaMonitorTransform result2;
+
+          result1 = meta_monitor_transform_transform (transform, other);
+          result2 =
+            meta_monitor_transform_transform (result1,
+                                              meta_monitor_transform_invert (other));
+          g_assert_cmpint (result2, ==, transform);
+
+          result1 =
+            meta_monitor_transform_transform (meta_monitor_transform_invert (transform),
+                                              other);
+          result2 = meta_monitor_transform_transform (transform, result1);
+          g_assert_cmpint (result2, ==, other);
+        }
+    }
 }
 
 void