backend/x11/nested: Enable emulating offscreen transform framebuffer

Enable the usage of an offscreen transform texture, as used in the
native backend when the GPU doesn't support the transform configured.

https://bugzilla.gnome.org/show_bug.cgi?id=779745

src/backends/meta-monitor-manager-dummy.c

@@ -42,6 +42,8 @@
 struct _MetaMonitorManagerDummy
 {
   MetaMonitorManager parent_instance;
+
+  gboolean is_transform_handled;
 };
 
 struct _MetaMonitorManagerDummyClass
@@ -526,7 +528,9 @@ meta_monitor_manager_dummy_is_transform_handled (MetaMonitorManager  *manager,
                                                  MetaCrtc            *crtc,
                                                  MetaMonitorTransform transform)
 {
-  return TRUE;
+  MetaMonitorManagerDummy *manager_dummy = META_MONITOR_MANAGER_DUMMY (manager);
+
+  return manager_dummy->is_transform_handled;
 }
 
 static void
@@ -544,4 +548,12 @@ meta_monitor_manager_dummy_class_init (MetaMonitorManagerDummyClass *klass)
 static void
 meta_monitor_manager_dummy_init (MetaMonitorManagerDummy *manager)
 {
+  const char *nested_offscreen_transform;
+
+  nested_offscreen_transform =
+    g_getenv ("MUTTER_DEBUG_NESTED_OFFSCREEN_TRANSFORM");
+  if (g_strcmp0 (nested_offscreen_transform, "1") == 0)
+    manager->is_transform_handled = FALSE;
+  else
+    manager->is_transform_handled = TRUE;
 }

src/backends/x11/meta-renderer-x11.c

@@ -77,15 +77,41 @@ meta_renderer_x11_create_cogl_renderer (MetaRenderer *renderer)
   return cogl_renderer;
 }
 
+static MetaMonitorTransform
+calculate_view_transform (MetaMonitorManager *monitor_manager,
+                          MetaLogicalMonitor *logical_monitor)
+{
+  MetaMonitor *main_monitor;
+  MetaOutput *main_output;
+  main_monitor = meta_logical_monitor_get_monitors (logical_monitor)->data;
+  main_output = meta_monitor_get_main_output (main_monitor);
+
+  /*
+   * Pick any monitor and output and check; all CRTCs of a logical monitor will
+   * always have the same transform assigned to them.
+   */
+
+  if (meta_monitor_manager_is_transform_handled (monitor_manager,
+                                                 main_output->crtc,
+                                                 main_output->crtc->transform))
+    return META_MONITOR_TRANSFORM_NORMAL;
+  else
+    return main_output->crtc->transform;
+}
+
 static MetaRendererView *
 meta_renderer_x11_create_view (MetaRenderer       *renderer,
                                MetaLogicalMonitor *logical_monitor)
 {
   MetaBackend *backend = meta_get_backend ();
+  MetaMonitorManager *monitor_manager =
+    meta_backend_get_monitor_manager (backend);
   ClutterBackend *clutter_backend = meta_backend_get_clutter_backend (backend);
   CoglContext *cogl_context = clutter_backend_get_cogl_context (clutter_backend);
+  MetaMonitorTransform view_transform;
   int width, height;
   CoglTexture2D *texture_2d;
+  CoglOffscreen *fake_onscreen;
   CoglOffscreen *offscreen;
   GError *error = NULL;
 
@@ -93,15 +119,32 @@ meta_renderer_x11_create_view (MetaRenderer       *renderer,
 
   width = logical_monitor->rect.width;
   height = logical_monitor->rect.height;
-  texture_2d = cogl_texture_2d_new_with_size (cogl_context, width, height);
-  offscreen = cogl_offscreen_new_with_texture (COGL_TEXTURE (texture_2d));
 
-  if (!cogl_framebuffer_allocate (COGL_FRAMEBUFFER (offscreen), &error))
+  view_transform = calculate_view_transform (monitor_manager, logical_monitor);
+
+  texture_2d = cogl_texture_2d_new_with_size (cogl_context, width, height);
+  fake_onscreen = cogl_offscreen_new_with_texture (COGL_TEXTURE (texture_2d));
+
+  if (!cogl_framebuffer_allocate (COGL_FRAMEBUFFER (fake_onscreen), &error))
     meta_fatal ("Couldn't allocate framebuffer: %s", error->message);
 
+  if (view_transform != META_MONITOR_TRANSFORM_NORMAL)
+    {
+      texture_2d = cogl_texture_2d_new_with_size (cogl_context, width, height);
+      offscreen = cogl_offscreen_new_with_texture (COGL_TEXTURE (texture_2d));
+      if (!cogl_framebuffer_allocate (COGL_FRAMEBUFFER (offscreen), &error))
+        meta_fatal ("Couldn't allocate offscreen framebuffer: %s", error->message);
+    }
+  else
+    {
+      offscreen = NULL;
+    }
+
   return g_object_new (META_TYPE_RENDERER_VIEW,
                        "layout", &logical_monitor->rect,
-                       "framebuffer", COGL_FRAMEBUFFER (offscreen),
+                       "framebuffer", COGL_FRAMEBUFFER (fake_onscreen),
+                       "offscreen", COGL_FRAMEBUFFER (offscreen),
+                       "transform", view_transform,
                        NULL);
 }
 

src/backends/x11/meta-stage-x11-nested.c

@@ -102,11 +102,17 @@ meta_stage_x11_nested_finish_frame (ClutterStageWindow *stage_window)
       cairo_rectangle_int_t view_layout;
       CoglFramebuffer *framebuffer;
       CoglTexture *texture;
+      CoglMatrix matrix;
 
       clutter_stage_view_get_layout (view, &view_layout);
 
-      framebuffer = clutter_stage_view_get_framebuffer (view);
+      framebuffer = clutter_stage_view_get_onscreen (view);
       texture = cogl_offscreen_get_texture (COGL_OFFSCREEN (framebuffer));
+
+      clutter_stage_view_get_offscreen_transformation_matrix (view, &matrix);
+      cogl_matrix_get_inverse (&matrix, &matrix);
+      cogl_pipeline_set_layer_matrix (stage_nested->pipeline, 0, &matrix);
+
       cogl_framebuffer_set_viewport (onscreen,
                                      view_layout.x,
                                      view_layout.y,