screen-cast: Track and always record cursors

Always force-track the cursor position (so that the X11 backend can keep
it up to date), and if the cursor wasn't part of the sampled
framebuffer when reading pixels into CPU memory, draw it in an extra
pass using cairo after the fact. The cairo based cursor painting only
happens on the X11 backend, as we otherwise inhibit the hw cursor.

https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1391

clutter/clutter/clutter-paint-context.h

@@ -33,6 +33,7 @@ typedef enum _ClutterPaintFlag
 {
   CLUTTER_PAINT_FLAG_NONE = 0,
   CLUTTER_PAINT_FLAG_NO_CURSORS = 1 << 0,
+  CLUTTER_PAINT_FLAG_FORCE_CURSORS = 1 << 1,
 } ClutterPaintFlag;
 
 #define CLUTTER_TYPE_PAINT_CONTEXT (clutter_paint_context_get_type ())

src/backends/meta-cursor-renderer.c

@@ -409,6 +409,18 @@ meta_cursor_renderer_get_cursor (MetaCursorRenderer *renderer)
   return priv->displayed_cursor;
 }
 
+gboolean
+meta_cursor_renderer_is_overlay_visible (MetaCursorRenderer *renderer)
+{
+  MetaCursorRendererPrivate *priv =
+    meta_cursor_renderer_get_instance_private (renderer);
+
+  if (!priv->stage_overlay)
+    return FALSE;
+
+  return meta_overlay_is_visible (priv->stage_overlay);
+}
+
 void
 meta_cursor_renderer_add_hw_cursor_inhibitor (MetaCursorRenderer    *renderer,
                                               MetaHwCursorInhibitor *inhibitor)

src/backends/meta-cursor-renderer.h

@@ -67,6 +67,8 @@ void meta_cursor_renderer_force_update (MetaCursorRenderer *renderer);
 
 MetaCursorSprite * meta_cursor_renderer_get_cursor (MetaCursorRenderer *renderer);
 
+gboolean meta_cursor_renderer_is_overlay_visible (MetaCursorRenderer *renderer);
+
 void meta_cursor_renderer_add_hw_cursor_inhibitor (MetaCursorRenderer    *renderer,
                                                    MetaHwCursorInhibitor *inhibitor);
 

src/backends/meta-screen-cast-area-stream-src.c

@@ -343,6 +343,7 @@ meta_screen_cast_area_stream_src_enable (MetaScreenCastStreamSrc *src)
         g_signal_connect_after (cursor_tracker, "cursor-changed",
                                 G_CALLBACK (cursor_changed),
                                 area_src);
+      meta_cursor_tracker_track_position (cursor_tracker);
       G_GNUC_FALLTHROUGH;
     case META_SCREEN_CAST_CURSOR_MODE_HIDDEN:
       add_view_painted_watches (area_src,
@@ -350,6 +351,7 @@ meta_screen_cast_area_stream_src_enable (MetaScreenCastStreamSrc *src)
       break;
     case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED:
       inhibit_hw_cursor (area_src);
+      meta_cursor_tracker_track_position (cursor_tracker);
       add_view_painted_watches (area_src,
                                 META_STAGE_WATCH_AFTER_ACTOR_PAINT);
       break;
@@ -363,6 +365,7 @@ meta_screen_cast_area_stream_src_disable (MetaScreenCastStreamSrc *src)
 {
   MetaScreenCastAreaStreamSrc *area_src =
     META_SCREEN_CAST_AREA_STREAM_SRC (src);
+  MetaScreenCastStream *stream = meta_screen_cast_stream_src_get_stream (src);
   MetaBackend *backend = get_backend (area_src);
   MetaCursorTracker *cursor_tracker = meta_backend_get_cursor_tracker (backend);
   ClutterStage *stage;
@@ -389,6 +392,16 @@ meta_screen_cast_area_stream_src_disable (MetaScreenCastStreamSrc *src)
                           cursor_tracker);
 
   g_clear_handle_id (&area_src->maybe_record_idle_id, g_source_remove);
+
+  switch (meta_screen_cast_stream_get_cursor_mode (stream))
+    {
+    case META_SCREEN_CAST_CURSOR_MODE_METADATA:
+    case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED:
+      meta_cursor_tracker_untrack_position (cursor_tracker);
+      break;
+    case META_SCREEN_CAST_CURSOR_MODE_HIDDEN:
+      break;
+    }
 }
 
 static gboolean
@@ -418,6 +431,7 @@ meta_screen_cast_area_stream_src_record_to_buffer (MetaScreenCastStreamSrc  *src
       paint_flags |= CLUTTER_PAINT_FLAG_NO_CURSORS;
       break;
     case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED:
+      paint_flags |= CLUTTER_PAINT_FLAG_FORCE_CURSORS;
       break;
     }
 
@@ -458,6 +472,7 @@ meta_screen_cast_area_stream_src_record_to_framebuffer (MetaScreenCastStreamSrc
       paint_flags |= CLUTTER_PAINT_FLAG_NO_CURSORS;
       break;
     case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED:
+      paint_flags |= CLUTTER_PAINT_FLAG_FORCE_CURSORS;
       break;
     }
   clutter_stage_paint_to_framebuffer (stage, framebuffer,

src/backends/meta-screen-cast-monitor-stream-src.c

@@ -343,6 +343,7 @@ meta_screen_cast_monitor_stream_src_enable (MetaScreenCastStreamSrc *src)
         g_signal_connect_after (cursor_tracker, "cursor-changed",
                                 G_CALLBACK (cursor_changed),
                                 monitor_src);
+      meta_cursor_tracker_track_position (cursor_tracker);
       G_GNUC_FALLTHROUGH;
     case META_SCREEN_CAST_CURSOR_MODE_HIDDEN:
       add_view_painted_watches (monitor_src,
@@ -350,6 +351,7 @@ meta_screen_cast_monitor_stream_src_enable (MetaScreenCastStreamSrc *src)
       break;
     case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED:
       inhibit_hw_cursor (monitor_src);
+      meta_cursor_tracker_track_position (cursor_tracker);
       add_view_painted_watches (monitor_src,
                                 META_STAGE_WATCH_AFTER_PAINT);
       break;
@@ -363,6 +365,7 @@ meta_screen_cast_monitor_stream_src_disable (MetaScreenCastStreamSrc *src)
 {
   MetaScreenCastMonitorStreamSrc *monitor_src =
     META_SCREEN_CAST_MONITOR_STREAM_SRC (src);
+  MetaScreenCastStream *stream = meta_screen_cast_stream_src_get_stream (src);
   MetaBackend *backend = get_backend (monitor_src);
   MetaCursorTracker *cursor_tracker = meta_backend_get_cursor_tracker (backend);
   ClutterStage *stage;
@@ -387,6 +390,79 @@ meta_screen_cast_monitor_stream_src_disable (MetaScreenCastStreamSrc *src)
                           cursor_tracker);
   g_clear_signal_handler (&monitor_src->cursor_changed_handler_id,
                           cursor_tracker);
+
+  switch (meta_screen_cast_stream_get_cursor_mode (stream))
+    {
+    case META_SCREEN_CAST_CURSOR_MODE_METADATA:
+    case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED:
+      meta_cursor_tracker_untrack_position (cursor_tracker);
+      break;
+    case META_SCREEN_CAST_CURSOR_MODE_HIDDEN:
+      break;
+    }
+}
+
+static void
+maybe_paint_cursor_sprite (MetaScreenCastMonitorStreamSrc *monitor_src,
+                           uint8_t                        *data)
+{
+  MetaScreenCastStreamSrc *src = META_SCREEN_CAST_STREAM_SRC (monitor_src);
+  MetaBackend *backend = get_backend (monitor_src);
+  MetaCursorRenderer *cursor_renderer =
+    meta_backend_get_cursor_renderer (backend);
+  MetaCursorSprite *cursor_sprite;
+  CoglTexture *sprite_texture;
+  int sprite_width, sprite_height, sprite_stride;
+  float sprite_scale;
+  uint8_t *sprite_data;
+  cairo_surface_t *sprite_surface;
+  graphene_rect_t sprite_rect;
+  int width, height, stride;
+  cairo_surface_t *surface;
+  cairo_t *cr;
+
+  cursor_sprite = meta_cursor_renderer_get_cursor (cursor_renderer);
+  if (!cursor_sprite)
+    return;
+
+  if (meta_cursor_renderer_is_overlay_visible (cursor_renderer))
+    return;
+
+  sprite_rect = meta_cursor_renderer_calculate_rect (cursor_renderer,
+                                                     cursor_sprite);
+  sprite_texture = meta_cursor_sprite_get_cogl_texture (cursor_sprite);
+  sprite_width = cogl_texture_get_width (sprite_texture);
+  sprite_height = cogl_texture_get_height (sprite_texture);
+  sprite_stride = sprite_width * 4;
+  sprite_scale = meta_cursor_sprite_get_texture_scale (cursor_sprite);
+  sprite_data = g_new0 (uint8_t, sprite_stride * sprite_height);
+  cogl_texture_get_data (sprite_texture,
+                         CLUTTER_CAIRO_FORMAT_ARGB32,
+                         sprite_stride,
+                         sprite_data);
+  sprite_surface = cairo_image_surface_create_for_data (sprite_data,
+                                                        CAIRO_FORMAT_ARGB32,
+                                                        sprite_width,
+                                                        sprite_height,
+                                                        sprite_stride);
+  cairo_surface_set_device_scale (sprite_surface, sprite_scale, sprite_scale);
+
+  stride = meta_screen_cast_stream_src_get_stride (src);
+  width = meta_screen_cast_stream_src_get_width (src);
+  height = meta_screen_cast_stream_src_get_height (src);
+  surface = cairo_image_surface_create_for_data (data,
+                                                 CAIRO_FORMAT_ARGB32,
+                                                 width, height, stride);
+
+  cr = cairo_create (surface);
+  cairo_set_source_surface (cr, sprite_surface,
+                            sprite_rect.origin.x,
+                            sprite_rect.origin.y);
+  cairo_paint (cr);
+  cairo_destroy (cr);
+  cairo_surface_destroy (sprite_surface);
+  cairo_surface_destroy (surface);
+  g_free (sprite_data);
 }
 
 static gboolean
@@ -396,6 +472,7 @@ meta_screen_cast_monitor_stream_src_record_to_buffer (MetaScreenCastStreamSrc  *
 {
   MetaScreenCastMonitorStreamSrc *monitor_src =
     META_SCREEN_CAST_MONITOR_STREAM_SRC (src);
+  MetaScreenCastStream *stream = meta_screen_cast_stream_src_get_stream (src);
   ClutterStage *stage;
   MetaMonitor *monitor;
   MetaLogicalMonitor *logical_monitor;
@@ -405,6 +482,16 @@ meta_screen_cast_monitor_stream_src_record_to_buffer (MetaScreenCastStreamSrc  *
   stage = get_stage (monitor_src);
   clutter_stage_capture_into (stage, FALSE, &logical_monitor->rect, data);
 
+  switch (meta_screen_cast_stream_get_cursor_mode (stream))
+    {
+    case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED:
+      maybe_paint_cursor_sprite (monitor_src, data);
+      break;
+    case META_SCREEN_CAST_CURSOR_MODE_METADATA:
+    case META_SCREEN_CAST_CURSOR_MODE_HIDDEN:
+      break;
+    }
+
   return TRUE;
 }
 

src/backends/meta-screen-cast-stream-src.c

@@ -1058,6 +1058,24 @@ meta_screen_cast_stream_src_get_stride (MetaScreenCastStreamSrc *src)
   return priv->video_stride;
 }
 
+int
+meta_screen_cast_stream_src_get_width (MetaScreenCastStreamSrc *src)
+{
+  MetaScreenCastStreamSrcPrivate *priv =
+    meta_screen_cast_stream_src_get_instance_private (src);
+
+  return priv->stream_width;
+}
+
+int
+meta_screen_cast_stream_src_get_height (MetaScreenCastStreamSrc *src)
+{
+  MetaScreenCastStreamSrcPrivate *priv =
+    meta_screen_cast_stream_src_get_instance_private (src);
+
+  return priv->stream_height;
+}
+
 MetaScreenCastStream *
 meta_screen_cast_stream_src_get_stream (MetaScreenCastStreamSrc *src)
 {

src/backends/meta-screen-cast-stream-src.h

@@ -80,6 +80,10 @@ gboolean meta_screen_cast_stream_src_pending_follow_up_frame (MetaScreenCastStre
 
 int meta_screen_cast_stream_src_get_stride (MetaScreenCastStreamSrc *src);
 
+int meta_screen_cast_stream_src_get_width (MetaScreenCastStreamSrc *src);
+
+int meta_screen_cast_stream_src_get_height (MetaScreenCastStreamSrc *src);
+
 MetaScreenCastStream * meta_screen_cast_stream_src_get_stream (MetaScreenCastStreamSrc *src);
 
 gboolean meta_screen_cast_stream_src_draw_cursor_into (MetaScreenCastStreamSrc  *src,

src/backends/meta-screen-cast-window-stream-src.c

@@ -23,6 +23,7 @@
 #include "backends/meta-screen-cast-window-stream-src.h"
 
 #include "backends/meta-backend-private.h"
+#include "backends/meta-cursor-tracker-private.h"
 #include "backends/meta-screen-cast-session.h"
 #include "backends/meta-screen-cast-window.h"
 #include "backends/meta-screen-cast-window-stream.h"
@@ -306,6 +307,8 @@ static void
 meta_screen_cast_window_stream_src_stop (MetaScreenCastWindowStreamSrc *window_src)
 
 {
+  MetaScreenCastStreamSrc *src = META_SCREEN_CAST_STREAM_SRC (window_src);
+  MetaScreenCastStream *stream = meta_screen_cast_stream_src_get_stream (src);
   MetaBackend *backend = get_backend (window_src);
   MetaCursorTracker *cursor_tracker = meta_backend_get_cursor_tracker (backend);
 
@@ -320,6 +323,16 @@ meta_screen_cast_window_stream_src_stop (MetaScreenCastWindowStreamSrc *window_s
                           cursor_tracker);
   g_clear_signal_handler (&window_src->cursor_changed_handler_id,
                           cursor_tracker);
+
+  switch (meta_screen_cast_stream_get_cursor_mode (stream))
+    {
+    case META_SCREEN_CAST_CURSOR_MODE_METADATA:
+    case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED:
+      meta_cursor_tracker_untrack_position (cursor_tracker);
+      break;
+    case META_SCREEN_CAST_CURSOR_MODE_HIDDEN:
+      break;
+    }
 }
 
 static void
@@ -438,6 +451,7 @@ meta_screen_cast_window_stream_src_enable (MetaScreenCastStreamSrc *src)
         g_signal_connect_after (cursor_tracker, "cursor-changed",
                                 G_CALLBACK (cursor_changed),
                                 window_src);
+      meta_cursor_tracker_track_position (cursor_tracker);
       break;
     case META_SCREEN_CAST_CURSOR_MODE_HIDDEN:
       break;

src/backends/meta-stage.c

@@ -25,6 +25,7 @@
 #include "backends/meta-stage-private.h"
 
 #include "backends/meta-backend-private.h"
+#include "backends/meta-cursor-tracker-private.h"
 #include "clutter/clutter-mutter.h"
 #include "meta/meta-backend.h"
 #include "meta/meta-monitor-manager.h"
@@ -127,7 +128,9 @@ meta_overlay_paint (MetaOverlay         *overlay,
   if (!overlay->texture)
     return;
 
-  if (!overlay->is_visible)
+  if (!overlay->is_visible &&
+      !(clutter_paint_context_get_paint_flags (paint_context) &
+        CLUTTER_PAINT_FLAG_FORCE_CURSORS))
     return;
 
   framebuffer = clutter_paint_context_get_framebuffer (paint_context);
@@ -207,10 +210,28 @@ meta_stage_paint (ClutterActor        *actor,
 
   g_signal_emit (stage, signals[ACTORS_PAINTED], 0);
 
+  if ((clutter_paint_context_get_paint_flags (paint_context) &
+       CLUTTER_PAINT_FLAG_FORCE_CURSORS))
+    {
+      MetaCursorTracker *cursor_tracker =
+        meta_backend_get_cursor_tracker (stage->backend);
+
+      meta_cursor_tracker_track_position (cursor_tracker);
+    }
+
   if (!(clutter_paint_context_get_paint_flags (paint_context) &
         CLUTTER_PAINT_FLAG_NO_CURSORS))
     g_list_foreach (stage->overlays, (GFunc) meta_overlay_paint, paint_context);
 
+  if ((clutter_paint_context_get_paint_flags (paint_context) &
+       CLUTTER_PAINT_FLAG_FORCE_CURSORS))
+    {
+      MetaCursorTracker *cursor_tracker =
+        meta_backend_get_cursor_tracker (stage->backend);
+
+      meta_cursor_tracker_untrack_position (cursor_tracker);
+    }
+
   if (view)
     {
       notify_watchers_for_mode (stage, view, paint_context,