From b4a82471918b5cc5ca116dde0532b6bdfa84cb7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20=C3=85dahl?= Date: Wed, 29 Jul 2020 11:43:44 +0200 Subject: [PATCH] screen-cast: Track and always record cursors Always force-track the cursor position (so that the X11 backend can keep it up to date), and if the cursor wasn't part of the sampled framebuffer when reading pixels into CPU memory, draw it in an extra pass using cairo after the fact. The cairo based cursor painting only happens on the X11 backend, as we otherwise inhibit the hw cursor. https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1391 --- clutter/clutter/clutter-paint-context.h | 1 + src/backends/meta-cursor-renderer.c | 12 +++ src/backends/meta-cursor-renderer.h | 2 + .../meta-screen-cast-area-stream-src.c | 15 ++++ .../meta-screen-cast-monitor-stream-src.c | 87 +++++++++++++++++++ src/backends/meta-screen-cast-stream-src.c | 18 ++++ src/backends/meta-screen-cast-stream-src.h | 4 + .../meta-screen-cast-window-stream-src.c | 14 +++ src/backends/meta-stage.c | 23 ++++- 9 files changed, 175 insertions(+), 1 deletion(-) diff --git a/clutter/clutter/clutter-paint-context.h b/clutter/clutter/clutter-paint-context.h index f781728c3..8461b947a 100644 --- a/clutter/clutter/clutter-paint-context.h +++ b/clutter/clutter/clutter-paint-context.h @@ -33,6 +33,7 @@ typedef enum _ClutterPaintFlag { CLUTTER_PAINT_FLAG_NONE = 0, CLUTTER_PAINT_FLAG_NO_CURSORS = 1 << 0, + CLUTTER_PAINT_FLAG_FORCE_CURSORS = 1 << 1, } ClutterPaintFlag; #define CLUTTER_TYPE_PAINT_CONTEXT (clutter_paint_context_get_type ()) diff --git a/src/backends/meta-cursor-renderer.c b/src/backends/meta-cursor-renderer.c index c8bf6578d..eacbf6724 100644 --- a/src/backends/meta-cursor-renderer.c +++ b/src/backends/meta-cursor-renderer.c @@ -409,6 +409,18 @@ meta_cursor_renderer_get_cursor (MetaCursorRenderer *renderer) return priv->displayed_cursor; } +gboolean +meta_cursor_renderer_is_overlay_visible (MetaCursorRenderer *renderer) +{ + MetaCursorRendererPrivate *priv = + meta_cursor_renderer_get_instance_private (renderer); + + if (!priv->stage_overlay) + return FALSE; + + return meta_overlay_is_visible (priv->stage_overlay); +} + void meta_cursor_renderer_add_hw_cursor_inhibitor (MetaCursorRenderer *renderer, MetaHwCursorInhibitor *inhibitor) diff --git a/src/backends/meta-cursor-renderer.h b/src/backends/meta-cursor-renderer.h index 15b2bfca6..386d029d0 100644 --- a/src/backends/meta-cursor-renderer.h +++ b/src/backends/meta-cursor-renderer.h @@ -67,6 +67,8 @@ void meta_cursor_renderer_force_update (MetaCursorRenderer *renderer); MetaCursorSprite * meta_cursor_renderer_get_cursor (MetaCursorRenderer *renderer); +gboolean meta_cursor_renderer_is_overlay_visible (MetaCursorRenderer *renderer); + void meta_cursor_renderer_add_hw_cursor_inhibitor (MetaCursorRenderer *renderer, MetaHwCursorInhibitor *inhibitor); diff --git a/src/backends/meta-screen-cast-area-stream-src.c b/src/backends/meta-screen-cast-area-stream-src.c index d1fe0ae24..c5521422c 100644 --- a/src/backends/meta-screen-cast-area-stream-src.c +++ b/src/backends/meta-screen-cast-area-stream-src.c @@ -343,6 +343,7 @@ meta_screen_cast_area_stream_src_enable (MetaScreenCastStreamSrc *src) g_signal_connect_after (cursor_tracker, "cursor-changed", G_CALLBACK (cursor_changed), area_src); + meta_cursor_tracker_track_position (cursor_tracker); G_GNUC_FALLTHROUGH; case META_SCREEN_CAST_CURSOR_MODE_HIDDEN: add_view_painted_watches (area_src, @@ -350,6 +351,7 @@ meta_screen_cast_area_stream_src_enable (MetaScreenCastStreamSrc *src) break; case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED: inhibit_hw_cursor (area_src); + meta_cursor_tracker_track_position (cursor_tracker); add_view_painted_watches (area_src, META_STAGE_WATCH_AFTER_ACTOR_PAINT); break; @@ -363,6 +365,7 @@ meta_screen_cast_area_stream_src_disable (MetaScreenCastStreamSrc *src) { MetaScreenCastAreaStreamSrc *area_src = META_SCREEN_CAST_AREA_STREAM_SRC (src); + MetaScreenCastStream *stream = meta_screen_cast_stream_src_get_stream (src); MetaBackend *backend = get_backend (area_src); MetaCursorTracker *cursor_tracker = meta_backend_get_cursor_tracker (backend); ClutterStage *stage; @@ -389,6 +392,16 @@ meta_screen_cast_area_stream_src_disable (MetaScreenCastStreamSrc *src) cursor_tracker); g_clear_handle_id (&area_src->maybe_record_idle_id, g_source_remove); + + switch (meta_screen_cast_stream_get_cursor_mode (stream)) + { + case META_SCREEN_CAST_CURSOR_MODE_METADATA: + case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED: + meta_cursor_tracker_untrack_position (cursor_tracker); + break; + case META_SCREEN_CAST_CURSOR_MODE_HIDDEN: + break; + } } static gboolean @@ -418,6 +431,7 @@ meta_screen_cast_area_stream_src_record_to_buffer (MetaScreenCastStreamSrc *src paint_flags |= CLUTTER_PAINT_FLAG_NO_CURSORS; break; case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED: + paint_flags |= CLUTTER_PAINT_FLAG_FORCE_CURSORS; break; } @@ -458,6 +472,7 @@ meta_screen_cast_area_stream_src_record_to_framebuffer (MetaScreenCastStreamSrc paint_flags |= CLUTTER_PAINT_FLAG_NO_CURSORS; break; case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED: + paint_flags |= CLUTTER_PAINT_FLAG_FORCE_CURSORS; break; } clutter_stage_paint_to_framebuffer (stage, framebuffer, diff --git a/src/backends/meta-screen-cast-monitor-stream-src.c b/src/backends/meta-screen-cast-monitor-stream-src.c index df8029aa9..cb6e81b28 100644 --- a/src/backends/meta-screen-cast-monitor-stream-src.c +++ b/src/backends/meta-screen-cast-monitor-stream-src.c @@ -343,6 +343,7 @@ meta_screen_cast_monitor_stream_src_enable (MetaScreenCastStreamSrc *src) g_signal_connect_after (cursor_tracker, "cursor-changed", G_CALLBACK (cursor_changed), monitor_src); + meta_cursor_tracker_track_position (cursor_tracker); G_GNUC_FALLTHROUGH; case META_SCREEN_CAST_CURSOR_MODE_HIDDEN: add_view_painted_watches (monitor_src, @@ -350,6 +351,7 @@ meta_screen_cast_monitor_stream_src_enable (MetaScreenCastStreamSrc *src) break; case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED: inhibit_hw_cursor (monitor_src); + meta_cursor_tracker_track_position (cursor_tracker); add_view_painted_watches (monitor_src, META_STAGE_WATCH_AFTER_PAINT); break; @@ -363,6 +365,7 @@ meta_screen_cast_monitor_stream_src_disable (MetaScreenCastStreamSrc *src) { MetaScreenCastMonitorStreamSrc *monitor_src = META_SCREEN_CAST_MONITOR_STREAM_SRC (src); + MetaScreenCastStream *stream = meta_screen_cast_stream_src_get_stream (src); MetaBackend *backend = get_backend (monitor_src); MetaCursorTracker *cursor_tracker = meta_backend_get_cursor_tracker (backend); ClutterStage *stage; @@ -387,6 +390,79 @@ meta_screen_cast_monitor_stream_src_disable (MetaScreenCastStreamSrc *src) cursor_tracker); g_clear_signal_handler (&monitor_src->cursor_changed_handler_id, cursor_tracker); + + switch (meta_screen_cast_stream_get_cursor_mode (stream)) + { + case META_SCREEN_CAST_CURSOR_MODE_METADATA: + case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED: + meta_cursor_tracker_untrack_position (cursor_tracker); + break; + case META_SCREEN_CAST_CURSOR_MODE_HIDDEN: + break; + } +} + +static void +maybe_paint_cursor_sprite (MetaScreenCastMonitorStreamSrc *monitor_src, + uint8_t *data) +{ + MetaScreenCastStreamSrc *src = META_SCREEN_CAST_STREAM_SRC (monitor_src); + MetaBackend *backend = get_backend (monitor_src); + MetaCursorRenderer *cursor_renderer = + meta_backend_get_cursor_renderer (backend); + MetaCursorSprite *cursor_sprite; + CoglTexture *sprite_texture; + int sprite_width, sprite_height, sprite_stride; + float sprite_scale; + uint8_t *sprite_data; + cairo_surface_t *sprite_surface; + graphene_rect_t sprite_rect; + int width, height, stride; + cairo_surface_t *surface; + cairo_t *cr; + + cursor_sprite = meta_cursor_renderer_get_cursor (cursor_renderer); + if (!cursor_sprite) + return; + + if (meta_cursor_renderer_is_overlay_visible (cursor_renderer)) + return; + + sprite_rect = meta_cursor_renderer_calculate_rect (cursor_renderer, + cursor_sprite); + sprite_texture = meta_cursor_sprite_get_cogl_texture (cursor_sprite); + sprite_width = cogl_texture_get_width (sprite_texture); + sprite_height = cogl_texture_get_height (sprite_texture); + sprite_stride = sprite_width * 4; + sprite_scale = meta_cursor_sprite_get_texture_scale (cursor_sprite); + sprite_data = g_new0 (uint8_t, sprite_stride * sprite_height); + cogl_texture_get_data (sprite_texture, + CLUTTER_CAIRO_FORMAT_ARGB32, + sprite_stride, + sprite_data); + sprite_surface = cairo_image_surface_create_for_data (sprite_data, + CAIRO_FORMAT_ARGB32, + sprite_width, + sprite_height, + sprite_stride); + cairo_surface_set_device_scale (sprite_surface, sprite_scale, sprite_scale); + + stride = meta_screen_cast_stream_src_get_stride (src); + width = meta_screen_cast_stream_src_get_width (src); + height = meta_screen_cast_stream_src_get_height (src); + surface = cairo_image_surface_create_for_data (data, + CAIRO_FORMAT_ARGB32, + width, height, stride); + + cr = cairo_create (surface); + cairo_set_source_surface (cr, sprite_surface, + sprite_rect.origin.x, + sprite_rect.origin.y); + cairo_paint (cr); + cairo_destroy (cr); + cairo_surface_destroy (sprite_surface); + cairo_surface_destroy (surface); + g_free (sprite_data); } static gboolean @@ -396,6 +472,7 @@ meta_screen_cast_monitor_stream_src_record_to_buffer (MetaScreenCastStreamSrc * { MetaScreenCastMonitorStreamSrc *monitor_src = META_SCREEN_CAST_MONITOR_STREAM_SRC (src); + MetaScreenCastStream *stream = meta_screen_cast_stream_src_get_stream (src); ClutterStage *stage; MetaMonitor *monitor; MetaLogicalMonitor *logical_monitor; @@ -405,6 +482,16 @@ meta_screen_cast_monitor_stream_src_record_to_buffer (MetaScreenCastStreamSrc * stage = get_stage (monitor_src); clutter_stage_capture_into (stage, FALSE, &logical_monitor->rect, data); + switch (meta_screen_cast_stream_get_cursor_mode (stream)) + { + case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED: + maybe_paint_cursor_sprite (monitor_src, data); + break; + case META_SCREEN_CAST_CURSOR_MODE_METADATA: + case META_SCREEN_CAST_CURSOR_MODE_HIDDEN: + break; + } + return TRUE; } diff --git a/src/backends/meta-screen-cast-stream-src.c b/src/backends/meta-screen-cast-stream-src.c index 45eb3852f..198cc6895 100644 --- a/src/backends/meta-screen-cast-stream-src.c +++ b/src/backends/meta-screen-cast-stream-src.c @@ -1058,6 +1058,24 @@ meta_screen_cast_stream_src_get_stride (MetaScreenCastStreamSrc *src) return priv->video_stride; } +int +meta_screen_cast_stream_src_get_width (MetaScreenCastStreamSrc *src) +{ + MetaScreenCastStreamSrcPrivate *priv = + meta_screen_cast_stream_src_get_instance_private (src); + + return priv->stream_width; +} + +int +meta_screen_cast_stream_src_get_height (MetaScreenCastStreamSrc *src) +{ + MetaScreenCastStreamSrcPrivate *priv = + meta_screen_cast_stream_src_get_instance_private (src); + + return priv->stream_height; +} + MetaScreenCastStream * meta_screen_cast_stream_src_get_stream (MetaScreenCastStreamSrc *src) { diff --git a/src/backends/meta-screen-cast-stream-src.h b/src/backends/meta-screen-cast-stream-src.h index 7e5ceace7..a2f681793 100644 --- a/src/backends/meta-screen-cast-stream-src.h +++ b/src/backends/meta-screen-cast-stream-src.h @@ -80,6 +80,10 @@ gboolean meta_screen_cast_stream_src_pending_follow_up_frame (MetaScreenCastStre int meta_screen_cast_stream_src_get_stride (MetaScreenCastStreamSrc *src); +int meta_screen_cast_stream_src_get_width (MetaScreenCastStreamSrc *src); + +int meta_screen_cast_stream_src_get_height (MetaScreenCastStreamSrc *src); + MetaScreenCastStream * meta_screen_cast_stream_src_get_stream (MetaScreenCastStreamSrc *src); gboolean meta_screen_cast_stream_src_draw_cursor_into (MetaScreenCastStreamSrc *src, diff --git a/src/backends/meta-screen-cast-window-stream-src.c b/src/backends/meta-screen-cast-window-stream-src.c index e80cd5e36..d41b6c41c 100644 --- a/src/backends/meta-screen-cast-window-stream-src.c +++ b/src/backends/meta-screen-cast-window-stream-src.c @@ -23,6 +23,7 @@ #include "backends/meta-screen-cast-window-stream-src.h" #include "backends/meta-backend-private.h" +#include "backends/meta-cursor-tracker-private.h" #include "backends/meta-screen-cast-session.h" #include "backends/meta-screen-cast-window.h" #include "backends/meta-screen-cast-window-stream.h" @@ -306,6 +307,8 @@ static void meta_screen_cast_window_stream_src_stop (MetaScreenCastWindowStreamSrc *window_src) { + MetaScreenCastStreamSrc *src = META_SCREEN_CAST_STREAM_SRC (window_src); + MetaScreenCastStream *stream = meta_screen_cast_stream_src_get_stream (src); MetaBackend *backend = get_backend (window_src); MetaCursorTracker *cursor_tracker = meta_backend_get_cursor_tracker (backend); @@ -320,6 +323,16 @@ meta_screen_cast_window_stream_src_stop (MetaScreenCastWindowStreamSrc *window_s cursor_tracker); g_clear_signal_handler (&window_src->cursor_changed_handler_id, cursor_tracker); + + switch (meta_screen_cast_stream_get_cursor_mode (stream)) + { + case META_SCREEN_CAST_CURSOR_MODE_METADATA: + case META_SCREEN_CAST_CURSOR_MODE_EMBEDDED: + meta_cursor_tracker_untrack_position (cursor_tracker); + break; + case META_SCREEN_CAST_CURSOR_MODE_HIDDEN: + break; + } } static void @@ -438,6 +451,7 @@ meta_screen_cast_window_stream_src_enable (MetaScreenCastStreamSrc *src) g_signal_connect_after (cursor_tracker, "cursor-changed", G_CALLBACK (cursor_changed), window_src); + meta_cursor_tracker_track_position (cursor_tracker); break; case META_SCREEN_CAST_CURSOR_MODE_HIDDEN: break; diff --git a/src/backends/meta-stage.c b/src/backends/meta-stage.c index af56a18f5..f2040362f 100644 --- a/src/backends/meta-stage.c +++ b/src/backends/meta-stage.c @@ -25,6 +25,7 @@ #include "backends/meta-stage-private.h" #include "backends/meta-backend-private.h" +#include "backends/meta-cursor-tracker-private.h" #include "clutter/clutter-mutter.h" #include "meta/meta-backend.h" #include "meta/meta-monitor-manager.h" @@ -127,7 +128,9 @@ meta_overlay_paint (MetaOverlay *overlay, if (!overlay->texture) return; - if (!overlay->is_visible) + if (!overlay->is_visible && + !(clutter_paint_context_get_paint_flags (paint_context) & + CLUTTER_PAINT_FLAG_FORCE_CURSORS)) return; framebuffer = clutter_paint_context_get_framebuffer (paint_context); @@ -207,10 +210,28 @@ meta_stage_paint (ClutterActor *actor, g_signal_emit (stage, signals[ACTORS_PAINTED], 0); + if ((clutter_paint_context_get_paint_flags (paint_context) & + CLUTTER_PAINT_FLAG_FORCE_CURSORS)) + { + MetaCursorTracker *cursor_tracker = + meta_backend_get_cursor_tracker (stage->backend); + + meta_cursor_tracker_track_position (cursor_tracker); + } + if (!(clutter_paint_context_get_paint_flags (paint_context) & CLUTTER_PAINT_FLAG_NO_CURSORS)) g_list_foreach (stage->overlays, (GFunc) meta_overlay_paint, paint_context); + if ((clutter_paint_context_get_paint_flags (paint_context) & + CLUTTER_PAINT_FLAG_FORCE_CURSORS)) + { + MetaCursorTracker *cursor_tracker = + meta_backend_get_cursor_tracker (stage->backend); + + meta_cursor_tracker_untrack_position (cursor_tracker); + } + if (view) { notify_watchers_for_mode (stage, view, paint_context,