From 986066ed407213f3c2ad79d8ddad3cd5bc1548e4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jonas=20=C3=85dahl?= <jadahl@gmail.com>
Date: Wed, 17 Jun 2020 17:50:17 +0200
Subject: [PATCH] screen-cast: Disable DMA buffer based screen casting for QXL

QXL doesn't support mmap():ing a DMA buffer allocated in mutter inside
the PipeWire stream consumer process. To make screen casting work again
on QXL, disable DMA buffer based screen casting for QXL.

Eventually, it should be the client that renegotiates the supported
buffer types, but until then we need this list.

https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1318
---
 src/backends/meta-screen-cast.c           | 11 +++++++
 src/backends/meta-screen-cast.h           |  2 ++
 src/backends/native/meta-backend-native.c | 40 +++++++++++++++++++++++
 3 files changed, 53 insertions(+)

diff --git a/src/backends/meta-screen-cast.c b/src/backends/meta-screen-cast.c
index 499598042..87e388163 100644
--- a/src/backends/meta-screen-cast.c
+++ b/src/backends/meta-screen-cast.c
@@ -46,6 +46,8 @@ struct _MetaScreenCast
 
   MetaDbusSessionWatcher *session_watcher;
   MetaBackend *backend;
+
+  gboolean disable_dma_bufs;
 };
 
 static void
@@ -94,6 +96,12 @@ meta_screen_cast_get_backend (MetaScreenCast *screen_cast)
   return screen_cast->backend;
 }
 
+void
+meta_screen_cast_disable_dma_bufs (MetaScreenCast *screen_cast)
+{
+  screen_cast->disable_dma_bufs = TRUE;
+}
+
 CoglDmaBufHandle *
 meta_screen_cast_create_dma_buf_handle (MetaScreenCast *screen_cast,
                                         int             width,
@@ -107,6 +115,9 @@ meta_screen_cast_create_dma_buf_handle (MetaScreenCast *screen_cast,
   g_autoptr (GError) error = NULL;
   CoglDmaBufHandle *dmabuf_handle;
 
+  if (screen_cast->disable_dma_bufs)
+    return NULL;
+
   dmabuf_handle = cogl_renderer_create_dma_buf (cogl_renderer,
                                                 width, height,
                                                 &error);
diff --git a/src/backends/meta-screen-cast.h b/src/backends/meta-screen-cast.h
index bc1d3d29c..e2ea5a5e4 100644
--- a/src/backends/meta-screen-cast.h
+++ b/src/backends/meta-screen-cast.h
@@ -56,6 +56,8 @@ GDBusConnection * meta_screen_cast_get_connection (MetaScreenCast *screen_cast);
 
 MetaBackend * meta_screen_cast_get_backend (MetaScreenCast *screen_cast);
 
+void meta_screen_cast_disable_dma_bufs (MetaScreenCast *screen_cast);
+
 CoglDmaBufHandle * meta_screen_cast_create_dma_buf_handle (MetaScreenCast *screen_cast,
                                                            int             width,
                                                            int             height);
diff --git a/src/backends/native/meta-backend-native.c b/src/backends/native/meta-backend-native.c
index 1808c6df2..cc7fc532d 100644
--- a/src/backends/native/meta-backend-native.c
+++ b/src/backends/native/meta-backend-native.c
@@ -64,6 +64,10 @@
 #include "core/meta-border.h"
 #include "meta/main.h"
 
+#ifdef HAVE_REMOTE_DESKTOP
+#include "backends/meta-screen-cast.h"
+#endif
+
 struct _MetaBackendNative
 {
   MetaBackend parent;
@@ -338,6 +342,38 @@ meta_backend_native_create_clutter_backend (MetaBackend *backend)
   return g_object_new (META_TYPE_CLUTTER_BACKEND_NATIVE, NULL);
 }
 
+#ifdef HAVE_REMOTE_DESKTOP
+static void
+maybe_disable_screen_cast_dma_bufs (MetaBackendNative *native)
+{
+  MetaBackend *backend = META_BACKEND (native);
+  MetaRenderer *renderer = meta_backend_get_renderer (backend);
+  MetaRendererNative *renderer_native = META_RENDERER_NATIVE (renderer);
+  MetaGpuKms *primary_gpu;
+  MetaKmsDevice *kms_device;
+  const char *driver_name;
+  static const char *disable_dma_buf_drivers[] = {
+    "qxl",
+    NULL,
+  };
+
+  primary_gpu = meta_renderer_native_get_primary_gpu (renderer_native);
+  kms_device = meta_gpu_kms_get_kms_device (primary_gpu);
+  driver_name = meta_kms_device_get_driver_name (kms_device);
+
+  if (g_strv_contains (disable_dma_buf_drivers, driver_name))
+    {
+      MetaScreenCast *screen_cast = meta_backend_get_screen_cast (backend);
+
+      g_message ("The '%s' driver doesn't support DMA buffer screen sharing, disabling.",
+                 driver_name);
+
+      meta_screen_cast_disable_dma_bufs (screen_cast);
+      return;
+    }
+}
+#endif /* HAVE_REMOTE_DESKTOP */
+
 static void
 meta_backend_native_post_init (MetaBackend *backend)
 {
@@ -368,6 +404,10 @@ meta_backend_native_post_init (MetaBackend *backend)
         g_warning ("Failed to set RT scheduler: %m");
     }
 
+#ifdef HAVE_REMOTE_DESKTOP
+  maybe_disable_screen_cast_dma_bufs (META_BACKEND_NATIVE (backend));
+#endif
+
 #ifdef HAVE_WAYLAND
   meta_backend_init_wayland (backend);
 #endif