From 9d328001d1ed908380a45129eb65deb7037102f7 Mon Sep 17 00:00:00 2001
From: columbarius <co1umbarius@protonmail.com>
Date: Sat, 6 Aug 2022 18:39:19 +0200
Subject: [PATCH] screen-cast-stream-src: Renegotiate when DMABUF allocation
 fails

After negotiation of DMABUF transport mutter will silently allocate SHM
buffers if the allocation in the add_buffer callback fails. It's cleaner
to renegotiate the supported formats without announcing DMABUF
capabilities in this case.

Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/2557>
---
 src/backends/meta-screen-cast-stream-src.c | 47 +++++++++++++---------
 1 file changed, 27 insertions(+), 20 deletions(-)

diff --git a/src/backends/meta-screen-cast-stream-src.c b/src/backends/meta-screen-cast-stream-src.c
index fb7c66b12..af76fa73f 100644
--- a/src/backends/meta-screen-cast-stream-src.c
+++ b/src/backends/meta-screen-cast-stream-src.c
@@ -1064,6 +1064,21 @@ build_format_params (MetaScreenCastStreamSrc *src,
   return n_params;
 }
 
+static void
+renegotiate_pipewire_stream (MetaScreenCastStreamSrc *src)
+{
+  MetaScreenCastStreamSrcPrivate *priv =
+    meta_screen_cast_stream_src_get_instance_private (src);
+  uint8_t buffer[1024];
+  struct spa_pod_builder pod_builder;
+  const struct spa_pod *params[4];
+  int n_params;
+
+  pod_builder = SPA_POD_BUILDER_INIT (buffer, sizeof (buffer));
+  n_params = build_format_params (src, &pod_builder, params);
+  pw_stream_update_params (priv->pipewire_stream, params, n_params);
+}
+
 static void
 on_stream_state_changed (void                 *data,
                          enum pw_stream_state  old,
@@ -1145,9 +1160,10 @@ on_stream_param_changed (void                 *data,
 
   pod_builder = SPA_POD_BUILDER_INIT (params_buffer, sizeof (params_buffer));
 
-  buffer_types = 1 << SPA_DATA_MemFd;
   if (spa_pod_find_prop (format, NULL, SPA_FORMAT_VIDEO_modifier))
-    buffer_types |= 1 << SPA_DATA_DmaBuf;
+    buffer_types = 1 << SPA_DATA_DmaBuf;
+  else
+    buffer_types = 1 << SPA_DATA_MemFd;
 
   params[n_params++] = spa_pod_builder_add_object (
     &pod_builder,
@@ -1210,34 +1226,23 @@ on_stream_add_buffer (void             *data,
 
       if (!cogl_pixel_format_from_spa_video_format (priv->video_format.format,
                                                     &cogl_format))
-        {
-          cogl_format = DEFAULT_COGL_PIXEL_FORMAT;
-        }
+        g_assert_not_reached ();
 
       dmabuf_handle =
         meta_screen_cast_create_dma_buf_handle (screen_cast,
                                                 cogl_format,
                                                 priv->video_format.size.width,
                                                 priv->video_format.size.height);
-
-      if (!dmabuf_handle && cogl_format != DEFAULT_COGL_PIXEL_FORMAT)
+      if (!dmabuf_handle)
         {
-          dmabuf_handle =
-            meta_screen_cast_create_dma_buf_handle (screen_cast,
-                                                    DEFAULT_COGL_PIXEL_FORMAT,
-                                                    priv->video_format.size.width,
-                                                    priv->video_format.size.height);
+          // TODO: Drop dmabuf support more granular
+          meta_screen_cast_disable_dma_bufs (screen_cast);
+          renegotiate_pipewire_stream (src);
+          return;
         }
-    }
-  else
-    {
-      dmabuf_handle = NULL;
-    }
 
-  priv->uses_dma_bufs = !!dmabuf_handle;
+      priv->uses_dma_bufs = TRUE;
 
-  if (dmabuf_handle)
-    {
       meta_topic (META_DEBUG_SCREEN_CAST,
                   "Allocating DMA buffer for pw_stream %u",
                   pw_stream_get_node_id (priv->pipewire_stream));
@@ -1257,6 +1262,8 @@ on_stream_add_buffer (void             *data,
     {
       unsigned int seals;
 
+      priv->uses_dma_bufs = FALSE;
+
       if (!(spa_data->type & (1 << SPA_DATA_MemFd)))
         {
           g_critical ("No supported PipeWire stream buffer data type could "