wayland/buffer: Only query Wayland EGL buffer if display bound

It's not allowed to call eglQueryWaylandBuffer() if the call to
eglBindWaylandDisplay() failed, and will result in an assert being hit
in mesa if called.

Avoid that by keeping track whether we succeeded to bind, and only
attempt to realize a legacy EGL wl_buffer if binding succeeded.

Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/2415>

src/wayland/meta-wayland-buffer.c

@@ -94,7 +94,8 @@ meta_wayland_buffer_destroy_handler (struct wl_listener *listener,
 }
 
 MetaWaylandBuffer *
-meta_wayland_buffer_from_resource (struct wl_resource *resource)
+meta_wayland_buffer_from_resource (MetaWaylandCompositor *compositor,
+                                   struct wl_resource    *resource)
 {
   MetaWaylandBuffer *buffer;
   struct wl_listener *listener;
@@ -112,6 +113,7 @@ meta_wayland_buffer_from_resource (struct wl_resource *resource)
       buffer = g_object_new (META_TYPE_WAYLAND_BUFFER, NULL);
 
       buffer->resource = resource;
+      buffer->compositor = compositor;
       buffer->destroy_listener.notify = meta_wayland_buffer_destroy_handler;
       wl_resource_add_destroy_listener (resource, &buffer->destroy_listener);
     }
@@ -134,12 +136,6 @@ meta_wayland_buffer_is_realized (MetaWaylandBuffer *buffer)
 gboolean
 meta_wayland_buffer_realize (MetaWaylandBuffer *buffer)
 {
-  EGLint format;
-  MetaBackend *backend = meta_get_backend ();
-  MetaEgl *egl = meta_backend_get_egl (backend);
-  ClutterBackend *clutter_backend = meta_backend_get_clutter_backend (backend);
-  CoglContext *cogl_context = clutter_backend_get_cogl_context (clutter_backend);
-  EGLDisplay egl_display = cogl_egl_context_get_egl_display (cogl_context);
 #ifdef HAVE_WAYLAND_EGLSTREAM
   MetaWaylandEglStream *stream;
 #endif
@@ -170,12 +166,24 @@ meta_wayland_buffer_realize (MetaWaylandBuffer *buffer)
     }
 #endif /* HAVE_WAYLAND_EGLSTREAM */
 
-  if (meta_egl_query_wayland_buffer (egl, egl_display, buffer->resource,
+  if (meta_wayland_compositor_is_egl_display_bound (buffer->compositor))
-                                     EGL_TEXTURE_FORMAT, &format,
-                                     NULL))
     {
-      buffer->type = META_WAYLAND_BUFFER_TYPE_EGL_IMAGE;
+      EGLint format;
-      return TRUE;
+      MetaBackend *backend = meta_get_backend ();
+      MetaEgl *egl = meta_backend_get_egl (backend);
+      ClutterBackend *clutter_backend =
+        meta_backend_get_clutter_backend (backend);
+      CoglContext *cogl_context =
+        clutter_backend_get_cogl_context (clutter_backend);
+      EGLDisplay egl_display = cogl_egl_context_get_egl_display (cogl_context);
+
+      if (meta_egl_query_wayland_buffer (egl, egl_display, buffer->resource,
+                                         EGL_TEXTURE_FORMAT, &format,
+                                         NULL))
+        {
+          buffer->type = META_WAYLAND_BUFFER_TYPE_EGL_IMAGE;
+          return TRUE;
+        }
     }
 
   dma_buf = meta_wayland_dma_buf_from_buffer (buffer);

src/wayland/meta-wayland-buffer.h

@@ -48,6 +48,7 @@ struct _MetaWaylandBuffer
 {
   GObject parent;
 
+  MetaWaylandCompositor *compositor;
   struct wl_resource *resource;
   struct wl_listener destroy_listener;
 
@@ -76,7 +77,8 @@ struct _MetaWaylandBuffer
 G_DECLARE_FINAL_TYPE (MetaWaylandBuffer, meta_wayland_buffer,
                       META, WAYLAND_BUFFER, GObject);
 
-MetaWaylandBuffer *     meta_wayland_buffer_from_resource       (struct wl_resource    *resource);
+MetaWaylandBuffer *     meta_wayland_buffer_from_resource       (MetaWaylandCompositor *compositor,
+                                                                 struct wl_resource    *resource);
 struct wl_resource *    meta_wayland_buffer_get_resource        (MetaWaylandBuffer     *buffer);
 gboolean                meta_wayland_buffer_is_realized         (MetaWaylandBuffer     *buffer);
 gboolean                meta_wayland_buffer_realize             (MetaWaylandBuffer     *buffer);

src/wayland/meta-wayland-dma-buf.c

@@ -133,6 +133,8 @@ struct _MetaWaylandDmaBufBuffer
 {
   GObject parent;
 
+  MetaWaylandDmaBufManager *manager;
+
   int width;
   int height;
   uint32_t drm_format;
@@ -783,7 +785,8 @@ buffer_params_create_common (struct wl_client   *client,
     wl_resource_create (client, &wl_buffer_interface, 1, buffer_id);
   wl_resource_set_implementation (buffer_resource, &dma_buf_buffer_impl,
                                   dma_buf, NULL);
-  buffer = meta_wayland_buffer_from_resource (buffer_resource);
+  buffer = meta_wayland_buffer_from_resource (dma_buf->manager->compositor,
+                                              buffer_resource);
 
   meta_wayland_buffer_realize (buffer);
   if (!meta_wayland_dma_buf_realize_texture (buffer, &error))
@@ -857,10 +860,13 @@ dma_buf_handle_create_buffer_params (struct wl_client   *client,
                                      struct wl_resource *dma_buf_resource,
                                      uint32_t            params_id)
 {
+  MetaWaylandDmaBufManager *dma_buf_manager =
+    wl_resource_get_user_data (dma_buf_resource);
   struct wl_resource *params_resource;
   MetaWaylandDmaBufBuffer *dma_buf;
 
   dma_buf = g_object_new (META_TYPE_WAYLAND_DMA_BUF_BUFFER, NULL);
+  dma_buf->manager = dma_buf_manager;
 
   params_resource =
     wl_resource_create (client,

src/wayland/meta-wayland-egl-stream.c

@@ -46,7 +46,10 @@ attach_eglstream_consumer (struct wl_client   *client,
                            struct wl_resource *wl_surface,
                            struct wl_resource *wl_eglstream)
 {
-  MetaWaylandBuffer *buffer = meta_wayland_buffer_from_resource (wl_eglstream);
+  MetaWaylandCompositor *compositor = wl_resource_get_user_data (wl_eglstream);
+  MetaWaylandBuffer *buffer;
+
+  buffer = meta_wayland_buffer_from_resource (compositor, wl_eglstream);
 
   if (!meta_wayland_buffer_is_realized (buffer))
     meta_wayland_buffer_realize (buffer);
@@ -111,7 +114,7 @@ meta_wayland_eglstream_controller_init (MetaWaylandCompositor *compositor)
 
   if (wl_global_create (compositor->wayland_display,
                         wl_eglstream_controller_interface_ptr, 1,
-                        NULL,
+                        compositor,
                         bind_eglstream_controller) == NULL)
     goto fail;
 

src/wayland/meta-wayland-private.h

@@ -104,4 +104,6 @@ struct _MetaWaylandCompositor
 G_DECLARE_FINAL_TYPE (MetaWaylandCompositor, meta_wayland_compositor,
                       META, WAYLAND_COMPOSITOR, GObject)
 
+gboolean meta_wayland_compositor_is_egl_display_bound (MetaWaylandCompositor *compositor);
+
 #endif /* META_WAYLAND_PRIVATE_H */

src/wayland/meta-wayland-surface.c

@@ -1046,11 +1046,12 @@ wl_surface_attach (struct wl_client   *client,
 {
   MetaWaylandSurface *surface =
     wl_resource_get_user_data (surface_resource);
+  MetaWaylandCompositor *compositor = surface->compositor;
   MetaWaylandSurfaceState *pending = surface->pending_state;
   MetaWaylandBuffer *buffer;
 
   if (buffer_resource)
-    buffer = meta_wayland_buffer_from_resource (buffer_resource);
+    buffer = meta_wayland_buffer_from_resource (compositor, buffer_resource);
   else
     buffer = NULL;
 

src/wayland/meta-wayland.c

@@ -57,7 +57,13 @@
 
 static char *_display_name_override;
 
-G_DEFINE_TYPE (MetaWaylandCompositor, meta_wayland_compositor, G_TYPE_OBJECT)
+typedef struct _MetaWaylandCompositorPrivate
+{
+  gboolean is_wayland_egl_display_bound;
+} MetaWaylandCompositorPrivate;
+
+G_DEFINE_TYPE_WITH_PRIVATE (MetaWaylandCompositor, meta_wayland_compositor,
+                            G_TYPE_OBJECT)
 
 MetaWaylandCompositor *
 meta_wayland_compositor_get_default (void)
@@ -504,6 +510,8 @@ meta_wayland_get_xwayland_auth_file (MetaWaylandCompositor *compositor)
 static void
 meta_wayland_init_egl (MetaWaylandCompositor *compositor)
 {
+  MetaWaylandCompositorPrivate *priv =
+    meta_wayland_compositor_get_instance_private (compositor);
   MetaBackend *backend = meta_get_backend ();
   MetaEgl *egl = meta_backend_get_egl (backend);
   ClutterBackend *clutter_backend = meta_backend_get_clutter_backend (backend);
@@ -524,10 +532,12 @@ meta_wayland_init_egl (MetaWaylandCompositor *compositor)
   meta_topic (META_DEBUG_WAYLAND,
               "Binding Wayland EGL display");
 
-  if (!meta_egl_bind_wayland_display (egl,
+  if (meta_egl_bind_wayland_display (egl,
-                                      egl_display,
+                                     egl_display,
-                                      compositor->wayland_display,
+                                     compositor->wayland_display,
-                                      &error))
+                                     &error))
+    priv->is_wayland_egl_display_bound = TRUE;
+  else
     g_warning ("Failed to bind Wayland display: %s", error->message);
 }
 
@@ -801,3 +811,12 @@ meta_wayland_compositor_notify_surface_id (MetaWaylandCompositor *compositor,
       meta_wayland_compositor_remove_surface_association (compositor, id);
     }
 }
+
+gboolean
+meta_wayland_compositor_is_egl_display_bound (MetaWaylandCompositor *compositor)
+{
+  MetaWaylandCompositorPrivate *priv =
+    meta_wayland_compositor_get_instance_private (compositor);
+
+  return priv->is_wayland_egl_display_bound;
+}