wayland: Add X11 interop service client protocol

This protocol is intended to let special clients create transient-for
relationships between X11 and Wayland windows. The client that needs
this is xdg-desktop-portal-gnome, which will create e.g. file chooser
Wayland dialogs that should be mapped on top of X11 windows.

Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/2810>

src/meson.build

@@ -684,6 +684,8 @@ if have_wayland
 
   if have_xwayland
     mutter_sources += [
+      'wayland/meta-wayland-x11-interop.c',
+      'wayland/meta-wayland-x11-interop.h',
       'wayland/meta-window-xwayland.c',
       'wayland/meta-window-xwayland.h',
       'wayland/meta-xwayland.c',
@@ -1025,6 +1027,11 @@ if have_wayland
       ['wayland-eglstream-controller', 'third-party', wayland_eglstream_protocols_dir],
     ]
   endif
+  if have_xwayland
+    wayland_protocols += [
+      ['mutter-x11-interop', 'private', ],
+    ]
+  endif
 
   wayland_scanner = find_program('wayland-scanner')
   protocols_dir = wayland_protocols_dep.get_variable('pkgdatadir')

src/tests/meson.build

@@ -437,6 +437,17 @@ if have_native_tests
         wayland_test_utils,
       ],
     },
+    {
+      'name': 'wayland-x11-interop',
+      'suite': 'wayland',
+      'sources': [
+        'wayland-x11-interop-tests.c',
+        wayland_test_client_utils,
+        wayland_test_utils,
+        wayland_protocol_files['mutter-x11-interop']['protocol-code'],
+        built_dbus_sources['meta-dbus-service-channel'],
+      ],
+    },
   ]
   if have_xwayland
     test_cases += [

src/tests/wayland-test-clients/meson.build

@@ -3,10 +3,13 @@ wayland_test_client_installed_tests_libexecdir = join_paths(
   'wayland-test-clients',
 )
 
-
+wayland_test_client_sources = files([
-common_sources = [
   'wayland-test-client-utils.c',
   'wayland-test-client-utils.h',
+])
+
+wayland_test_client_utils = [
+  wayland_test_client_sources,
   wayland_protocol_client_headers,
   wayland_protocol_sources,
   test_driver_client_header,
@@ -81,7 +84,7 @@ foreach test : wayland_test_clients
 
   test_client_sources = [
     '@0@.c'.format(test_name),
-    common_sources,
+    wayland_test_client_utils,
   ]
   if test.has_key('extra_sources')
     test_client_sources += test['extra_sources']

src/tests/wayland-x11-interop-tests.c

@@ -0,0 +1,297 @@
+/*
+ * Copyright (C) 2023 Red Hat Inc.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
+ * 02111-1307, USA.
+ *
+ */
+
+#include "config.h"
+
+#include <wayland-client.h>
+
+#include "meta-test/meta-context-test.h"
+#include "tests/meta-test-utils.h"
+#include "tests/meta-wayland-test-driver.h"
+#include "tests/meta-wayland-test-utils.h"
+#include "tests/wayland-test-clients/wayland-test-client-utils.h"
+#include "wayland/meta-wayland-client-private.h"
+#include "wayland/meta-window-wayland.h"
+#include "x11/window-x11.h"
+
+#include "mutter-x11-interop-client-protocol.h"
+#include "mutter-x11-interop-server-protocol.h"
+
+typedef enum _ServiceClientType
+{
+  SERVICE_CLIENT_TYPE_NONE,
+  SERVICE_CLIENT_TYPE_PORTAL_BACKEND,
+} ServiceClientType;
+
+static MetaContext *test_context;
+static MetaWaylandTestDriver *test_driver;
+static MetaVirtualMonitor *virtual_monitor;
+
+static void
+handle_registry_global (void               *user_data,
+                        struct wl_registry *registry,
+                        uint32_t            id,
+                        const char         *interface,
+                        uint32_t            version)
+{
+  struct mutter_x11_interop **x11_interop = user_data;
+
+  if (strcmp (interface, mutter_x11_interop_interface.name) == 0)
+    {
+      *x11_interop = wl_registry_bind (registry, id,
+                                       &mutter_x11_interop_interface, 1);
+    }
+}
+
+static void
+handle_registry_global_remove (void               *user_data,
+                               struct wl_registry *registry,
+                               uint32_t            name)
+{
+}
+
+static const struct wl_registry_listener registry_listener = {
+  handle_registry_global,
+  handle_registry_global_remove
+};
+
+static struct mutter_x11_interop *
+get_x11_interop (WaylandDisplay *display)
+{
+  struct wl_registry *registry;
+  struct mutter_x11_interop *x11_interop = NULL;
+
+  registry = wl_display_get_registry (display->display);
+  wl_registry_add_listener (registry, &registry_listener, &x11_interop);
+  wl_display_roundtrip (display->display);
+
+  return x11_interop;
+}
+
+static gpointer
+regular_client_thread_func (gpointer user_data)
+{
+  gboolean *client_terminated = user_data;
+  WaylandDisplay *display;
+  struct mutter_x11_interop *x11_interop = NULL;
+
+  display = wayland_display_new (WAYLAND_DISPLAY_CAPABILITY_NONE);
+
+  x11_interop = get_x11_interop (display);
+  g_assert_null (x11_interop);
+
+  g_object_unref (display);
+
+  g_atomic_int_set (client_terminated, TRUE);
+  return NULL;
+}
+
+static void
+meta_test_wayland_client_x11_interop_hidden_by_default (void)
+{
+  g_autoptr (GThread) thread = NULL;
+  gboolean client_terminated = FALSE;
+
+  thread = g_thread_new ("regular client thread",
+                         regular_client_thread_func,
+                         &client_terminated);
+
+  g_debug ("Waiting for client to disconnect itself");
+
+  while (!client_terminated)
+    g_main_context_iteration (NULL, TRUE);
+
+  g_debug ("Waiting for thread to terminate");
+  g_thread_join (thread);
+}
+
+typedef struct
+{
+  Window xwindow;
+  gboolean client_terminated;
+} X11ParentTestdata;
+
+static gpointer
+service_client_thread_func (gpointer user_data)
+{
+  X11ParentTestdata *data = user_data;
+  g_autoptr (GError) error = NULL;
+  g_autoptr (GDBusProxy) service_channel = NULL;
+  GVariant *service_client_type_variant;
+  g_autoptr (GVariant) result = NULL;
+  g_autoptr (GVariant) fd_variant = NULL;
+  g_autoptr (GUnixFDList) fd_list = NULL;
+  int fd;
+  struct wl_display *wayland_display;
+  WaylandDisplay *display;
+  WaylandSurface *surface;
+  struct mutter_x11_interop *x11_interop;
+
+  service_channel = g_dbus_proxy_new_for_bus_sync (G_BUS_TYPE_SESSION,
+                                                   G_DBUS_PROXY_FLAGS_NONE,
+                                                   NULL,
+                                                   "org.gnome.Mutter.ServiceChannel",
+                                                   "/org/gnome/Mutter/ServiceChannel",
+                                                   "org.gnome.Mutter.ServiceChannel",
+                                                   NULL,
+                                                   &error);
+  g_assert_nonnull (service_channel);
+
+  service_client_type_variant =
+    g_variant_new ("(u)", SERVICE_CLIENT_TYPE_PORTAL_BACKEND);
+  result =
+    g_dbus_proxy_call_with_unix_fd_list_sync (service_channel,
+                                              "OpenWaylandServiceConnection",
+                                              service_client_type_variant,
+                                              G_DBUS_CALL_FLAGS_NO_AUTO_START,
+                                              -1,
+                                              NULL,
+                                              &fd_list,
+                                              NULL, &error);
+  g_assert_nonnull (result);
+
+  g_variant_get (result, "(@h)", &fd_variant);
+  fd = g_unix_fd_list_get (fd_list, g_variant_get_handle (fd_variant), &error);
+  g_assert_cmpint (fd, >=, 0);
+
+  wayland_display = wl_display_connect_to_fd (fd);
+  g_assert_nonnull (wayland_display);
+
+  display = wayland_display_new_full (WAYLAND_DISPLAY_CAPABILITY_TEST_DRIVER,
+                                      wayland_display);
+
+  x11_interop = get_x11_interop (display);
+  g_assert_nonnull (x11_interop);
+
+  surface = wayland_surface_new (display, "test service window",
+                                 100, 100, 0xffabcdff);
+  mutter_x11_interop_set_x11_parent (x11_interop,
+                                     surface->wl_surface,
+                                     data->xwindow);
+  wl_surface_commit (surface->wl_surface);
+
+  wait_for_sync_event (display, 1);
+
+  mutter_x11_interop_destroy (x11_interop);
+  g_object_unref (display);
+
+  g_atomic_int_set (&data->client_terminated, TRUE);
+
+  return NULL;
+}
+
+static void
+meta_test_wayland_client_x11_interop_x11_parent (void)
+{
+  X11ParentTestdata data = {};
+  g_autoptr (GThread) thread = NULL;
+  g_autoptr (GError) error = NULL;
+  MetaTestClient *x11_client;
+  MetaWindow *x11_window;
+  MetaWindow *wayland_window;
+
+  x11_client = meta_test_client_new (test_context,
+                                     "x11-client",
+                                     META_WINDOW_CLIENT_TYPE_X11,
+                                     &error);
+  g_assert_nonnull (x11_client);
+  meta_test_client_run (x11_client,
+                        "create win\n"
+                        "show win\n");
+  x11_window = meta_wait_for_client_window (test_context, "test/x11-client/win");
+  g_assert_true (META_IS_WINDOW_X11 (x11_window));
+
+  g_debug ("Spawning Wayland client");
+  data = (X11ParentTestdata) {
+    .xwindow = meta_window_x11_get_toplevel_xwindow (x11_window),
+    .client_terminated = FALSE,
+  };
+  thread = g_thread_new ("service client thread",
+                         service_client_thread_func,
+                         &data);
+  wayland_window = meta_wait_for_client_window (test_context,
+                                                "test service window");
+  g_assert_true (META_IS_WINDOW_WAYLAND (wayland_window));
+
+  g_assert_true (meta_window_get_transient_for (wayland_window) ==
+                 x11_window);
+
+  meta_wayland_test_driver_emit_sync_event (test_driver, 1);
+
+  g_debug ("Waiting for client to disconnect");
+  while (!g_atomic_int_get (&data.client_terminated))
+    g_main_context_iteration (NULL, TRUE);
+
+  meta_test_client_destroy (x11_client);
+
+  g_debug ("Waiting for thread to terminate");
+  g_thread_join (thread);
+}
+
+static void
+on_before_tests (void)
+{
+  MetaWaylandCompositor *compositor =
+    meta_context_get_wayland_compositor (test_context);
+
+  test_driver = meta_wayland_test_driver_new (compositor);
+  virtual_monitor = meta_create_test_monitor (test_context,
+                                              400, 400, 60.0);
+}
+
+static void
+on_after_tests (void)
+{
+  g_clear_object (&test_driver);
+  g_clear_object (&virtual_monitor);
+}
+
+static void
+init_tests (void)
+{
+  g_test_add_func ("/wayland/client/x11-interop/hidden-by-default",
+                   meta_test_wayland_client_x11_interop_hidden_by_default);
+  g_test_add_func ("/wayland/client/x11-interop/x11-parent",
+                   meta_test_wayland_client_x11_interop_x11_parent);
+}
+
+int
+main (int    argc,
+      char **argv)
+{
+  g_autoptr (MetaContext) context = NULL;
+  g_autoptr (GError) error = NULL;
+
+  context = meta_create_test_context (META_CONTEXT_TEST_TYPE_HEADLESS,
+                                      META_CONTEXT_TEST_FLAG_TEST_CLIENT);
+  g_assert (meta_context_configure (context, &argc, &argv, NULL));
+
+  test_context = context;
+
+  init_tests ();
+
+  g_signal_connect (context, "before-tests",
+                    G_CALLBACK (on_before_tests), NULL);
+  g_signal_connect (context, "after-tests",
+                    G_CALLBACK (on_after_tests), NULL);
+
+  return meta_context_test_run_tests (META_CONTEXT_TEST (context),
+                                      META_TEST_RUN_FLAG_CAN_SKIP);
+}

src/wayland/meta-wayland-versions.h

@@ -57,5 +57,6 @@
 #define META_WP_PRESENTATION_VERSION        1
 #define META_XDG_ACTIVATION_V1_VERSION 1
 #define META_WP_SINGLE_PIXEL_BUFFER_V1_VERSION 1
+#define META_MUTTER_X11_INTEROP_VERSION 1
 
 #endif

src/wayland/meta-wayland-x11-interop.c

@@ -0,0 +1,133 @@
+/*
+ * Copyright 2023 Red Hat
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
+ * 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include "wayland/meta-wayland-x11-interop.h"
+
+#include "core/meta-service-channel.h"
+#include "core/window-private.h"
+#include "wayland/meta-wayland-client-private.h"
+#include "wayland/meta-wayland-filter-manager.h"
+#include "wayland/meta-wayland-surface.h"
+#include "wayland/meta-wayland-versions.h"
+#include "wayland/meta-wayland.h"
+#include "x11/meta-x11-display-private.h"
+
+#include "mutter-x11-interop-server-protocol.h"
+
+static void
+mutter_x11_interop_destroy (struct wl_client   *client,
+                            struct wl_resource *resource)
+{
+  wl_resource_destroy (resource);
+}
+
+static void
+mutter_x11_interop_set_x11_parent (struct wl_client   *client,
+                                   struct wl_resource *resource,
+                                   struct wl_resource *surface_resource,
+                                   uint32_t            xwindow_id)
+{
+  MetaWaylandSurface *surface = wl_resource_get_user_data (surface_resource);
+  MetaWaylandCompositor *compositor =
+    meta_wayland_surface_get_compositor (surface);
+  MetaContext *context = meta_wayland_compositor_get_context (compositor);
+  MetaDisplay *display = meta_context_get_display (context);
+  MetaX11Display *x11_display = meta_display_get_x11_display (display);
+  MetaWindow *x11_window;
+  MetaWindow *wayland_window;
+
+  if (!x11_display)
+    return;
+
+  x11_window = meta_x11_display_lookup_x_window (x11_display, xwindow_id);
+  if (!x11_window)
+    return;
+
+  wayland_window = meta_wayland_surface_get_window (surface);
+  if (!wayland_window)
+    return;
+
+  meta_window_set_transient_for (wayland_window, x11_window);
+}
+
+static const struct mutter_x11_interop_interface meta_wayland_x11_interop_interface = {
+  mutter_x11_interop_destroy,
+  mutter_x11_interop_set_x11_parent,
+};
+
+static void
+bind_x11_interop (struct wl_client *client,
+                  void             *user_data,
+                  uint32_t          version,
+                  uint32_t          id)
+{
+  MetaWaylandCompositor *compositor = user_data;
+  struct wl_resource *resource;
+
+  resource = wl_resource_create (client, &mutter_x11_interop_interface,
+                                 version, id);
+  wl_resource_set_implementation (resource,
+                                  &meta_wayland_x11_interop_interface,
+                                  compositor, NULL);
+}
+
+static MetaWaylandAccess
+x11_interop_filter (const struct wl_client *client,
+                    const struct wl_global *global,
+                    gpointer                user_data)
+{
+  MetaWaylandCompositor *compositor = user_data;
+  MetaContext *context = meta_wayland_compositor_get_context (compositor);
+  MetaServiceChannel *service_channel =
+    meta_context_get_service_channel (context);
+  MetaWaylandClient *service_client;
+
+  service_client =
+    meta_service_channel_get_service_client (service_channel,
+                                             META_SERVICE_CLIENT_TYPE_PORTAL_BACKEND);
+  if (!service_client)
+    return META_WAYLAND_ACCESS_DENIED;
+
+  if (meta_wayland_client_matches (service_client, client))
+    return META_WAYLAND_ACCESS_ALLOWED;
+  else
+    return META_WAYLAND_ACCESS_DENIED;
+}
+
+void
+meta_wayland_x11_interop_init (MetaWaylandCompositor *compositor)
+{
+  MetaWaylandFilterManager *filter_manager =
+    meta_wayland_compositor_get_filter_manager (compositor);
+  struct wl_display *wayland_display =
+    meta_wayland_compositor_get_wayland_display (compositor);
+  struct wl_global *global;
+
+  global = wl_global_create (wayland_display,
+                             &mutter_x11_interop_interface,
+                             META_MUTTER_X11_INTEROP_VERSION,
+                             compositor, bind_x11_interop);
+
+  meta_wayland_filter_manager_add_global (filter_manager,
+                                          global,
+                                          x11_interop_filter,
+                                          compositor);
+}

src/wayland/meta-wayland-x11-interop.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright 2023 Red Hat
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
+ * 02111-1307, USA.
+ */
+
+#ifndef META_WAYLAND_X11_INTEROP_H
+#define META_WAYLAND_X11_INTEROP_H
+
+#include "wayland/meta-wayland-types.h"
+
+void meta_wayland_x11_interop_init (MetaWaylandCompositor *compositor);
+
+#endif /* META_WAYLAND_X11_INTEROP_H */

src/wayland/meta-wayland.c

@@ -53,6 +53,7 @@
 #include "wayland/meta-wayland-xdg-foreign.h"
 
 #ifdef HAVE_XWAYLAND
+#include "wayland/meta-wayland-x11-interop.h"
 #include "wayland/meta-xwayland-grab-keyboard.h"
 #include "wayland/meta-xwayland-private.h"
 #include "wayland/meta-xwayland.h"
@@ -666,6 +667,8 @@ meta_wayland_compositor_new (MetaContext *context)
 #endif /* HAVE_WAYLAND_EGLSTREAM */
 
 #ifdef HAVE_XWAYLAND
+  meta_wayland_x11_interop_init (compositor);
+
   x11_display_policy =
     meta_context_get_x11_display_policy (compositor->context);
   if (x11_display_policy != META_X11_DISPLAY_POLICY_DISABLED)

src/wayland/protocol/mutter-x11-interop.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<protocol name="mutter_x11_interop">
+  <description summary="X11 interoperability helper">
+    This protocol is intended to be used by the portal backend to map Wayland
+    dialogs as modal dialogs on top of X11 windows.
+  </description>
+
+  <interface name="mutter_x11_interop" version="1">
+    <description summary="X11 interoperability helper"/>
+
+    <request name="destroy" type="destructor"/>
+
+    <request name="set_x11_parent">
+      <arg name="surface" type="object" interface="wl_surface"/>
+      <arg name="xwindow" type="uint"/>
+    </request>
+  </interface>
+</protocol>

src/x11/window-x11.h

@@ -33,6 +33,7 @@
 G_BEGIN_DECLS
 
 #define META_TYPE_WINDOW_X11 (meta_window_x11_get_type())
+META_EXPORT_TEST
 G_DECLARE_DERIVABLE_TYPE (MetaWindowX11, meta_window_x11,
                           META, WINDOW_X11, MetaWindow)
 
@@ -78,6 +79,7 @@ gboolean meta_window_x11_client_message          (MetaWindow *window,
 void     meta_window_x11_configure_notify        (MetaWindow      *window,
                                                   XConfigureEvent *event);
 
+META_EXPORT_TEST
 Window   meta_window_x11_get_toplevel_xwindow    (MetaWindow *window);
 
 void     meta_window_x11_freeze_commits          (MetaWindow *window);