diff --git a/src/wayland/meta-wayland-private.h b/src/wayland/meta-wayland-private.h
index 46a2470a7..82a256f60 100644
--- a/src/wayland/meta-wayland-private.h
+++ b/src/wayland/meta-wayland-private.h
@@ -56,6 +56,9 @@ typedef struct
   MetaXWaylandConnection private_connection;
   MetaXWaylandConnection public_connection;
 
+  guint abstract_fd_watch_id;
+  guint unix_fd_watch_id;
+
   guint xserver_grace_period_id;
   struct wl_display *wayland_display;
   struct wl_client *client;
diff --git a/src/wayland/meta-xwayland.c b/src/wayland/meta-xwayland.c
index 783a58d35..1343a7d05 100644
--- a/src/wayland/meta-xwayland.c
+++ b/src/wayland/meta-xwayland.c
@@ -990,11 +990,16 @@ xdisplay_connection_activity_cb (gint         fd,
                                  GIOCondition cond,
                                  gpointer     user_data)
 {
+  MetaXWaylandManager *manager = user_data;
   MetaDisplay *display = meta_get_display ();
 
   meta_display_init_x11 (display, NULL,
                          (GAsyncReadyCallback) on_init_x11_cb, NULL);
 
+  /* Stop watching both file descriptors */
+  g_clear_handle_id (&manager->abstract_fd_watch_id, g_source_remove);
+  g_clear_handle_id (&manager->unix_fd_watch_id, g_source_remove);
+
   return G_SOURCE_REMOVE;
 }
 
@@ -1098,8 +1103,12 @@ meta_xwayland_init (MetaXWaylandManager  *manager,
 
   if (policy == META_DISPLAY_POLICY_ON_DEMAND)
     {
-      g_unix_fd_add (manager->public_connection.abstract_fd, G_IO_IN,
-                     xdisplay_connection_activity_cb, manager);
+      manager->abstract_fd_watch_id =
+        g_unix_fd_add (manager->public_connection.abstract_fd, G_IO_IN,
+                       xdisplay_connection_activity_cb, manager);
+      manager->unix_fd_watch_id =
+        g_unix_fd_add (manager->public_connection.unix_fd, G_IO_IN,
+                       xdisplay_connection_activity_cb, manager);
     }
 
   return TRUE;