diff --git a/src/core/main.c b/src/core/main.c
index e8464720f..492c74a9d 100644
--- a/src/core/main.c
+++ b/src/core/main.c
@@ -606,22 +606,6 @@ meta_init (void)
     meta_fatal ("Can't specify both SM save file and SM client id\n");
 
   meta_main_loop = g_main_loop_new (NULL, FALSE);
-
-  /*
-   * We need to make sure the first client connecting to the X server
-   * (e.g. Xwayland started from meta_wayland_init() above) is a permanent one,
-   * so prepare the GDK X11 connection now already. Without doing this, if
-   * there are any functionality that relies on X11 after here before
-   * meta_display_open(), the X server will terminate itself when such a client
-   * disconnects before the permanent GDK client connects.
-   */
-  if (meta_should_autostart_x11_display ())
-    {
-      GError *error = NULL;
-
-      if (!meta_x11_init_gdk_display (&error))
-        g_error ("Failed to open X11 display: %s", error->message);
-    }
 }
 
 /**
diff --git a/src/x11/meta-x11-display.c b/src/x11/meta-x11-display.c
index 49f7abb5c..e9a73b249 100644
--- a/src/x11/meta-x11-display.c
+++ b/src/x11/meta-x11-display.c
@@ -1110,6 +1110,9 @@ meta_x11_display_new (MetaDisplay *display, GError **error)
   };
   Atom atoms[G_N_ELEMENTS(atom_names)];
 
+  if (!meta_x11_init_gdk_display (error))
+    return NULL;
+
   g_assert (prepared_gdk_display);
   gdk_display = g_steal_pointer (&prepared_gdk_display);