diff --git a/src/wayland/meta-xwayland.c b/src/wayland/meta-xwayland.c
index 56a50b134..7dc7077f6 100644
--- a/src/wayland/meta-xwayland.c
+++ b/src/wayland/meta-xwayland.c
@@ -649,9 +649,6 @@ open_display_sockets (MetaXWaylandManager  *manager,
 {
   int abstract_fd, unix_fd;
 
-  if (!ensure_x11_unix_dir (error))
-    return FALSE;
-
   abstract_fd = bind_to_abstract_socket (display_index, fatal, error);
   if (abstract_fd < 0)
     return FALSE;
@@ -684,6 +681,9 @@ choose_xdisplay (MetaXWaylandManager     *manager,
   else if (g_getenv ("RUNNING_UNDER_GDM"))
     display = 1024;
 
+  if (!ensure_x11_unix_dir (error))
+    return FALSE;
+
   do
     {
       g_autoptr (GError) local_error = NULL;