diff --git a/src/wayland/meta-wayland.c b/src/wayland/meta-wayland.c
index dccf0da10..86435400e 100644
--- a/src/wayland/meta-wayland.c
+++ b/src/wayland/meta-wayland.c
@@ -421,7 +421,12 @@ meta_wayland_log_func (const char *fmt,
 void
 meta_wayland_compositor_prepare_shutdown (MetaWaylandCompositor *compositor)
 {
-  meta_xwayland_shutdown (&compositor->xwayland_manager);
+  MetaX11DisplayPolicy x11_display_policy;
+
+  x11_display_policy =
+    meta_context_get_x11_display_policy (compositor->context);
+  if (x11_display_policy != META_X11_DISPLAY_POLICY_DISABLED)
+    meta_xwayland_shutdown (&compositor->xwayland_manager);
 
   if (compositor->wayland_display)
     wl_display_destroy_clients (compositor->wayland_display);
diff --git a/src/wayland/meta-xwayland.c b/src/wayland/meta-xwayland.c
index c6421a430..77fb366ea 100644
--- a/src/wayland/meta-xwayland.c
+++ b/src/wayland/meta-xwayland.c
@@ -1338,16 +1338,21 @@ meta_xwayland_shutdown (MetaXWaylandManager *manager)
 
   meta_xwayland_terminate (manager);
 
-  snprintf (path, sizeof path, "%s%d", X11_TMP_UNIX_PATH,
-            manager->public_connection.display_index);
-  unlink (path);
+  if (manager->public_connection.name)
+    {
+      snprintf (path, sizeof path, "%s%d", X11_TMP_UNIX_PATH,
+                manager->public_connection.display_index);
+      unlink (path);
+      g_clear_pointer (&manager->public_connection.name, g_free);
+    }
 
-  snprintf (path, sizeof path, "%s%d", X11_TMP_UNIX_PATH,
-            manager->private_connection.display_index);
-  unlink (path);
-
-  g_clear_pointer (&manager->public_connection.name, g_free);
-  g_clear_pointer (&manager->private_connection.name, g_free);
+  if (manager->private_connection.name)
+    {
+      snprintf (path, sizeof path, "%s%d", X11_TMP_UNIX_PATH,
+                manager->private_connection.display_index);
+      unlink (path);
+      g_clear_pointer (&manager->private_connection.name, g_free);
+    }
 
   meta_xwayland_connection_release (&manager->public_connection);
   meta_xwayland_connection_release (&manager->private_connection);