diff --git a/src/core/display.c b/src/core/display.c
index 93ed140c2..395b541bb 100644
--- a/src/core/display.c
+++ b/src/core/display.c
@@ -49,6 +49,7 @@
 #include "backends/x11/meta-backend-x11.h"
 #include "backends/x11/cm/meta-backend-x11-cm.h"
 #include "clutter/x11/clutter-x11.h"
+#include "compositor/compositor-private.h"
 #include "core/bell.h"
 #include "core/boxes-private.h"
 #include "core/display-private.h"
@@ -643,12 +644,14 @@ meta_display_init_x11 (MetaDisplay  *display,
 
   display->x11_display = x11_display;
   g_signal_emit (display, display_signals[X11_DISPLAY_OPENED], 0);
+
   meta_x11_display_create_guard_window (x11_display);
 
   if (!display->display_opening)
-    meta_display_manage_all_xwindows (display);
-
-  meta_compositor_redirect_x11_windows (display->compositor);
+    {
+      meta_display_manage_all_xwindows (display);
+      meta_compositor_redirect_x11_windows (display->compositor);
+    }
 
   return TRUE;
 }
diff --git a/src/wayland/meta-wayland-private.h b/src/wayland/meta-wayland-private.h
index 6acbafc1f..9b99674e0 100644
--- a/src/wayland/meta-wayland-private.h
+++ b/src/wayland/meta-wayland-private.h
@@ -48,6 +48,7 @@ typedef struct
   char *lock_file;
   int abstract_fd;
   int unix_fd;
+  guint xserver_grace_period_id;
   struct wl_display *wayland_display;
   struct wl_client *client;
   struct wl_resource *xserver_resource;
diff --git a/src/wayland/meta-xwayland.c b/src/wayland/meta-xwayland.c
index 333cd3064..0f7ecb85a 100644
--- a/src/wayland/meta-xwayland.c
+++ b/src/wayland/meta-xwayland.c
@@ -41,6 +41,7 @@
 
 #include "compositor/meta-surface-actor-wayland.h"
 #include "compositor/meta-window-actor-private.h"
+#include "core/main-private.h"
 #include "meta/main.h"
 #include "wayland/meta-wayland-actor-surface.h"
 #include "x11/meta-x11-display-private.h"
@@ -71,6 +72,8 @@ G_DEFINE_TYPE (MetaWaylandSurfaceRoleXWayland,
 
 static int display_number_override = -1;
 
+static void meta_xwayland_stop_xserver (MetaXWaylandManager *manager);
+
 void
 meta_xwayland_associate_window_with_surface (MetaWindow          *window,
                                              MetaWaylandSurface  *surface)
@@ -369,24 +372,50 @@ xserver_died (GObject      *source,
       g_warning ("Failed to finish waiting for Xwayland: %s", error->message);
     }
   else if (!g_subprocess_get_successful (proc))
-    g_warning ("X Wayland crashed; exiting");
-  else
     {
-      /* For now we simply abort if we see the server exit.
-       *
-       * In the future X will only be loaded lazily for legacy X support
-       * but for now it's a hard requirement. */
-      g_warning ("Spurious exit of X Wayland server");
+      if (meta_get_x11_display_policy () == META_DISPLAY_POLICY_MANDATORY)
+        g_warning ("X Wayland crashed; exiting");
+      else
+        g_warning ("X Wayland crashed; attempting to recover");
     }
 
-  meta_exit (META_EXIT_ERROR);
+  if (meta_get_x11_display_policy () == META_DISPLAY_POLICY_MANDATORY)
+    {
+      meta_exit (META_EXIT_ERROR);
+    }
+  else if (meta_get_x11_display_policy () == META_DISPLAY_POLICY_ON_DEMAND)
+    {
+      MetaWaylandCompositor *compositor = meta_wayland_compositor_get_default ();
+      MetaDisplay *display = meta_get_display ();
+
+      if (display->x11_display)
+        meta_display_shutdown_x11 (display);
+
+      if (!meta_xwayland_init (&compositor->xwayland_manager,
+                               compositor->wayland_display))
+        g_warning ("Failed to init X sockets");
+    }
+}
+
+static gboolean
+shutdown_xwayland_cb (gpointer data)
+{
+  MetaXWaylandManager *manager = data;
+
+  meta_verbose ("Shutting down Xwayland");
+  manager->xserver_grace_period_id = 0;
+  meta_display_shutdown_x11 (meta_get_display ());
+  meta_xwayland_stop_xserver (manager);
+  return G_SOURCE_REMOVE;
 }
 
 static int
 x_io_error (Display *display)
 {
   g_warning ("Connection to xwayland lost");
-  meta_exit (META_EXIT_ERROR);
+
+  if (meta_get_x11_display_policy () == META_DISPLAY_POLICY_MANDATORY)
+    meta_exit (META_EXIT_ERROR);
 
   return 0;
 }
@@ -553,6 +582,7 @@ on_displayfd_ready (int          fd,
                     gpointer     user_data)
 {
   MetaXWaylandManager *manager = user_data;
+  MetaDisplay *display = meta_get_display ();
 
   /* The server writes its display name to the displayfd
    * socket when it's ready. We don't care about the data
@@ -560,6 +590,9 @@ on_displayfd_ready (int          fd,
    * that means it's ready. */
   xserver_finished_init (manager);
 
+  if (meta_get_x11_display_policy () == META_DISPLAY_POLICY_ON_DEMAND)
+    meta_display_init_x11 (display, NULL);
+
   return G_SOURCE_REMOVE;
 }
 
@@ -637,6 +670,19 @@ meta_xwayland_start_xserver (MetaXWaylandManager *manager)
   return TRUE;
 }
 
+static gboolean
+xdisplay_connection_activity_cb (gint         fd,
+                                 GIOCondition cond,
+                                 gpointer     user_data)
+{
+  MetaXWaylandManager *manager = user_data;
+
+  if (!meta_xwayland_start_xserver (manager))
+    g_critical ("Could not start Xserver");
+
+  return G_SOURCE_REMOVE;
+}
+
 static void
 window_unmanaged_cb (MetaWindow          *window,
                      MetaXWaylandManager *manager)
@@ -645,6 +691,12 @@ window_unmanaged_cb (MetaWindow          *window,
   g_signal_handlers_disconnect_by_func (window,
                                         window_unmanaged_cb,
                                         manager);
+  if (!manager->x11_windows)
+    {
+      meta_verbose ("All X11 windows gone, setting shutdown timeout");
+      manager->xserver_grace_period_id =
+        g_timeout_add_seconds (10, shutdown_xwayland_cb, manager);
+    }
 }
 
 static void
@@ -660,20 +712,62 @@ window_created_cb (MetaDisplay         *display,
   manager->x11_windows = g_list_prepend (manager->x11_windows, window);
   g_signal_connect (window, "unmanaged",
                     G_CALLBACK (window_unmanaged_cb), manager);
+
+  if (manager->xserver_grace_period_id)
+    {
+      g_source_remove (manager->xserver_grace_period_id);
+      manager->xserver_grace_period_id = 0;
+    }
+}
+
+static void
+meta_xwayland_stop_xserver (MetaXWaylandManager *manager)
+{
+  if (manager->proc)
+    g_subprocess_send_signal (manager->proc, SIGTERM);
+  g_signal_handlers_disconnect_by_func (meta_get_display (),
+                                        window_created_cb,
+                                        manager);
+  g_clear_object (&manager->xserver_died_cancellable);
+  g_clear_object (&manager->proc);
 }
 
 gboolean
 meta_xwayland_init (MetaXWaylandManager *manager,
                     struct wl_display   *wl_display)
 {
-  if (!choose_xdisplay (manager))
-    return FALSE;
+  MetaDisplayPolicy policy;
+  gboolean fatal;
+
+  if (!manager->display_name)
+    {
+      if (!choose_xdisplay (manager))
+        return FALSE;
+    }
+  else
+    {
+      if (!open_display_sockets (manager, manager->display_index, &fatal))
+        return FALSE;
+    }
 
   if (!prepare_auth_file (manager))
     return FALSE;
 
   manager->wayland_display = wl_display;
-  return meta_xwayland_start_xserver (manager);
+  policy = meta_get_x11_display_policy ();
+
+  if (policy == META_DISPLAY_POLICY_MANDATORY)
+    {
+      return meta_xwayland_start_xserver (manager);
+    }
+  else if (policy == META_DISPLAY_POLICY_ON_DEMAND)
+    {
+      g_unix_fd_add (manager->abstract_fd, G_IO_IN,
+                     xdisplay_connection_activity_cb, manager);
+      return TRUE;
+    }
+
+  return FALSE;
 }
 
 static void
@@ -712,13 +806,7 @@ meta_xwayland_shutdown (MetaXWaylandManager *manager)
 {
   char path[256];
 
-  g_signal_handlers_disconnect_by_func (meta_get_display (),
-                                        window_created_cb,
-                                        manager);
-
   g_cancellable_cancel (manager->xserver_died_cancellable);
-  g_clear_object (&manager->proc);
-  g_clear_object (&manager->xserver_died_cancellable);
 
   snprintf (path, sizeof path, "/tmp/.X11-unix/X%d", manager->display_index);
   unlink (path);