diff --git a/src/core/display-private.h b/src/core/display-private.h
index 122387c09..5f664ff41 100644
--- a/src/core/display-private.h
+++ b/src/core/display-private.h
@@ -404,8 +404,7 @@ gboolean meta_display_show_resize_popup (MetaDisplay *display,
                                          int display_w,
                                          int display_h);
 
-void meta_restart_init (void);
-void meta_restart_finish (void);
+void meta_set_is_restart (gboolean whether);
 
 void meta_display_cancel_touch (MetaDisplay *display);
 
diff --git a/src/core/errors.c b/src/core/errors.c
index 61085607e..fb01c9a1d 100644
--- a/src/core/errors.c
+++ b/src/core/errors.c
@@ -46,32 +46,17 @@
 void
 meta_error_trap_push (MetaX11Display *x11_display)
 {
-  GdkDisplay *gdk_display;
-
-  gdk_display = gdk_x11_lookup_xdisplay (x11_display->xdisplay);
-  g_assert (gdk_display != NULL);
-
-  gdk_x11_display_error_trap_push (gdk_display);
+  gdk_x11_display_error_trap_push (x11_display->gdk_display);
 }
 
 void
 meta_error_trap_pop (MetaX11Display *x11_display)
 {
-  GdkDisplay *gdk_display;
-
-  gdk_display = gdk_x11_lookup_xdisplay (x11_display->xdisplay);
-  g_assert (gdk_display != NULL);
-
-  gdk_x11_display_error_trap_pop_ignored (gdk_display);
+  gdk_x11_display_error_trap_pop_ignored (x11_display->gdk_display);
 }
 
 int
 meta_error_trap_pop_with_return (MetaX11Display *x11_display)
 {
-  GdkDisplay *gdk_display;
-
-  gdk_display = gdk_x11_lookup_xdisplay (x11_display->xdisplay);
-  g_assert (gdk_display != NULL);
-
-  return gdk_x11_display_error_trap_pop (gdk_display);
+  return gdk_x11_display_error_trap_pop (x11_display->gdk_display);
 }
diff --git a/src/core/main.c b/src/core/main.c
index f3e4b3290..f294b7511 100644
--- a/src/core/main.c
+++ b/src/core/main.c
@@ -599,10 +599,6 @@ meta_init (void)
     meta_fatal ("Can't specify both SM save file and SM client id\n");
 
   meta_main_loop = g_main_loop_new (NULL, FALSE);
-
-  meta_ui_init ();
-
-  meta_restart_init ();
 }
 
 /**
diff --git a/src/core/restart.c b/src/core/restart.c
index cce79d47e..f1f8c7686 100644
--- a/src/core/restart.c
+++ b/src/core/restart.c
@@ -50,15 +50,9 @@ static gboolean restart_message_shown = FALSE;
 static gboolean is_restart = FALSE;
 
 void
-meta_restart_init (void)
+meta_set_is_restart (gboolean whether)
 {
-  Display *xdisplay = meta_ui_get_display ();
-  Atom atom_restart_helper = XInternAtom (xdisplay, "_MUTTER_RESTART_HELPER", False);
-  Window restart_helper_window = None;
-
-  restart_helper_window = XGetSelectionOwner (xdisplay, atom_restart_helper);
-  if (restart_helper_window)
-    is_restart = TRUE;
+  is_restart = whether;
 }
 
 static void
@@ -187,18 +181,6 @@ meta_restart (const char *message)
   return;
 }
 
-void
-meta_restart_finish (void)
-{
-  if (is_restart)
-    {
-      MetaDisplay *display = meta_get_display ();
-      Display *xdisplay = meta_x11_display_get_xdisplay (display->x11_display);
-      Atom atom_restart_helper = XInternAtom (xdisplay, "_MUTTER_RESTART_HELPER", False);
-      XSetSelectionOwner (xdisplay, atom_restart_helper, None, META_CURRENT_TIME);
-    }
-}
-
 /**
  * meta_is_restart:
  *
diff --git a/src/ui/ui.c b/src/ui/ui.c
index 1e075b62d..86b9d89c6 100644
--- a/src/ui/ui.c
+++ b/src/ui/ui.c
@@ -26,6 +26,7 @@
 #include <meta/util.h>
 #include "core.h"
 #include "theme-private.h"
+#include "x11/meta-x11-display-private.h"
 
 #include <string.h>
 #include <stdlib.h>
@@ -44,52 +45,18 @@ struct _MetaUI
   guint32 button_click_time;
 };
 
-void
-meta_ui_init (void)
+MetaUI *
+meta_ui_new (MetaX11Display *x11_display)
 {
-  const char *gdk_gl_env = NULL;
-  gdk_set_allowed_backends ("x11");
-
-  gdk_gl_env = g_getenv ("GDK_GL");
-  g_setenv("GDK_GL", "disable", TRUE);
-
-  if (!gtk_init_check (NULL, NULL))
-    meta_fatal ("Unable to open X display %s\n", XDisplayName (NULL));
-
-  if (gdk_gl_env)
-    g_setenv("GDK_GL", gdk_gl_env, TRUE);
-  else
-    unsetenv("GDK_GL");
-
-  /* We need to be able to fully trust that the window and monitor sizes
-     that Gdk reports corresponds to the X ones, so we disable the automatic
-     scale handling */
-  gdk_x11_display_set_window_scale (gdk_display_get_default (), 1);
-}
-
-Display*
-meta_ui_get_display (void)
-{
-  return GDK_DISPLAY_XDISPLAY (gdk_display_get_default ());
-}
-
-gint
-meta_ui_get_screen_number (void)
-{
-  return gdk_screen_get_number (gdk_screen_get_default ());
-}
-
-MetaUI*
-meta_ui_new (Display *xdisplay)
-{
-  GdkDisplay *gdisplay;
   MetaUI *ui;
 
-  ui = g_new0 (MetaUI, 1);
-  ui->xdisplay = xdisplay;
+  if (!gtk_init_check (NULL, NULL))
+    meta_fatal ("Unable to initialize GTK");
 
-  gdisplay = gdk_x11_lookup_xdisplay (xdisplay);
-  g_assert (gdisplay == gdk_display_get_default ());
+  g_assert (x11_display->gdk_display == gdk_display_get_default ());
+
+  ui = g_new0 (MetaUI, 1);
+  ui->xdisplay = x11_display->xdisplay;
 
   ui->frames = meta_frames_new ();
   /* GTK+ needs the frame-sync protocol to work in order to properly
@@ -100,7 +67,7 @@ meta_ui_new (Display *xdisplay)
    */
   gtk_widget_show (GTK_WIDGET (ui->frames));
 
-  g_object_set_data (G_OBJECT (gdisplay), "meta-ui", ui);
+  g_object_set_data (G_OBJECT (x11_display->gdk_display), "meta-ui", ui);
 
   return ui;
 }
@@ -108,12 +75,12 @@ meta_ui_new (Display *xdisplay)
 void
 meta_ui_free (MetaUI *ui)
 {
-  GdkDisplay *gdisplay;
+  GdkDisplay *gdk_display;
 
   gtk_widget_destroy (GTK_WIDGET (ui->frames));
 
-  gdisplay = gdk_x11_lookup_xdisplay (ui->xdisplay);
-  g_object_set_data (G_OBJECT (gdisplay), "meta-ui", NULL);
+  gdk_display = gdk_x11_lookup_xdisplay (ui->xdisplay);
+  g_object_set_data (G_OBJECT (gdk_display), "meta-ui", NULL);
 
   g_free (ui);
 }
diff --git a/src/ui/ui.h b/src/ui/ui.h
index 91c26c384..15daaf270 100644
--- a/src/ui/ui.h
+++ b/src/ui/ui.h
@@ -36,13 +36,7 @@ typedef struct _MetaUIFrame MetaUIFrame;
 
 typedef gboolean (* MetaEventFunc) (XEvent *xevent, gpointer data);
 
-void meta_ui_init (void);
-
-Display* meta_ui_get_display (void);
-
-gint meta_ui_get_screen_number (void);
-
-MetaUI* meta_ui_new (Display *xdisplay);
+MetaUI *meta_ui_new  (MetaX11Display *x11_display);
 void    meta_ui_free (MetaUI *ui);
 
 void meta_ui_theme_get_frame_borders (MetaUI *ui,
diff --git a/src/x11/meta-x11-display-private.h b/src/x11/meta-x11-display-private.h
index 41ad8d88c..05019e8cf 100644
--- a/src/x11/meta-x11-display-private.h
+++ b/src/x11/meta-x11-display-private.h
@@ -47,6 +47,7 @@ struct _MetaX11Display
   GObject parent;
 
   MetaDisplay *display;
+  GdkDisplay *gdk_display;
 
   char *name;
   char *screen_name;
diff --git a/src/x11/meta-x11-display.c b/src/x11/meta-x11-display.c
index 2bd0864d0..5827ad884 100644
--- a/src/x11/meta-x11-display.c
+++ b/src/x11/meta-x11-display.c
@@ -33,6 +33,8 @@
 #include "core/display-private.h"
 #include "x11/meta-x11-display-private.h"
 
+#include <gdk/gdk.h>
+
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
@@ -189,7 +191,8 @@ meta_x11_display_dispose (GObject *object)
       XSelectInput (x11_display->xdisplay, x11_display->xroot, 0);
       if (meta_error_trap_pop_with_return (x11_display) != Success)
         meta_warning ("Could not release screen %d on display \"%s\"\n",
-                      meta_ui_get_screen_number (), x11_display->name);
+                      DefaultScreen (x11_display->xdisplay),
+                      x11_display->name);
 
       x11_display->xroot = None;
     }
@@ -202,6 +205,12 @@ meta_x11_display_dispose (GObject *object)
       x11_display->xdisplay = NULL;
     }
 
+  if (x11_display->gdk_display)
+    {
+      gdk_display_close (x11_display->gdk_display);
+      x11_display->gdk_display = NULL;
+    }
+
   g_free (x11_display->name);
   x11_display->name = NULL;
 
@@ -979,6 +988,46 @@ meta_x11_display_new (MetaDisplay *display, GError **error)
   guint32 timestamp;
   MetaWorkspace *current_workspace;
   uint32_t current_workspace_index = 0;
+  Atom atom_restart_helper;
+  Window restart_helper_window = None;
+  GdkDisplay *gdk_display;
+  const char *gdk_gl_env = NULL;
+  const char *xdisplay_name;
+
+  xdisplay_name = g_getenv ("DISPLAY");
+  if (!xdisplay_name)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Unable to open display, DISPLAY not set");
+      return FALSE;
+    }
+
+  gdk_set_allowed_backends ("x11");
+
+  gdk_gl_env = g_getenv ("GDK_GL");
+  g_setenv("GDK_GL", "disable", TRUE);
+
+  gdk_display = gdk_display_open (xdisplay_name);
+
+  if (!gdk_display)
+    {
+      meta_warning (_("Failed to initialize GDK\n"));
+
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Failed to initialize GDK");
+
+      return NULL;
+    }
+
+  if (gdk_gl_env)
+    g_setenv("GDK_GL", gdk_gl_env, TRUE);
+  else
+    unsetenv("GDK_GL");
+
+  /* We need to be able to fully trust that the window and monitor sizes
+     that Gdk reports corresponds to the X ones, so we disable the automatic
+     scale handling */
+  gdk_x11_display_set_window_scale (gdk_display, 1);
 
   /* A list of all atom names, so that we can intern them in one go. */
   const char *atom_names[] = {
@@ -990,7 +1039,7 @@ meta_x11_display_new (MetaDisplay *display, GError **error)
 
   meta_verbose ("Opening display '%s'\n", XDisplayName (NULL));
 
-  xdisplay = meta_ui_get_display ();
+  xdisplay = GDK_DISPLAY_XDISPLAY (gdk_display);
 
   if (xdisplay == NULL)
     {
@@ -1000,6 +1049,8 @@ meta_x11_display_new (MetaDisplay *display, GError **error)
       g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
                    "Failed to open X11 display");
 
+      gdk_display_close (gdk_display);
+
       return NULL;
     }
 
@@ -1013,7 +1064,10 @@ meta_x11_display_new (MetaDisplay *display, GError **error)
 
   replace_current_wm = meta_get_replace_current_wm ();
 
-  number = meta_ui_get_screen_number ();
+  /* According to _gdk_x11_display_open (), this will be returned
+   * by gdk_display_get_default_screen ()
+   */
+  number = DefaultScreen (xdisplay);
 
   xroot = RootWindow (xdisplay, number);
 
@@ -1031,12 +1085,20 @@ meta_x11_display_new (MetaDisplay *display, GError **error)
       XFlush (xdisplay);
       XCloseDisplay (xdisplay);
 
+      gdk_display_close (gdk_display);
+
       return NULL;
     }
 
   xscreen = ScreenOfDisplay (xdisplay, number);
 
+  atom_restart_helper = XInternAtom (xdisplay, "_MUTTER_RESTART_HELPER", False);
+  restart_helper_window = XGetSelectionOwner (xdisplay, atom_restart_helper);
+  if (restart_helper_window)
+    meta_set_is_restart (TRUE);
+
   x11_display = g_object_new (META_TYPE_X11_DISPLAY, NULL);
+  x11_display->gdk_display = gdk_display;
   x11_display->display = display;
 
   /* here we use XDisplayName which is what the user
@@ -1144,7 +1206,8 @@ meta_x11_display_new (MetaDisplay *display, GError **error)
 
   /* Now that we've gotten taken a reference count on the COW, we
    * can close the helper that is holding on to it */
-  meta_restart_finish ();
+  if (meta_is_restart ())
+    XSetSelectionOwner (xdisplay, atom_restart_helper, None, META_CURRENT_TIME);
 
   /* Handle creating a no_focus_window for this screen */
   x11_display->no_focus_window =
@@ -1166,7 +1229,7 @@ meta_x11_display_new (MetaDisplay *display, GError **error)
 
   set_desktop_geometry_hint (x11_display);
 
-  x11_display->ui = meta_ui_new (xdisplay);
+  x11_display->ui = meta_ui_new (x11_display);
 
   x11_display->keys_grabbed = FALSE;
   meta_x11_display_grab_keys (x11_display);
@@ -1255,7 +1318,7 @@ meta_x11_display_new (MetaDisplay *display, GError **error)
 int
 meta_x11_display_get_screen_number (MetaX11Display *x11_display)
 {
-  return meta_ui_get_screen_number ();
+  return DefaultScreen (x11_display->xdisplay);
 }
 
 /**
@@ -1588,7 +1651,7 @@ meta_x11_display_set_cm_selection (MetaX11Display *x11_display)
 
   timestamp = meta_x11_display_get_current_time_roundtrip (x11_display);
   g_snprintf (selection, sizeof (selection), "_NET_WM_CM_S%d",
-              meta_ui_get_screen_number ());
+              DefaultScreen (x11_display->xdisplay));
   a = XInternAtom (x11_display->xdisplay, selection, False);
 
   x11_display->wm_cm_selection_window = take_manager_selection (x11_display, x11_display->xroot, a, timestamp, TRUE);