diff --git a/src/core/frame.c b/src/core/frame.c
index be1f4e2bf..e647644dd 100644
--- a/src/core/frame.c
+++ b/src/core/frame.c
@@ -65,7 +65,7 @@ meta_window_ensure_frame (MetaWindow *window)
                 frame->rect.x, frame->rect.y,
                 frame->rect.width, frame->rect.height);
 
-  frame->ui_frame = meta_ui_create_frame (window->screen->ui,
+  frame->ui_frame = meta_ui_create_frame (x11_display->ui,
                                           x11_display->xdisplay,
                                           frame->window,
                                           window->xvisual,
@@ -118,7 +118,7 @@ meta_window_ensure_frame (MetaWindow *window)
   meta_frame_update_style (frame);
   meta_frame_update_title (frame);
 
-  meta_ui_map_frame (frame->window->screen->ui, frame->xwindow);
+  meta_ui_map_frame (x11_display->ui, frame->xwindow);
 
   {
     MetaBackend *backend = meta_get_backend ();
diff --git a/src/core/keybindings-private.h b/src/core/keybindings-private.h
index a91e47e02..a6c8545e0 100644
--- a/src/core/keybindings-private.h
+++ b/src/core/keybindings-private.h
@@ -114,8 +114,6 @@ typedef struct
 
 void     meta_display_init_keys             (MetaDisplay *display);
 void     meta_display_shutdown_keys         (MetaDisplay *display);
-void     meta_screen_grab_keys              (MetaScreen  *screen);
-void     meta_screen_ungrab_keys            (MetaScreen  *screen);
 void     meta_window_grab_keys              (MetaWindow  *window);
 void     meta_window_ungrab_keys            (MetaWindow  *window);
 gboolean meta_window_grab_all_keys          (MetaWindow  *window,
@@ -139,4 +137,7 @@ GList *meta_prefs_get_keybindings (void);
 void meta_prefs_get_overlay_binding (MetaKeyCombo *combo);
 const char *meta_prefs_get_iso_next_group_option (void);
 
+void meta_x11_display_grab_keys   (MetaX11Display *x11_display);
+void meta_x11_display_ungrab_keys (MetaX11Display *x11_display);
+
 #endif
diff --git a/src/core/keybindings.c b/src/core/keybindings.c
index 8bf19ca56..db50ab9cd 100644
--- a/src/core/keybindings.c
+++ b/src/core/keybindings.c
@@ -703,7 +703,8 @@ ungrab_key_bindings (MetaDisplay *display)
 {
   GSList *windows, *l;
 
-  meta_screen_ungrab_keys (display->screen);
+  if (display->x11_display)
+    meta_x11_display_ungrab_keys (display->x11_display);
 
   windows = meta_display_list_windows (display, META_LIST_DEFAULT);
   for (l = windows; l; l = l->next)
@@ -720,7 +721,8 @@ grab_key_bindings (MetaDisplay *display)
 {
   GSList *windows, *l;
 
-  meta_screen_grab_keys (display->screen);
+  if (display->x11_display)
+    meta_x11_display_grab_keys (display->x11_display);
 
   windows = meta_display_list_windows (display, META_LIST_DEFAULT);
   for (l = windows; l; l = l->next)
@@ -1298,45 +1300,44 @@ change_binding_keygrabs (MetaKeyBindingManager *keys,
 }
 
 static void
-meta_screen_change_keygrabs (MetaScreen *screen,
-                             gboolean    grab)
+meta_x11_display_change_keygrabs (MetaX11Display *x11_display,
+                                  gboolean        grab)
 {
-  MetaDisplay *display = screen->display;
-  MetaKeyBindingManager *keys = &display->key_binding_manager;
+  MetaKeyBindingManager *keys = &x11_display->display->key_binding_manager;
   int i;
 
   if (keys->overlay_resolved_key_combo.len != 0)
-    meta_change_keygrab (keys, display->x11_display->xroot,
+    meta_change_keygrab (keys, x11_display->xroot,
                          grab, &keys->overlay_resolved_key_combo);
 
   for (i = 0; i < keys->n_iso_next_group_combos; i++)
-    meta_change_keygrab (keys, display->x11_display->xroot,
+    meta_change_keygrab (keys, x11_display->xroot,
                          grab, &keys->iso_next_group_combo[i]);
 
-  change_binding_keygrabs (keys, display->x11_display->xroot,
+  change_binding_keygrabs (keys, x11_display->xroot,
                            FALSE, grab);
 }
 
 void
-meta_screen_grab_keys (MetaScreen *screen)
+meta_x11_display_grab_keys (MetaX11Display *x11_display)
 {
-  if (screen->keys_grabbed)
+  if (x11_display->keys_grabbed)
     return;
 
-  meta_screen_change_keygrabs (screen, TRUE);
+  meta_x11_display_change_keygrabs (x11_display, TRUE);
 
-  screen->keys_grabbed = TRUE;
+  x11_display->keys_grabbed = TRUE;
 }
 
 void
-meta_screen_ungrab_keys (MetaScreen  *screen)
+meta_x11_display_ungrab_keys (MetaX11Display *x11_display)
 {
-  if (!screen->keys_grabbed)
+  if (!x11_display->keys_grabbed)
     return;
 
-  meta_screen_change_keygrabs (screen, FALSE);
+  meta_x11_display_change_keygrabs (x11_display, FALSE);
 
-  screen->keys_grabbed = FALSE;
+  x11_display->keys_grabbed = FALSE;
 }
 
 static void
diff --git a/src/core/screen-private.h b/src/core/screen-private.h
index f4ee639e0..6cd028ca0 100644
--- a/src/core/screen-private.h
+++ b/src/core/screen-private.h
@@ -45,7 +45,6 @@ struct _MetaScreen
   GObject parent_instance;
 
   MetaDisplay *display;
-  MetaUI *ui;
 
   MetaWorkspace *active_workspace;
 
@@ -57,8 +56,6 @@ struct _MetaScreen
   guint vertical_workspaces : 1;
   guint workspace_layout_overridden : 1;
 
-  guint keys_grabbed : 1;
-
   int closing;
 };
 
diff --git a/src/core/screen.c b/src/core/screen.c
index 73e12636c..61aa9f96f 100644
--- a/src/core/screen.c
+++ b/src/core/screen.c
@@ -209,7 +209,6 @@ meta_screen_new (MetaDisplay *display,
   MetaScreen *screen;
   int number;
   Window xroot = meta_x11_display_get_xroot (display->x11_display);
-  Display *xdisplay = meta_x11_display_get_xdisplay (display->x11_display);
 
   number = meta_ui_get_screen_number ();
 
@@ -240,11 +239,6 @@ meta_screen_new (MetaDisplay *display,
    */
   meta_workspace_new (screen);
 
-  screen->keys_grabbed = FALSE;
-  meta_screen_grab_keys (screen);
-
-  screen->ui = meta_ui_new (xdisplay);
-
   meta_prefs_add_listener (prefs_changed_callback, screen);
 
   meta_verbose ("Added screen %d ('%s') root 0x%lx\n",
@@ -304,10 +298,6 @@ meta_screen_free (MetaScreen *screen,
 
   meta_prefs_remove_listener (prefs_changed_callback, screen);
 
-  meta_screen_ungrab_keys (screen);
-
-  meta_ui_free (screen->ui);
-
   g_object_unref (screen);
 }
 
diff --git a/src/tests/test-runner.c b/src/tests/test-runner.c
index 3f148eeca..5fb43e4b4 100644
--- a/src/tests/test-runner.c
+++ b/src/tests/test-runner.c
@@ -602,7 +602,7 @@ test_case_assert_stacking (TestCase *test,
            * is managed as a MetaWindow.
            */
           if (META_STACK_ID_IS_X11 (windows[i]) &&
-              meta_ui_window_is_dummy (display->screen->ui, windows[i]))
+              meta_ui_window_is_dummy (display->x11_display->ui, windows[i]))
             continue;
 
           if (stack_string->len > 0)
diff --git a/src/x11/events.c b/src/x11/events.c
index 814312cab..c1283ac3a 100644
--- a/src/x11/events.c
+++ b/src/x11/events.c
@@ -865,7 +865,7 @@ handle_input_xevent (MetaX11Display *x11_display,
            NULL;
 
   /* If this is an event for a GTK+ widget, let GTK+ handle it. */
-  if (meta_ui_window_is_widget (screen->ui, modified))
+  if (meta_ui_window_is_widget (x11_display->ui, modified))
     return FALSE;
 
   switch (input_event->evtype)
@@ -944,7 +944,6 @@ static void
 process_request_frame_extents (MetaX11Display *x11_display,
                                XEvent         *event)
 {
-  MetaDisplay *display = x11_display->display;
   /* The X window whose frame extents will be set. */
   Window xwindow = event->xclient.window;
   unsigned long data[4] = { 0, 0, 0, 0 };
@@ -964,7 +963,7 @@ process_request_frame_extents (MetaX11Display *x11_display,
       MetaFrameBorders borders;
 
       /* Return estimated frame extents for a normal window. */
-      meta_ui_theme_get_frame_borders (display->screen->ui,
+      meta_ui_theme_get_frame_borders (x11_display->ui,
                                        META_FRAME_TYPE_NORMAL,
                                        0,
                                        &borders);
diff --git a/src/x11/meta-x11-display-private.h b/src/x11/meta-x11-display-private.h
index a0bdbac78..72e68d544 100644
--- a/src/x11/meta-x11-display-private.h
+++ b/src/x11/meta-x11-display-private.h
@@ -33,6 +33,7 @@
 #include "meta/common.h"
 #include "meta/types.h"
 #include "meta/meta-x11-display.h"
+#include "ui/ui.h"
 
 typedef struct _MetaGroupPropHooks  MetaGroupPropHooks;
 typedef struct _MetaWindowPropHooks MetaWindowPropHooks;
@@ -112,6 +113,10 @@ struct _MetaX11Display
   MetaAlarmFilter alarm_filter;
   gpointer alarm_filter_data;
 
+  MetaUI *ui;
+
+  guint keys_grabbed : 1;
+
   int composite_event_base;
   int composite_error_base;
   int composite_major_version;
diff --git a/src/x11/meta-x11-display.c b/src/x11/meta-x11-display.c
index 33d9c7d21..ae0139c43 100644
--- a/src/x11/meta-x11-display.c
+++ b/src/x11/meta-x11-display.c
@@ -91,6 +91,14 @@ meta_x11_display_dispose (GObject *object)
 {
   MetaX11Display *x11_display = META_X11_DISPLAY (object);
 
+  meta_x11_display_ungrab_keys (x11_display);
+
+  if (x11_display->ui)
+    {
+      meta_ui_free (x11_display->ui);
+      x11_display->ui = NULL;
+    }
+
   if (x11_display->no_focus_window != None)
     {
       XUnmapWindow (x11_display->xdisplay, x11_display->no_focus_window);
@@ -821,6 +829,7 @@ meta_x11_display_new (MetaDisplay *display, GError **error)
                                         meta_unsigned_long_equal);
 
   x11_display->groups_by_leader = NULL;
+  x11_display->ui = NULL;
   x11_display->composite_overlay_window = None;
   x11_display->guard_window = None;
   x11_display->leader_window = None;
@@ -906,6 +915,11 @@ meta_x11_display_new (MetaDisplay *display, GError **error)
 
   set_desktop_geometry_hint (x11_display);
 
+  x11_display->ui = meta_ui_new (xdisplay);
+
+  x11_display->keys_grabbed = FALSE;
+  meta_x11_display_grab_keys (x11_display);
+
   return x11_display;
 }