diff --git a/src/core/display-private.h b/src/core/display-private.h
index 860159ce1..efd92a4ac 100644
--- a/src/core/display-private.h
+++ b/src/core/display-private.h
@@ -55,15 +55,6 @@ typedef enum
 #define _NET_WM_STATE_ADD           1    /* add/set property */
 #define _NET_WM_STATE_TOGGLE        2    /* toggle property  */
 
-/* This is basically a bogus number, just has to be large enough
- * to handle the expected case of the alt+tab operation, where
- * we want to ignore serials from UnmapNotify on the tab popup,
- * and the LeaveNotify/EnterNotify from the pointer ungrab. It
- * also has to be big enough to hold ignored serials from the point
- * where we reshape the stage to the point where we get events back.
- */
-#define N_IGNORED_CROSSING_SERIALS  10
-
 typedef enum
 {
   META_TILE_NONE,
@@ -114,12 +105,6 @@ struct _MetaDisplay
   GHashTable *stamps;
   GHashTable *wayland_windows;
 
-  /* serials of leave/unmap events that may
-   * correspond to an enter event we should
-   * ignore
-   */
-  unsigned long ignored_crossing_serials[N_IGNORED_CROSSING_SERIALS];
-
   guint32 current_time;
 
   /* We maintain a sequence counter, incremented for each #MetaWindow
diff --git a/src/core/display.c b/src/core/display.c
index 60c3f5fdc..71b3747a5 100644
--- a/src/core/display.c
+++ b/src/core/display.c
@@ -857,7 +857,6 @@ meta_display_new (MetaContext  *context,
   MetaBackend *backend = meta_context_get_backend (context);
   MetaDisplay *display;
   MetaDisplayPrivate *priv;
-  int i;
   guint32 timestamp;
   Window old_active_xwindow = None;
   MetaMonitorManager *monitor_manager;
@@ -886,13 +885,6 @@ meta_display_new (MetaContext  *context,
   display->allow_terminal_deactivation = TRUE; /* Only relevant for when a
                                                   terminal has the focus */
 
-  i = 0;
-  while (i < N_IGNORED_CROSSING_SERIALS)
-    {
-      display->ignored_crossing_serials[i] = 0;
-      ++i;
-    }
-
   display->current_time = META_CURRENT_TIME;
 
   meta_display_init_keys (display);
diff --git a/src/x11/events.c b/src/x11/events.c
index 0708c5edf..3bd215210 100644
--- a/src/x11/events.c
+++ b/src/x11/events.c
@@ -927,7 +927,7 @@ crossing_serial_is_ignored (MetaX11Display *x11_display,
   i = 0;
   while (i < N_IGNORED_CROSSING_SERIALS)
     {
-      if (x11_display->display->ignored_crossing_serials[i] == serial)
+      if (x11_display->ignored_crossing_serials[i] == serial)
         return TRUE;
       ++i;
     }
@@ -942,7 +942,7 @@ reset_ignored_crossing_serials (MetaX11Display *x11_display)
   i = 0;
   while (i < N_IGNORED_CROSSING_SERIALS)
     {
-      x11_display->display->ignored_crossing_serials[i] = 0;
+      x11_display->ignored_crossing_serials[i] = 0;
       ++i;
     }
 }
diff --git a/src/x11/meta-x11-display-private.h b/src/x11/meta-x11-display-private.h
index 466625e72..798b9ab30 100644
--- a/src/x11/meta-x11-display-private.h
+++ b/src/x11/meta-x11-display-private.h
@@ -39,6 +39,15 @@
 #include "meta-x11-stack-private.h"
 #include "x11/meta-sync-counter.h"
 
+/* This is basically a bogus number, just has to be large enough
+ * to handle the expected case of the alt+tab operation, where
+ * we want to ignore serials from UnmapNotify on the tab popup,
+ * and the LeaveNotify/EnterNotify from the pointer ungrab. It
+ * also has to be big enough to hold ignored serials from the point
+ * where we reshape the stage to the point where we get events back.
+ */
+#define N_IGNORED_CROSSING_SERIALS  10
+
 typedef struct _MetaGroupPropHooks  MetaGroupPropHooks;
 typedef struct _MetaWindowPropHooks MetaWindowPropHooks;
 
@@ -155,6 +164,12 @@ struct _MetaX11Display
 
   guint closing : 1;
 
+  /* serials of leave/unmap events that may
+   * correspond to an enter event we should
+   * ignore
+   */
+  unsigned long ignored_crossing_serials[N_IGNORED_CROSSING_SERIALS];
+
   /* we use property updates as sentinels for certain window focus events
    * to avoid some race conditions on EnterNotify events
    */
diff --git a/src/x11/meta-x11-display.c b/src/x11/meta-x11-display.c
index af26ff1b8..33d9ad5cb 100644
--- a/src/x11/meta-x11-display.c
+++ b/src/x11/meta-x11-display.c
@@ -1338,6 +1338,13 @@ meta_x11_display_new (MetaDisplay  *display,
   x11_display->server_focus_window = None;
   x11_display->server_focus_serial = 0;
 
+  i = 0;
+  while (i < N_IGNORED_CROSSING_SERIALS)
+    {
+      x11_display->ignored_crossing_serials[i] = 0;
+      ++i;
+    }
+
   x11_display->prop_hooks = NULL;
   meta_x11_display_init_window_prop_hooks (x11_display);
   x11_display->group_prop_hooks = NULL;
@@ -2386,24 +2393,23 @@ static void
 meta_x11_display_add_ignored_crossing_serial (MetaX11Display *x11_display,
                                               unsigned long   serial)
 {
-  MetaDisplay *display = x11_display->display;
   int i;
 
   /* don't add the same serial more than once */
   if (serial ==
-      display->ignored_crossing_serials[N_IGNORED_CROSSING_SERIALS - 1])
+      x11_display->ignored_crossing_serials[N_IGNORED_CROSSING_SERIALS - 1])
     return;
 
   /* shift serials to the left */
   i = 0;
   while (i < (N_IGNORED_CROSSING_SERIALS - 1))
     {
-      display->ignored_crossing_serials[i] =
-        display->ignored_crossing_serials[i + 1];
+      x11_display->ignored_crossing_serials[i] =
+        x11_display->ignored_crossing_serials[i + 1];
       ++i;
     }
   /* put new one on the end */
-  display->ignored_crossing_serials[i] = serial;
+  x11_display->ignored_crossing_serials[i] = serial;
 }
 
 void