diff --git a/src/backends/x11/meta-backend-x11.c b/src/backends/x11/meta-backend-x11.c
index d44f83a73..50fd57812 100644
--- a/src/backends/x11/meta-backend-x11.c
+++ b/src/backends/x11/meta-backend-x11.c
@@ -942,6 +942,8 @@ meta_backend_x11_dispose (GObject *object)
     }
 
   G_OBJECT_CLASS (meta_backend_x11_parent_class)->dispose (object);
+
+  g_clear_pointer (&priv->xdisplay, XCloseDisplay);
 }
 
 static void
diff --git a/src/backends/x11/meta-clutter-backend-x11.c b/src/backends/x11/meta-clutter-backend-x11.c
index 345c1424d..2c43e9f35 100644
--- a/src/backends/x11/meta-clutter-backend-x11.c
+++ b/src/backends/x11/meta-clutter-backend-x11.c
@@ -153,8 +153,6 @@ meta_clutter_backend_x11_finalize (GObject *gobject)
                                           cogl_xlib_filter,
                                           clutter_backend_x11);
 
-  XCloseDisplay (clutter_backend_x11->xdisplay);
-
   G_OBJECT_CLASS (meta_clutter_backend_x11_parent_class)->finalize (gobject);
 }