diff --git a/cogl/cogl/cogl-renderer-private.h b/cogl/cogl/cogl-renderer-private.h
index efe0eb0cb..26de4fcf4 100644
--- a/cogl/cogl/cogl-renderer-private.h
+++ b/cogl/cogl/cogl-renderer-private.h
@@ -56,10 +56,6 @@ struct _CoglRenderer
 
   CoglList idle_closures;
 
-#ifdef HAVE_X11
-  Display *foreign_xdpy;
-#endif
-
   CoglDriverId driver_id;
   unsigned long private_features
     [COGL_FLAGS_N_LONGS_FOR_SIZE (COGL_N_PRIVATE_FEATURES)];
diff --git a/cogl/cogl/cogl-renderer.c b/cogl/cogl/cogl-renderer.c
index 9daba770f..bc41fb89f 100644
--- a/cogl/cogl/cogl-renderer.c
+++ b/cogl/cogl/cogl-renderer.c
@@ -65,6 +65,7 @@
 
 #ifdef HAVE_X11
 #include "cogl/cogl-xlib-renderer.h"
+#include "cogl/cogl-xlib-renderer-private.h"
 #endif
 
 
@@ -200,13 +201,14 @@ void
 cogl_xlib_renderer_set_foreign_display (CoglRenderer *renderer,
                                         Display *xdisplay)
 {
+  CoglXlibRenderer *xlib_renderer;
   g_return_if_fail (COGL_IS_RENDERER (renderer));
 
   /* NB: Renderers are considered immutable once connected */
   g_return_if_fail (!renderer->connected);
 
-  renderer->foreign_xdpy = xdisplay;
-
+  xlib_renderer = _cogl_xlib_renderer_get_data (renderer);
+  xlib_renderer->xdpy = xdisplay;
 }
 #endif /* HAVE_X11 */
 
diff --git a/cogl/cogl/cogl-xlib-renderer.c b/cogl/cogl/cogl-xlib-renderer.c
index ba76a79c4..19860450f 100644
--- a/cogl/cogl/cogl-xlib-renderer.c
+++ b/cogl/cogl/cogl-xlib-renderer.c
@@ -46,7 +46,6 @@
 #include <stdlib.h>
 #include <string.h>
 
-static char *_cogl_x11_display_name = NULL;
 static GList *_cogl_xlib_renderers = NULL;
 
 static void
@@ -88,33 +87,6 @@ unregister_xlib_renderer (CoglRenderer *renderer)
   _cogl_xlib_renderers = g_list_remove (_cogl_xlib_renderers, renderer);
 }
 
-static Display *
-assert_xlib_display (CoglRenderer *renderer, GError **error)
-{
-  Display *xdpy = renderer->foreign_xdpy;
-  CoglXlibRenderer *xlib_renderer = _cogl_xlib_renderer_get_data (renderer);
-
-  /* A foreign display may have already been set... */
-  if (xdpy)
-    {
-      xlib_renderer->xdpy = xdpy;
-      return xdpy;
-    }
-
-  xdpy = XOpenDisplay (_cogl_x11_display_name);
-  if (xdpy == NULL)
-    {
-      g_set_error (error,
-                   COGL_RENDERER_ERROR,
-                   COGL_RENDERER_ERROR_XLIB_DISPLAY_OPEN,
-                   "Failed to open X Display %s", _cogl_x11_display_name);
-      return NULL;
-    }
-
-  xlib_renderer->xdpy = xdpy;
-  return xdpy;
-}
-
 static void
 free_xlib_output (CoglXlibOutput *output)
 {
@@ -402,8 +374,7 @@ _cogl_xlib_renderer_connect (CoglRenderer *renderer, GError **error)
   int damage_error;
   int randr_error;
 
-  if (!assert_xlib_display (renderer, error))
-    return FALSE;
+  g_return_val_if_fail (xlib_renderer->xdpy != NULL, FALSE);
 
   if (getenv ("COGL_X11_SYNC"))
     XSynchronize (xlib_renderer->xdpy, TRUE);
@@ -445,9 +416,6 @@ _cogl_xlib_renderer_disconnect (CoglRenderer *renderer)
   g_list_free_full (xlib_renderer->outputs, (GDestroyNotify) free_xlib_output);
   xlib_renderer->outputs = NULL;
 
-  if (!renderer->foreign_xdpy && xlib_renderer->xdpy)
-    XCloseDisplay (xlib_renderer->xdpy);
-
   g_clear_pointer (&renderer->custom_winsys_user_data, _xlib_renderer_data_free);
 
   unregister_xlib_renderer (renderer);