diff --git a/cogl/cogl/cogl-xlib-renderer.c b/cogl/cogl/cogl-xlib-renderer.c
index 68dfbe147..ea1a47e46 100644
--- a/cogl/cogl/cogl-xlib-renderer.c
+++ b/cogl/cogl/cogl-xlib-renderer.c
@@ -54,41 +54,27 @@ static char *_cogl_x11_display_name = NULL;
 static GList *_cogl_xlib_renderers = NULL;
 
 static void
-destroy_xlib_renderer_data (void *user_data)
+_xlib_renderer_data_free (CoglXlibRenderer *data)
 {
-  CoglXlibRenderer *data = user_data;
-
   if (data->xvisinfo)
     XFree (data->xvisinfo);
 
-  g_slice_free (CoglXlibRenderer, user_data);
+  g_slice_free (CoglXlibRenderer, data);
 }
 
 CoglXlibRenderer *
 _cogl_xlib_renderer_get_data (CoglRenderer *renderer)
 {
-  static CoglUserDataKey key;
-  CoglXlibRenderer *data;
-
   /* Constructs a CoglXlibRenderer struct on demand and attaches it to
      the object using user data. It's done this way instead of using a
      subclassing hierarchy in the winsys data because all EGL winsys's
      need the EGL winsys data but only one of them wants the Xlib
      data. */
 
-  data = cogl_object_get_user_data (COGL_OBJECT (renderer), &key);
+  if (!renderer->custom_winsys_user_data)
+    renderer->custom_winsys_user_data = g_slice_new0 (CoglXlibRenderer);
 
-  if (data == NULL)
-    {
-      data = g_slice_new0 (CoglXlibRenderer);
-
-      cogl_object_set_user_data (COGL_OBJECT (renderer),
-                                 &key,
-                                 data,
-                                 destroy_xlib_renderer_data);
-    }
-
-  return data;
+  return renderer->custom_winsys_user_data;
 }
 
 static void
@@ -570,6 +556,8 @@ _cogl_xlib_renderer_disconnect (CoglRenderer *renderer)
   if (!renderer->foreign_xdpy && xlib_renderer->xdpy)
     XCloseDisplay (xlib_renderer->xdpy);
 
+  g_clear_pointer (&renderer->custom_winsys_user_data, _xlib_renderer_data_free);
+
   unregister_xlib_renderer (renderer);
 }