diff --git a/cogl/cogl-glx-renderer-private.h b/cogl/cogl-glx-renderer-private.h
index 2d36a5f88..d76ef6220 100644
--- a/cogl/cogl-glx-renderer-private.h
+++ b/cogl/cogl-glx-renderer-private.h
@@ -31,8 +31,6 @@
 
 typedef struct _CoglGLXRenderer
 {
-  CoglXlibRenderer _parent;
-
   int glx_major;
   int glx_minor;
 
diff --git a/cogl/cogl-xlib-renderer-private.h b/cogl/cogl-xlib-renderer-private.h
index c6ed58221..bad70babb 100644
--- a/cogl/cogl-xlib-renderer-private.h
+++ b/cogl/cogl-xlib-renderer-private.h
@@ -74,4 +74,7 @@ int
 _cogl_xlib_renderer_untrap_errors (CoglRenderer *renderer,
                                    CoglXlibTrapState *state);
 
+CoglXlibRenderer *
+_cogl_xlib_renderer_get_data (CoglRenderer *renderer);
+
 #endif /* __COGL_RENDERER_XLIB_PRIVATE_H */
diff --git a/cogl/cogl-xlib-renderer.c b/cogl/cogl-xlib-renderer.c
index 2c5571324..3a96b5170 100644
--- a/cogl/cogl-xlib-renderer.c
+++ b/cogl/cogl-xlib-renderer.c
@@ -46,6 +46,39 @@
 static char *_cogl_x11_display_name = NULL;
 static GList *_cogl_xlib_renderers = NULL;
 
+static void
+destroy_xlib_renderer_data (void *user_data)
+{
+  g_slice_free (CoglXlibRenderer, user_data);
+}
+
+CoglXlibRenderer *
+_cogl_xlib_renderer_get_data (CoglRenderer *renderer)
+{
+  static CoglUserDataKey key;
+  CoglXlibRenderer *data;
+
+  /* Constructs a CoglXlibRenderer struct on demand and attaches it to
+     the object using user data. It's done this way instead of using a
+     subclassing hierarchy in the winsys data because all EGL winsys's
+     need the EGL winsys data but only one of them wants the Xlib
+     data. */
+
+  data = cogl_object_get_user_data (COGL_OBJECT (renderer), &key);
+
+  if (data == NULL)
+    {
+      data = g_slice_new0 (CoglXlibRenderer);
+
+      cogl_object_set_user_data (COGL_OBJECT (renderer),
+                                 &key,
+                                 data,
+                                 destroy_xlib_renderer_data);
+    }
+
+  return data;
+}
+
 static void
 register_xlib_renderer (CoglRenderer *renderer)
 {
@@ -72,7 +105,8 @@ get_renderer_for_xdisplay (Display *xdpy)
   for (l = _cogl_xlib_renderers; l; l = l->next)
     {
       CoglRenderer *renderer = l->data;
-      CoglXlibRenderer *xlib_renderer = renderer->winsys;
+      CoglXlibRenderer *xlib_renderer =
+        _cogl_xlib_renderer_get_data (renderer);
 
       if (xlib_renderer->xdpy == xdpy)
         return renderer;
@@ -90,7 +124,7 @@ error_handler (Display *xdpy,
 
   renderer = get_renderer_for_xdisplay (xdpy);
 
-  xlib_renderer = renderer->winsys;
+  xlib_renderer = _cogl_xlib_renderer_get_data (renderer);
   g_assert (xlib_renderer->trap_state);
 
   xlib_renderer->trap_state->trapped_error_code = error->error_code;
@@ -104,7 +138,7 @@ _cogl_xlib_renderer_trap_errors (CoglRenderer *renderer,
 {
   CoglXlibRenderer *xlib_renderer;
 
-  xlib_renderer = renderer->winsys;
+  xlib_renderer = _cogl_xlib_renderer_get_data (renderer);
 
   state->trapped_error_code = 0;
   state->old_error_handler = XSetErrorHandler (error_handler);
@@ -119,7 +153,7 @@ _cogl_xlib_renderer_untrap_errors (CoglRenderer *renderer,
 {
   CoglXlibRenderer *xlib_renderer;
 
-  xlib_renderer = renderer->winsys;
+  xlib_renderer = _cogl_xlib_renderer_get_data (renderer);
   g_assert (state == xlib_renderer->trap_state);
 
   XSetErrorHandler (state->old_error_handler);
@@ -133,7 +167,7 @@ static Display *
 assert_xlib_display (CoglRenderer *renderer, GError **error)
 {
   Display *xdpy = cogl_xlib_renderer_get_foreign_display (renderer);
-  CoglXlibRenderer *xlib_renderer = renderer->winsys;
+  CoglXlibRenderer *xlib_renderer = _cogl_xlib_renderer_get_data (renderer);
 
   /* A foreign display may have already been set... */
   if (xdpy)
@@ -159,8 +193,10 @@ assert_xlib_display (CoglRenderer *renderer, GError **error)
 gboolean
 _cogl_xlib_renderer_connect (CoglRenderer *renderer, GError **error)
 {
-  CoglXlibRenderer *xlib_renderer = renderer->winsys;
-  CoglX11Renderer *x11_renderer = renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (renderer);
+  CoglX11Renderer *x11_renderer =
+    (CoglX11Renderer *) xlib_renderer;
   int damage_error;
 
   if (!assert_xlib_display (renderer, error))
@@ -185,7 +221,8 @@ _cogl_xlib_renderer_connect (CoglRenderer *renderer, GError **error)
 void
 _cogl_xlib_renderer_disconnect (CoglRenderer *renderer)
 {
-  CoglXlibRenderer *xlib_renderer = renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (renderer);
 
   if (!renderer->foreign_xdpy && xlib_renderer->xdpy)
     XCloseDisplay (xlib_renderer->xdpy);
@@ -200,7 +237,7 @@ cogl_xlib_renderer_get_display (CoglRenderer *renderer)
 
   _COGL_RETURN_VAL_IF_FAIL (cogl_is_renderer (renderer), NULL);
 
-  xlib_renderer = renderer->winsys;
+  xlib_renderer = _cogl_xlib_renderer_get_data (renderer);
 
   return xlib_renderer->xdpy;
 }
diff --git a/cogl/cogl-xlib.c b/cogl/cogl-xlib.c
index 2c96872dc..6c8ebea3a 100644
--- a/cogl/cogl-xlib.c
+++ b/cogl/cogl-xlib.c
@@ -102,6 +102,7 @@ _cogl_xlib_get_damage_base (void)
   CoglX11Renderer *x11_renderer;
   _COGL_GET_CONTEXT (ctxt, -1);
 
-  x11_renderer = ctxt->display->renderer->winsys;
+  x11_renderer =
+    (CoglX11Renderer *) _cogl_xlib_renderer_get_data (ctxt->display->renderer);
   return x11_renderer->damage_base;
 }
diff --git a/cogl/winsys/cogl-winsys-egl-private.h b/cogl/winsys/cogl-winsys-egl-private.h
index cb6b13ae1..cd0896f2c 100644
--- a/cogl/winsys/cogl-winsys-egl-private.h
+++ b/cogl/winsys/cogl-winsys-egl-private.h
@@ -66,10 +66,6 @@ typedef enum _CoglEGLWinsysFeature
 
 typedef struct _CoglRendererEGL
 {
-#ifdef COGL_HAS_EGL_PLATFORM_POWERVR_X11_SUPPORT
-  CoglXlibRenderer _parent;
-#endif
-
   CoglEGLWinsysFeature private_features;
 
 #ifdef COGL_HAS_EGL_PLATFORM_WAYLAND_SUPPORT
diff --git a/cogl/winsys/cogl-winsys-egl.c b/cogl/winsys/cogl-winsys-egl.c
index 269152f57..f3141c94d 100644
--- a/cogl/winsys/cogl-winsys-egl.c
+++ b/cogl/winsys/cogl-winsys-egl.c
@@ -304,7 +304,7 @@ _cogl_winsys_renderer_connect (CoglRenderer *renderer,
 
 #ifdef COGL_HAS_EGL_PLATFORM_POWERVR_X11_SUPPORT
     case COGL_WINSYS_ID_EGL_X11:
-      xlib_renderer = renderer->winsys;
+      xlib_renderer = _cogl_xlib_renderer_get_data (renderer);
 
       if (!_cogl_xlib_renderer_connect (renderer, error))
         goto error;
@@ -454,7 +454,8 @@ update_winsys_features (CoglContext *context, GError **error)
 static XVisualInfo *
 get_visual_info (CoglDisplay *display, EGLConfig egl_config)
 {
-  CoglXlibRenderer *xlib_renderer = display->renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (display->renderer);
   CoglRendererEGL *egl_renderer = display->renderer->winsys;
   XVisualInfo visinfo_template;
   int template_mask = 0;
@@ -571,7 +572,8 @@ try_create_context (CoglDisplay *display,
   CoglDisplayEGL *egl_display = display->winsys;
 #ifdef COGL_HAS_EGL_PLATFORM_POWERVR_X11_SUPPORT
   CoglXlibDisplay *xlib_display = display->winsys;
-  CoglXlibRenderer *xlib_renderer = renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (renderer);
 #endif
   CoglRendererEGL *egl_renderer = renderer->winsys;
   EGLDisplay edpy;
@@ -881,7 +883,8 @@ cleanup_context (CoglDisplay *display)
   CoglRendererEGL *egl_renderer = renderer->winsys;
 #ifdef COGL_HAS_EGL_PLATFORM_POWERVR_X11_SUPPORT
   CoglXlibDisplay *xlib_display = display->winsys;
-  CoglXlibRenderer *xlib_renderer = renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (renderer);
 #endif
 
   if (egl_renderer->platform_vtable &&
@@ -1165,7 +1168,8 @@ _cogl_winsys_onscreen_init (CoglOnscreen *onscreen,
   CoglRenderer *renderer = display->renderer;
   CoglRendererEGL *egl_renderer = renderer->winsys;
 #ifdef COGL_HAS_EGL_PLATFORM_POWERVR_X11_SUPPORT
-  CoglXlibRenderer *xlib_renderer = renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (renderer);
   CoglOnscreenXlib *xlib_onscreen;
   Window xwin;
 #endif
@@ -1426,7 +1430,8 @@ _cogl_winsys_onscreen_deinit (CoglOnscreen *onscreen)
   CoglRenderer *renderer = context->display->renderer;
   CoglRendererEGL *egl_renderer = renderer->winsys;
 #ifdef COGL_HAS_EGL_PLATFORM_POWERVR_X11_SUPPORT
-  CoglXlibRenderer *xlib_renderer = renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (renderer);
   CoglXlibTrapState old_state;
   CoglOnscreenXlib *xlib_onscreen = onscreen->winsys;
 #endif
@@ -1589,7 +1594,8 @@ _cogl_winsys_onscreen_set_visibility (CoglOnscreen *onscreen,
 {
   CoglContext *context = COGL_FRAMEBUFFER (onscreen)->context;
   CoglRenderer *renderer = context->display->renderer;
-  CoglXlibRenderer *xlib_renderer = renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (renderer);
   CoglOnscreenXlib *xlib_onscreen = onscreen->winsys;
 
   if (renderer->winsys_vtable->id != COGL_WINSYS_ID_EGL_X11)
diff --git a/cogl/winsys/cogl-winsys-glx.c b/cogl/winsys/cogl-winsys-glx.c
index 58640afc9..e1dbdbe1f 100644
--- a/cogl/winsys/cogl-winsys-glx.c
+++ b/cogl/winsys/cogl-winsys-glx.c
@@ -314,7 +314,7 @@ _cogl_winsys_renderer_connect (CoglRenderer *renderer,
   renderer->winsys = g_slice_new0 (CoglGLXRenderer);
 
   glx_renderer = renderer->winsys;
-  xlib_renderer = renderer->winsys;
+  xlib_renderer = _cogl_xlib_renderer_get_data (renderer);
 
   if (!_cogl_xlib_renderer_connect (renderer, error))
     goto error;
@@ -378,7 +378,8 @@ static gboolean
 update_winsys_features (CoglContext *context, GError **error)
 {
   CoglGLXDisplay *glx_display = context->display->winsys;
-  CoglXlibRenderer *xlib_renderer = context->display->renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (context->display->renderer);
   CoglGLXRenderer *glx_renderer = context->display->renderer->winsys;
   const char *glx_extensions;
   int default_screen;
@@ -514,7 +515,8 @@ find_fbconfig (CoglDisplay *display,
                GLXFBConfig *config_ret,
                GError **error)
 {
-  CoglXlibRenderer *xlib_renderer = display->renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (display->renderer);
   CoglGLXRenderer *glx_renderer = display->renderer->winsys;
   GLXFBConfig *configs = NULL;
   int n_configs;
@@ -582,7 +584,8 @@ create_context (CoglDisplay *display, GError **error)
 {
   CoglGLXDisplay *glx_display = display->winsys;
   CoglXlibDisplay *xlib_display = display->winsys;
-  CoglXlibRenderer *xlib_renderer = display->renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (display->renderer);
   CoglGLXRenderer *glx_renderer = display->renderer->winsys;
   gboolean support_transparent_windows =
     display->onscreen_template->config.swap_chain->has_alpha;
@@ -712,7 +715,8 @@ _cogl_winsys_display_destroy (CoglDisplay *display)
 {
   CoglGLXDisplay *glx_display = display->winsys;
   CoglXlibDisplay *xlib_display = display->winsys;
-  CoglXlibRenderer *xlib_renderer = display->renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (display->renderer);
   CoglGLXRenderer *glx_renderer = display->renderer->winsys;
 
   _COGL_RETURN_IF_FAIL (glx_display != NULL);
@@ -796,7 +800,8 @@ _cogl_winsys_onscreen_init (CoglOnscreen *onscreen,
   CoglContext *context = framebuffer->context;
   CoglDisplay *display = context->display;
   CoglGLXDisplay *glx_display = display->winsys;
-  CoglXlibRenderer *xlib_renderer = display->renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (display->renderer);
   CoglGLXRenderer *glx_renderer = display->renderer->winsys;
   Window xwin;
   CoglOnscreenXlib *xlib_onscreen;
@@ -982,7 +987,8 @@ _cogl_winsys_onscreen_deinit (CoglOnscreen *onscreen)
 {
   CoglFramebuffer *framebuffer = COGL_FRAMEBUFFER (onscreen);
   CoglContext *context = framebuffer->context;
-  CoglXlibRenderer *xlib_renderer = context->display->renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (context->display->renderer);
   CoglGLXRenderer *glx_renderer = context->display->renderer->winsys;
   CoglXlibTrapState old_state;
   CoglOnscreenXlib *xlib_onscreen = onscreen->winsys;
@@ -1024,7 +1030,8 @@ _cogl_winsys_onscreen_bind (CoglOnscreen *onscreen)
   CoglContextGLX *glx_context = context->winsys;
   CoglXlibDisplay *xlib_display = context->display->winsys;
   CoglGLXDisplay *glx_display = context->display->winsys;
-  CoglXlibRenderer *xlib_renderer = context->display->renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (context->display->renderer);
   CoglGLXRenderer *glx_renderer = context->display->renderer->winsys;
   CoglOnscreenXlib *xlib_onscreen = onscreen->winsys;
   CoglOnscreenGLX *glx_onscreen = onscreen->winsys;
@@ -1184,7 +1191,8 @@ _cogl_winsys_onscreen_swap_region (CoglOnscreen *onscreen,
 {
   CoglFramebuffer *framebuffer = COGL_FRAMEBUFFER (onscreen);
   CoglContext *context = framebuffer->context;
-  CoglXlibRenderer *xlib_renderer = context->display->renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (context->display->renderer);
   CoglGLXRenderer *glx_renderer = context->display->renderer->winsys;
   CoglOnscreenXlib *xlib_onscreen = onscreen->winsys;
   CoglOnscreenGLX *glx_onscreen = onscreen->winsys;
@@ -1338,7 +1346,8 @@ _cogl_winsys_onscreen_swap_buffers (CoglOnscreen *onscreen)
 {
   CoglFramebuffer *framebuffer = COGL_FRAMEBUFFER (onscreen);
   CoglContext *context = framebuffer->context;
-  CoglXlibRenderer *xlib_renderer = context->display->renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (context->display->renderer);
   CoglGLXRenderer *glx_renderer = context->display->renderer->winsys;
   CoglOnscreenXlib *xlib_onscreen = onscreen->winsys;
   CoglOnscreenGLX *glx_onscreen = onscreen->winsys;
@@ -1478,7 +1487,8 @@ _cogl_winsys_onscreen_set_visibility (CoglOnscreen *onscreen,
                                       gboolean visibility)
 {
   CoglContext *context = COGL_FRAMEBUFFER (onscreen)->context;
-  CoglXlibRenderer *xlib_renderer = context->display->renderer->winsys;
+  CoglXlibRenderer *xlib_renderer =
+    _cogl_xlib_renderer_get_data (context->display->renderer);
   CoglOnscreenXlib *xlib_onscreen = onscreen->winsys;
 
   if (visibility)
@@ -1500,7 +1510,7 @@ _cogl_winsys_xlib_get_visual_info (void)
   _COGL_RETURN_VAL_IF_FAIL (ctx->display->winsys, FALSE);
 
   glx_display = ctx->display->winsys;
-  xlib_renderer = ctx->display->renderer->winsys;
+  xlib_renderer = _cogl_xlib_renderer_get_data (ctx->display->renderer);
   glx_renderer = ctx->display->renderer->winsys;
 
   if (!glx_display->found_fbconfig)
@@ -1526,7 +1536,7 @@ get_fbconfig_for_depth (CoglContext *context,
   int spare_cache_slot = 0;
   gboolean found = FALSE;
 
-  xlib_renderer = context->display->renderer->winsys;
+  xlib_renderer = _cogl_xlib_renderer_get_data (context->display->renderer);
   glx_renderer = context->display->renderer->winsys;
   glx_display = context->display->winsys;
 
@@ -1721,7 +1731,7 @@ try_create_glx_pixmap (CoglContext *context,
   Visual* visual = tex_pixmap->visual;
 
   renderer = context->display->renderer;
-  xlib_renderer = renderer->winsys;
+  xlib_renderer = _cogl_xlib_renderer_get_data (renderer);
   glx_renderer = renderer->winsys;
   dpy = xlib_renderer->xdpy;
 
@@ -1844,7 +1854,7 @@ free_glx_pixmap (CoglContext *context,
   CoglGLXRenderer *glx_renderer;
 
   renderer = context->display->renderer;
-  xlib_renderer = renderer->winsys;
+  xlib_renderer = _cogl_xlib_renderer_get_data (renderer);
   glx_renderer = renderer->winsys;
 
   if (glx_tex_pixmap->pixmap_bound)
@@ -2008,7 +2018,8 @@ _cogl_winsys_texture_pixmap_x11_update (CoglTexturePixmapX11 *tex_pixmap,
   if (glx_tex_pixmap->bind_tex_image_queued)
     {
       GLuint gl_handle, gl_target;
-      CoglXlibRenderer *xlib_renderer = ctx->display->renderer->winsys;
+      CoglXlibRenderer *xlib_renderer =
+        _cogl_xlib_renderer_get_data (ctx->display->renderer);
 
       cogl_texture_get_gl_texture (glx_tex_pixmap->glx_tex,
                                    &gl_handle, &gl_target);