diff --git a/src/backends/meta-cursor.c b/src/backends/meta-cursor.c
index 22a8a76c7..e70b32850 100644
--- a/src/backends/meta-cursor.c
+++ b/src/backends/meta-cursor.c
@@ -255,22 +255,30 @@ meta_cursor_image_load_from_xcursor_image (MetaCursorImage   *image,
 #endif
 }
 
+static void
+load_cursor_image (MetaCursorReference *cursor)
+{
+  XcursorImage *image;
+
+  /* Either cursors are loaded from X cursors or buffers. Since
+   * buffers are converted over immediately, we can make sure to
+   * load this directly. */
+  g_assert (cursor->cursor != META_CURSOR_NONE);
+
+  image = load_cursor_on_client (cursor->cursor);
+  if (!image)
+    return;
+
+  meta_cursor_image_load_from_xcursor_image (&cursor->image, image);
+  XcursorImageDestroy (image);
+}
+
 MetaCursorReference *
 meta_cursor_reference_from_theme (MetaCursor cursor)
 {
-  MetaCursorReference *self;
-  XcursorImage *image;
-
-  image = load_cursor_on_client (cursor);
-  if (!image)
-    return NULL;
-
-  self = g_slice_new0 (MetaCursorReference);
+  MetaCursorReference *self = g_slice_new0 (MetaCursorReference);
   self->ref_count = 1;
   self->cursor = cursor;
-  meta_cursor_image_load_from_xcursor_image (&self->image, image);
-
-  XcursorImageDestroy (image);
   return self;
 }
 
@@ -380,10 +388,14 @@ meta_cursor_reference_get_cogl_texture (MetaCursorReference *cursor,
                                         int                 *hot_x,
                                         int                 *hot_y)
 {
+  if (!cursor->image.texture)
+    load_cursor_image (cursor);
+
   if (hot_x)
     *hot_x = cursor->image.hot_x;
   if (hot_y)
     *hot_y = cursor->image.hot_y;
+
   return COGL_TEXTURE (cursor->image.texture);
 }
 
@@ -393,6 +405,9 @@ meta_cursor_reference_get_gbm_bo (MetaCursorReference *cursor,
                                   int                 *hot_x,
                                   int                 *hot_y)
 {
+  if (!cursor->image.bo)
+    load_cursor_image (cursor);
+
   if (hot_x)
     *hot_x = cursor->image.hot_x;
   if (hot_y)