diff --git a/src/backends/meta-cursor-renderer.c b/src/backends/meta-cursor-renderer.c
index bf10d5e2b..ae398730f 100644
--- a/src/backends/meta-cursor-renderer.c
+++ b/src/backends/meta-cursor-renderer.c
@@ -283,13 +283,3 @@ meta_cursor_renderer_get_cursor (MetaCursorRenderer *renderer)
 
   return priv->displayed_cursor;
 }
-
-void
-meta_cursor_renderer_realize_cursor_sprite (MetaCursorRenderer *renderer,
-                                            MetaCursorSprite   *cursor_sprite)
-{
-  MetaCursorRendererClass *renderer_class = META_CURSOR_RENDERER_GET_CLASS (renderer);
-
-  if (renderer_class->realize_cursor_sprite)
-    renderer_class->realize_cursor_sprite (renderer, cursor_sprite);
-}
diff --git a/src/backends/meta-cursor-renderer.h b/src/backends/meta-cursor-renderer.h
index 82581995f..830d16ef6 100644
--- a/src/backends/meta-cursor-renderer.h
+++ b/src/backends/meta-cursor-renderer.h
@@ -40,8 +40,6 @@ struct _MetaCursorRendererClass
 
   gboolean (* update_cursor) (MetaCursorRenderer *renderer,
                               MetaCursorSprite   *cursor_sprite);
-  void (* realize_cursor_sprite) (MetaCursorRenderer *renderer,
-                                  MetaCursorSprite   *cursor_sprite);
 };
 
 MetaCursorRenderer * meta_cursor_renderer_new (void);
@@ -60,9 +58,6 @@ MetaCursorSprite * meta_cursor_renderer_get_cursor (MetaCursorRenderer *renderer
 ClutterRect meta_cursor_renderer_calculate_rect (MetaCursorRenderer *renderer,
                                                  MetaCursorSprite   *cursor_sprite);
 
-void meta_cursor_renderer_realize_cursor_sprite (MetaCursorRenderer *renderer,
-                                                 MetaCursorSprite   *cursor_sprite);
-
 void meta_cursor_renderer_emit_painted (MetaCursorRenderer *renderer,
                                         MetaCursorSprite   *cursor_sprite);
 
diff --git a/src/backends/meta-cursor-sprite-xcursor.c b/src/backends/meta-cursor-sprite-xcursor.c
index bd5d6a94a..49d10b94c 100644
--- a/src/backends/meta-cursor-sprite-xcursor.c
+++ b/src/backends/meta-cursor-sprite-xcursor.c
@@ -20,7 +20,6 @@
 
 #include "backends/meta-cursor-sprite-xcursor.h"
 
-#include "backends/meta-backend-private.h"
 #include "backends/meta-cursor.h"
 #include "backends/meta-cursor-renderer.h"
 #include "clutter/clutter.h"
@@ -118,8 +117,6 @@ static void
 load_from_current_xcursor_image (MetaCursorSpriteXcursor *sprite_xcursor)
 {
   MetaCursorSprite *sprite = META_CURSOR_SPRITE (sprite_xcursor);
-  MetaBackend *backend = meta_get_backend ();
-  MetaCursorRenderer *renderer = meta_backend_get_cursor_renderer (backend);
   XcursorImage *xc_image;
   int width, height, rowstride;
   CoglPixelFormat cogl_format;
@@ -160,8 +157,6 @@ load_from_current_xcursor_image (MetaCursorSpriteXcursor *sprite_xcursor)
                                   xc_image->xhot, xc_image->yhot);
 
   g_clear_pointer (&texture, cogl_object_unref);
-
-  meta_cursor_renderer_realize_cursor_sprite (renderer, sprite);
 }
 
 void
diff --git a/src/backends/native/meta-cursor-renderer-native.c b/src/backends/native/meta-cursor-renderer-native.c
index 310200bc4..fcecea126 100644
--- a/src/backends/native/meta-cursor-renderer-native.c
+++ b/src/backends/native/meta-cursor-renderer-native.c
@@ -119,6 +119,10 @@ static GQuark quark_cursor_renderer_native_gpu_data = 0;
 
 G_DEFINE_TYPE_WITH_PRIVATE (MetaCursorRendererNative, meta_cursor_renderer_native, META_TYPE_CURSOR_RENDERER);
 
+static void
+realize_cursor_sprite (MetaCursorRenderer *renderer,
+                       MetaCursorSprite   *cursor_sprite);
+
 static MetaCursorNativeGpuState *
 get_cursor_gpu_state (MetaCursorNativePrivate *cursor_priv,
                       MetaGpuKms              *gpu_kms);
@@ -675,7 +679,7 @@ meta_cursor_renderer_native_update_cursor (MetaCursorRenderer *renderer,
     meta_cursor_renderer_native_get_instance_private (native);
 
   if (cursor_sprite)
-    meta_cursor_sprite_realize_texture (cursor_sprite);
+    realize_cursor_sprite (renderer, cursor_sprite);
 
   maybe_schedule_cursor_sprite_animation_frame (native, cursor_sprite);
 
@@ -717,6 +721,24 @@ ensure_cursor_gpu_state (MetaCursorNativePrivate *cursor_priv,
   return cursor_gpu_state;
 }
 
+static void
+on_cursor_sprite_texture_changed (MetaCursorSprite *cursor_sprite)
+{
+  MetaCursorNativePrivate *cursor_priv = get_cursor_priv (cursor_sprite);
+  GHashTableIter iter;
+  MetaCursorNativeGpuState *cursor_gpu_state;
+
+  g_hash_table_iter_init (&iter, cursor_priv->gpu_states);
+  while (g_hash_table_iter_next (&iter, NULL, (gpointer *) &cursor_gpu_state))
+    {
+      guint pending_bo;
+      pending_bo = get_pending_cursor_sprite_gbm_bo_index (cursor_gpu_state);
+      g_clear_pointer (&cursor_gpu_state->bos[pending_bo],
+                       (GDestroyNotify) gbm_bo_destroy);
+      cursor_gpu_state->pending_bo_state = META_CURSOR_GBM_BO_STATE_INVALIDATED;
+    }
+}
+
 static void
 cursor_priv_free (MetaCursorNativePrivate *cursor_priv)
 {
@@ -749,6 +771,9 @@ ensure_cursor_priv (MetaCursorSprite *cursor_sprite)
                            cursor_priv,
                            (GDestroyNotify) cursor_priv_free);
 
+  g_signal_connect (cursor_sprite, "texture-changed",
+                    G_CALLBACK (on_cursor_sprite_texture_changed), NULL);
+
   return cursor_priv;
 }
 
@@ -816,26 +841,31 @@ load_cursor_sprite_gbm_buffer_for_gpu (MetaCursorRendererNative *native,
     }
 }
 
-static void
-invalidate_pending_cursor_sprite_gbm_bo (MetaCursorSprite *cursor_sprite,
-                                         MetaGpuKms       *gpu_kms)
+static gboolean
+is_cursor_hw_state_valid (MetaCursorSprite *cursor_sprite,
+                          MetaGpuKms       *gpu_kms)
 {
   MetaCursorNativePrivate *cursor_priv;
   MetaCursorNativeGpuState *cursor_gpu_state;
-  guint pending_bo;
 
   cursor_priv = get_cursor_priv (cursor_sprite);
   if (!cursor_priv)
-    return;
+    return FALSE;
 
   cursor_gpu_state = get_cursor_gpu_state (cursor_priv, gpu_kms);
   if (!cursor_gpu_state)
-    return;
+    return FALSE;
 
-  pending_bo = get_pending_cursor_sprite_gbm_bo_index (cursor_gpu_state);
-  g_clear_pointer (&cursor_gpu_state->bos[pending_bo],
-                   (GDestroyNotify) gbm_bo_destroy);
-  cursor_gpu_state->pending_bo_state = META_CURSOR_GBM_BO_STATE_INVALIDATED;
+  switch (cursor_gpu_state->pending_bo_state)
+    {
+    case META_CURSOR_GBM_BO_STATE_SET:
+    case META_CURSOR_GBM_BO_STATE_NONE:
+      return TRUE;
+    case META_CURSOR_GBM_BO_STATE_INVALIDATED:
+      return FALSE;
+    }
+
+  g_assert_not_reached ();
 }
 
 #ifdef HAVE_WAYLAND
@@ -858,10 +888,8 @@ realize_cursor_sprite_from_wl_buffer_for_gpu (MetaCursorRenderer      *renderer,
   if (!cursor_renderer_gpu_data || cursor_renderer_gpu_data->hw_cursor_broken)
     return;
 
-  /* Destroy any previous pending cursor buffer; we'll always either fail (which
-   * should unset, or succeed, which will set new buffer.
-   */
-  invalidate_pending_cursor_sprite_gbm_bo (cursor_sprite, gpu_kms);
+  if (is_cursor_hw_state_valid (cursor_sprite, gpu_kms))
+    return;
 
   texture = meta_cursor_sprite_get_cogl_texture (cursor_sprite);
   width = cogl_texture_get_width (texture);
@@ -990,7 +1018,8 @@ realize_cursor_sprite_from_xcursor_for_gpu (MetaCursorRenderer      *renderer,
   if (!cursor_renderer_gpu_data || cursor_renderer_gpu_data->hw_cursor_broken)
     return;
 
-  invalidate_pending_cursor_sprite_gbm_bo (cursor_sprite, gpu_kms);
+  if (is_cursor_hw_state_valid (cursor_sprite, gpu_kms))
+    return;
 
   load_cursor_sprite_gbm_buffer_for_gpu (native,
                                          gpu_kms,
@@ -1027,9 +1056,11 @@ realize_cursor_sprite_from_xcursor (MetaCursorRenderer      *renderer,
 }
 
 static void
-meta_cursor_renderer_native_realize_cursor_sprite (MetaCursorRenderer *renderer,
-                                                   MetaCursorSprite   *cursor_sprite)
+realize_cursor_sprite (MetaCursorRenderer *renderer,
+                       MetaCursorSprite   *cursor_sprite)
 {
+  meta_cursor_sprite_realize_texture (cursor_sprite);
+
   if (META_IS_CURSOR_SPRITE_XCURSOR (cursor_sprite))
     {
       MetaCursorSpriteXcursor *sprite_xcursor =
@@ -1056,8 +1087,6 @@ meta_cursor_renderer_native_class_init (MetaCursorRendererNativeClass *klass)
 
   object_class->finalize = meta_cursor_renderer_native_finalize;
   renderer_class->update_cursor = meta_cursor_renderer_native_update_cursor;
-  renderer_class->realize_cursor_sprite =
-    meta_cursor_renderer_native_realize_cursor_sprite;
 
   quark_cursor_sprite = g_quark_from_static_string ("-meta-cursor-native");
   quark_cursor_renderer_native_gpu_data =
diff --git a/src/wayland/meta-wayland-cursor-surface.c b/src/wayland/meta-wayland-cursor-surface.c
index 7b7d63c28..d08af9e8c 100644
--- a/src/wayland/meta-wayland-cursor-surface.c
+++ b/src/wayland/meta-wayland-cursor-surface.c
@@ -71,16 +71,6 @@ update_cursor_sprite_texture (MetaWaylandCursorSurface *cursor_surface)
                                       buffer->texture,
                                       priv->hot_x * surface->scale,
                                       priv->hot_y * surface->scale);
-
-      if (priv->buffer)
-        {
-          g_assert (priv->buffer == buffer);
-          meta_cursor_renderer_realize_cursor_sprite (priv->cursor_renderer,
-                                                      cursor_sprite);
-
-          meta_wayland_surface_unref_buffer_use_count (surface);
-          g_clear_object (&priv->buffer);
-        }
     }
   else
     {