diff --git a/src/backends/native/meta-crtc-kms.c b/src/backends/native/meta-crtc-kms.c
index 82b68ed6f..59570e60b 100644
--- a/src/backends/native/meta-crtc-kms.c
+++ b/src/backends/native/meta-crtc-kms.c
@@ -45,6 +45,7 @@ struct _MetaCrtcKms
   MetaKmsPlane *primary_plane;
 
   gpointer cursor_renderer_private;
+  GDestroyNotify cursor_renderer_private_destroy_notify;
 };
 
 static GQuark kms_crtc_crtc_kms_quark;
@@ -58,10 +59,15 @@ meta_crtc_kms_get_cursor_renderer_private (MetaCrtcKms *crtc_kms)
 }
 
 void
-meta_crtc_kms_set_cursor_renderer_private (MetaCrtcKms *crtc_kms,
-                                           gpointer     cursor_renderer_private)
+meta_crtc_kms_set_cursor_renderer_private (MetaCrtcKms    *crtc_kms,
+                                           gpointer        cursor_renderer_private,
+                                           GDestroyNotify  destroy_notify)
 {
+  g_clear_pointer (&crtc_kms->cursor_renderer_private,
+                   crtc_kms->cursor_renderer_private_destroy_notify);
+
   crtc_kms->cursor_renderer_private = cursor_renderer_private;
+  crtc_kms->cursor_renderer_private_destroy_notify = destroy_notify;
 }
 
 gboolean
@@ -298,6 +304,17 @@ meta_crtc_kms_new (MetaGpuKms  *gpu_kms,
   return crtc_kms;
 }
 
+static void
+meta_crtc_kms_dispose (GObject *object)
+{
+  MetaCrtcKms *crtc_kms = META_CRTC_KMS (object);
+
+  g_clear_pointer (&crtc_kms->cursor_renderer_private,
+                   crtc_kms->cursor_renderer_private_destroy_notify);
+
+  G_OBJECT_CLASS (meta_crtc_kms_parent_class)->dispose (object);
+}
+
 static void
 meta_crtc_kms_init (MetaCrtcKms *crtc_kms)
 {
@@ -306,4 +323,7 @@ meta_crtc_kms_init (MetaCrtcKms *crtc_kms)
 static void
 meta_crtc_kms_class_init (MetaCrtcKmsClass *klass)
 {
+  GObjectClass *object_class = G_OBJECT_CLASS (klass);
+
+  object_class->dispose = meta_crtc_kms_dispose;
 }
diff --git a/src/backends/native/meta-crtc-kms.h b/src/backends/native/meta-crtc-kms.h
index c13034eeb..0c599844e 100644
--- a/src/backends/native/meta-crtc-kms.h
+++ b/src/backends/native/meta-crtc-kms.h
@@ -40,8 +40,9 @@ G_DECLARE_FINAL_TYPE (MetaCrtcKms, meta_crtc_kms,
 
 gpointer meta_crtc_kms_get_cursor_renderer_private (MetaCrtcKms *crtc_kms);
 
-void meta_crtc_kms_set_cursor_renderer_private (MetaCrtcKms *crtc_kms,
-                                                gpointer     cursor_renderer_private);
+void meta_crtc_kms_set_cursor_renderer_private (MetaCrtcKms    *crtc_kms,
+                                                gpointer        cursor_renderer_private,
+                                                GDestroyNotify  destroy_notify);
 
 gboolean meta_crtc_kms_is_transform_handled (MetaCrtcKms          *crtc_kms,
                                              MetaMonitorTransform  transform);
diff --git a/src/backends/native/meta-cursor-renderer-native.c b/src/backends/native/meta-cursor-renderer-native.c
index 781d719b8..ff175e8e2 100644
--- a/src/backends/native/meta-cursor-renderer-native.c
+++ b/src/backends/native/meta-cursor-renderer-native.c
@@ -77,6 +77,11 @@
 
 static GQuark quark_cursor_sprite = 0;
 
+typedef struct _CrtcCursorData
+{
+  MetaDrmBuffer *buffer;
+} CrtcCursorData;
+
 struct _MetaCursorRendererNative
 {
   MetaCursorRenderer parent;
@@ -252,6 +257,23 @@ calculate_crtc_cursor_hotspot (MetaCursorSprite *cursor_sprite,
   *cursor_hotspot_y = (int) roundf (hot_y * scale);
 }
 
+static CrtcCursorData *
+ensure_crtc_cursor_data (MetaCrtcKms *crtc_kms)
+{
+  CrtcCursorData *crtc_cursor_data;
+
+  crtc_cursor_data = meta_crtc_kms_get_cursor_renderer_private (crtc_kms);
+  if (!crtc_cursor_data)
+    {
+      crtc_cursor_data = g_new0 (CrtcCursorData, 1);
+      meta_crtc_kms_set_cursor_renderer_private (crtc_kms,
+                                                 crtc_cursor_data,
+                                                 g_free);
+    }
+
+  return crtc_cursor_data;
+}
+
 static void
 set_crtc_cursor (MetaCursorRendererNative *native,
                  MetaKmsUpdate            *kms_update,
@@ -278,6 +300,7 @@ set_crtc_cursor (MetaCursorRendererNative *native,
   MetaRectangle dst_rect;
   MetaDrmBuffer *crtc_buffer;
   MetaKmsAssignPlaneFlag flags;
+  CrtcCursorData *crtc_cursor_data;
   int cursor_hotspot_x;
   int cursor_hotspot_y;
   MetaKmsPlaneAssignment *plane_assignment;
@@ -308,7 +331,8 @@ set_crtc_cursor (MetaCursorRendererNative *native,
   };
 
   flags = META_KMS_ASSIGN_PLANE_FLAG_ALLOW_FAIL;
-  crtc_buffer = meta_crtc_kms_get_cursor_renderer_private (crtc_kms);
+  crtc_cursor_data = ensure_crtc_cursor_data (crtc_kms);
+  crtc_buffer = crtc_cursor_data->buffer;
   if (!priv->hw_state_invalidated && buffer == crtc_buffer)
     flags |= META_KMS_ASSIGN_PLANE_FLAG_FB_UNCHANGED;
 
@@ -327,7 +351,7 @@ set_crtc_cursor (MetaCursorRendererNative *native,
                                                 cursor_hotspot_x,
                                                 cursor_hotspot_y);
 
-  meta_crtc_kms_set_cursor_renderer_private (crtc_kms, buffer);
+  crtc_cursor_data->buffer = buffer;
 
   if (cursor_gpu_state->pending_buffer_state == META_CURSOR_BUFFER_STATE_SET)
     {
@@ -348,6 +372,7 @@ unset_crtc_cursor (MetaCursorRendererNative *native,
   MetaKmsDevice *kms_device;
   MetaKmsPlane *cursor_plane;
   MetaDrmBuffer *crtc_buffer;
+  CrtcCursorData *crtc_cursor_data;
 
   crtc_buffer = meta_crtc_kms_get_cursor_renderer_private (crtc_kms);
   if (!priv->hw_state_invalidated && !crtc_buffer)
@@ -360,7 +385,9 @@ unset_crtc_cursor (MetaCursorRendererNative *native,
   if (cursor_plane)
     meta_kms_update_unassign_plane (kms_update, kms_crtc, cursor_plane);
 
-  meta_crtc_kms_set_cursor_renderer_private (crtc_kms, NULL);
+  crtc_cursor_data = meta_crtc_kms_get_cursor_renderer_private (crtc_kms);
+  if (crtc_cursor_data)
+    crtc_cursor_data->buffer = NULL;
 }
 
 static float
@@ -1126,7 +1153,7 @@ unset_crtc_cursor_renderer_privates (MetaGpu       *gpu,
 
       crtc_buffer = meta_crtc_kms_get_cursor_renderer_private (crtc_kms);
       if (buffer == crtc_buffer)
-        meta_crtc_kms_set_cursor_renderer_private (crtc_kms, NULL);
+        meta_crtc_kms_set_cursor_renderer_private (crtc_kms, NULL, NULL);
     }
 }