diff --git a/src/backends/native/meta-drm-buffer-dumb.c b/src/backends/native/meta-drm-buffer-dumb.c
index b0c713dec..87ed4c9ce 100644
--- a/src/backends/native/meta-drm-buffer-dumb.c
+++ b/src/backends/native/meta-drm-buffer-dumb.c
@@ -251,7 +251,7 @@ init_dumb_buffer (MetaDrmBufferDumb  *buffer_dumb,
     .handles = { create_arg.handle },
     .strides = { create_arg.pitch },
   };
-  if (!meta_drm_buffer_ensure_fb_id (buffer, &fb_args, error))
+  if (!meta_drm_buffer_do_ensure_fb_id (buffer, &fb_args, error))
     goto err_add_fb;
 
   map_arg = (struct drm_mode_map_dumb) {
diff --git a/src/backends/native/meta-drm-buffer-gbm.c b/src/backends/native/meta-drm-buffer-gbm.c
index 7a3abaa6b..30a8f5cbc 100644
--- a/src/backends/native/meta-drm-buffer-gbm.c
+++ b/src/backends/native/meta-drm-buffer-gbm.c
@@ -133,11 +133,12 @@ meta_drm_buffer_gbm_get_modifier (MetaDrmBuffer *buffer)
 }
 
 static gboolean
-init_fb_id (MetaDrmBufferGbm  *buffer_gbm,
-            struct gbm_bo     *bo,
-            GError           **error)
+meta_drm_buffer_gbm_ensure_fb_id (MetaDrmBuffer  *buffer,
+                                  GError        **error)
 {
+  MetaDrmBufferGbm *buffer_gbm = META_DRM_BUFFER_GBM (buffer);
   MetaDrmFbArgs fb_args = { 0, };
+  struct gbm_bo *bo = buffer_gbm->bo;
 
   if (gbm_bo_get_handle_for_plane (bo, 0).s32 == -1)
     {
@@ -164,8 +165,8 @@ init_fb_id (MetaDrmBufferGbm  *buffer_gbm,
   fb_args.height = gbm_bo_get_height (bo);
   fb_args.format = gbm_bo_get_format (bo);
 
-  if (!meta_drm_buffer_ensure_fb_id (META_DRM_BUFFER (buffer_gbm),
-                                     &fb_args, error))
+  if (!meta_drm_buffer_do_ensure_fb_id (META_DRM_BUFFER (buffer_gbm),
+                                        &fb_args, error))
     return FALSE;
 
   return TRUE;
@@ -185,7 +186,7 @@ lock_front_buffer (MetaDrmBufferGbm  *buffer_gbm,
       return FALSE;
     }
 
-  return init_fb_id (buffer_gbm, buffer_gbm->bo, error);
+  return meta_drm_buffer_gbm_ensure_fb_id (META_DRM_BUFFER (buffer_gbm), error);
 }
 
 MetaDrmBufferGbm *
@@ -223,13 +224,6 @@ meta_drm_buffer_gbm_new_take (MetaDeviceFile      *device_file,
                              "device-file", device_file,
                              "flags", flags,
                              NULL);
-
-  if (!init_fb_id (buffer_gbm, bo, error))
-    {
-      g_object_unref (buffer_gbm);
-      return NULL;
-    }
-
   buffer_gbm->bo = bo;
 
   return buffer_gbm;
@@ -510,6 +504,7 @@ meta_drm_buffer_gbm_class_init (MetaDrmBufferGbmClass *klass)
   object_class->finalize = meta_drm_buffer_gbm_finalize;
 
   buffer_class->export_fd = meta_drm_buffer_gbm_export_fd;
+  buffer_class->ensure_fb_id = meta_drm_buffer_gbm_ensure_fb_id;
   buffer_class->get_width = meta_drm_buffer_gbm_get_width;
   buffer_class->get_height = meta_drm_buffer_gbm_get_height;
   buffer_class->get_stride = meta_drm_buffer_gbm_get_stride;
diff --git a/src/backends/native/meta-drm-buffer-import.c b/src/backends/native/meta-drm-buffer-import.c
index 4769e5326..698bbda08 100644
--- a/src/backends/native/meta-drm-buffer-import.c
+++ b/src/backends/native/meta-drm-buffer-import.c
@@ -181,9 +181,9 @@ import_gbm_buffer (MetaDrmBufferImport  *buffer_import,
 
   fb_args.handles[0] = gbm_bo_get_handle (imported_bo).u32;
 
-  ret = meta_drm_buffer_ensure_fb_id (META_DRM_BUFFER (buffer_import),
-                                      &fb_args,
-                                      error);
+  ret = meta_drm_buffer_do_ensure_fb_id (META_DRM_BUFFER (buffer_import),
+                                         &fb_args,
+                                         error);
 
   gbm_bo_destroy (imported_bo);
 
diff --git a/src/backends/native/meta-drm-buffer-private.h b/src/backends/native/meta-drm-buffer-private.h
index 08d8ae88e..9b41379f1 100644
--- a/src/backends/native/meta-drm-buffer-private.h
+++ b/src/backends/native/meta-drm-buffer-private.h
@@ -42,6 +42,9 @@ struct _MetaDrmBufferClass
   int (* export_fd) (MetaDrmBuffer  *buffer,
                      GError        **error);
 
+  gboolean (* ensure_fb_id) (MetaDrmBuffer  *buffer,
+                             GError        **error);
+
   int (* get_width) (MetaDrmBuffer *buffer);
   int (* get_height) (MetaDrmBuffer *buffer);
   int (* get_stride) (MetaDrmBuffer *buffer);
@@ -58,8 +61,8 @@ struct _MetaDrmBufferClass
 
 MetaDeviceFile * meta_drm_buffer_get_device_file (MetaDrmBuffer *buffer);
 
-gboolean meta_drm_buffer_ensure_fb_id (MetaDrmBuffer        *buffer,
-                                       const MetaDrmFbArgs  *fb_args,
-                                       GError              **error);
+gboolean meta_drm_buffer_do_ensure_fb_id (MetaDrmBuffer        *buffer,
+                                          const MetaDrmFbArgs  *fb_args,
+                                          GError              **error);
 
 #endif /* META_DRM_BUFFER_PRIVATE_H */
diff --git a/src/backends/native/meta-drm-buffer.c b/src/backends/native/meta-drm-buffer.c
index 09e07b61d..c526263f7 100644
--- a/src/backends/native/meta-drm-buffer.c
+++ b/src/backends/native/meta-drm-buffer.c
@@ -67,9 +67,21 @@ meta_drm_buffer_get_device_file (MetaDrmBuffer *buffer)
 }
 
 gboolean
-meta_drm_buffer_ensure_fb_id (MetaDrmBuffer        *buffer,
-                              const MetaDrmFbArgs  *fb_args,
-                              GError              **error)
+meta_drm_buffer_ensure_fb_id (MetaDrmBuffer  *buffer,
+                              GError        **error)
+{
+  MetaDrmBufferPrivate *priv = meta_drm_buffer_get_instance_private (buffer);
+
+  if (priv->fb_id)
+    return TRUE;
+
+  return META_DRM_BUFFER_GET_CLASS (buffer)->ensure_fb_id (buffer, error);
+}
+
+gboolean
+meta_drm_buffer_do_ensure_fb_id (MetaDrmBuffer        *buffer,
+                                 const MetaDrmFbArgs  *fb_args,
+                                 GError              **error)
 {
   MetaDrmBufferPrivate *priv = meta_drm_buffer_get_instance_private (buffer);
   int fd;
diff --git a/src/backends/native/meta-drm-buffer.h b/src/backends/native/meta-drm-buffer.h
index 2a8035d8d..441473aca 100644
--- a/src/backends/native/meta-drm-buffer.h
+++ b/src/backends/native/meta-drm-buffer.h
@@ -43,6 +43,9 @@ G_DECLARE_DERIVABLE_TYPE (MetaDrmBuffer,
 int meta_drm_buffer_export_fd (MetaDrmBuffer  *buffer,
                                GError        **error);
 
+gboolean meta_drm_buffer_ensure_fb_id (MetaDrmBuffer  *buffer,
+                                       GError        **error);
+
 uint32_t meta_drm_buffer_get_fb_id (MetaDrmBuffer *buffer);
 
 int meta_drm_buffer_get_width (MetaDrmBuffer *buffer);
diff --git a/src/backends/native/meta-kms-impl-device-atomic.c b/src/backends/native/meta-kms-impl-device-atomic.c
index e1adf20b3..7e14cff01 100644
--- a/src/backends/native/meta-kms-impl-device-atomic.c
+++ b/src/backends/native/meta-kms-impl-device-atomic.c
@@ -421,6 +421,9 @@ process_plane_assignment (MetaKmsImplDevice  *impl_device,
 
   buffer = plane_assignment->buffer;
 
+  if (buffer && !meta_drm_buffer_ensure_fb_id (buffer, error))
+    return FALSE;
+
   meta_topic (META_DEBUG_KMS,
               "[atomic] Assigning %s plane (%u, %s) to %u, "
               "%hdx%hd+%hd+%hd -> %dx%d+%d+%d",
diff --git a/src/backends/native/meta-kms-impl-device-simple.c b/src/backends/native/meta-kms-impl-device-simple.c
index 28d512720..4391ebaec 100644
--- a/src/backends/native/meta-kms-impl-device-simple.c
+++ b/src/backends/native/meta-kms-impl-device-simple.c
@@ -396,6 +396,9 @@ process_mode_set (MetaKmsImplDevice  *impl_device,
         }
 
       buffer = plane_assignment->buffer;
+      if (!meta_drm_buffer_ensure_fb_id (buffer, error))
+        return FALSE;
+
       fb_id = meta_drm_buffer_get_fb_id (buffer);
 
       for (l = mode_set->connectors; l; l = l->next)
@@ -822,6 +825,9 @@ mode_set_fallback (MetaKmsImplDeviceSimple  *impl_device_simple,
       return FALSE;
     }
 
+  if (!meta_drm_buffer_ensure_fb_id (plane_assignment->buffer, error))
+    return FALSE;
+
   fill_connector_ids_array (cached_mode_set->connectors,
                             &connectors,
                             &n_connectors);
@@ -926,6 +932,10 @@ dispatch_page_flip (MetaKmsImplDevice    *impl_device,
       return TRUE;
     }
 
+  if (plane_assignment && plane_assignment->buffer &&
+      !meta_drm_buffer_ensure_fb_id (plane_assignment->buffer, error))
+    return FALSE;
+
   fd = meta_kms_impl_device_get_fd (impl_device);
   if (custom_page_flip)
     {