diff --git a/src/backends/native/meta-onscreen-native.c b/src/backends/native/meta-onscreen-native.c
index 763eb559e..cc2a8ca76 100644
--- a/src/backends/native/meta-onscreen-native.c
+++ b/src/backends/native/meta-onscreen-native.c
@@ -2219,3 +2219,9 @@ meta_onscreen_native_class_init (MetaOnscreenNativeClass *klass)
     meta_onscreen_native_swap_buffers_with_damage;
   onscreen_class->direct_scanout = meta_onscreen_native_direct_scanout;
 }
+
+MetaCrtc *
+meta_onscreen_native_get_crtc (MetaOnscreenNative *onscreen_native)
+{
+  return onscreen_native->crtc;
+}
diff --git a/src/backends/native/meta-onscreen-native.h b/src/backends/native/meta-onscreen-native.h
index 020fc8a14..5c99181d3 100644
--- a/src/backends/native/meta-onscreen-native.h
+++ b/src/backends/native/meta-onscreen-native.h
@@ -56,4 +56,6 @@ MetaOnscreenNative * meta_onscreen_native_new (MetaRendererNative *renderer_nati
                                                int                 width,
                                                int                 height);
 
+MetaCrtc * meta_onscreen_native_get_crtc (MetaOnscreenNative *onscreen_native);
+
 #endif /* META_ONSCREEN_NATIVE_H */
diff --git a/src/backends/native/meta-renderer-native.c b/src/backends/native/meta-renderer-native.c
index 440b33c0d..b38fceecf 100644
--- a/src/backends/native/meta-renderer-native.c
+++ b/src/backends/native/meta-renderer-native.c
@@ -93,6 +93,8 @@ struct _MetaRendererNative
   gboolean pending_mode_set;
 
   GList *kept_alive_onscreens;
+  GList *lingering_onscreens;
+  guint release_unused_gpus_idle_id;
 
   GList *power_save_page_flip_onscreens;
   guint power_save_page_flip_source_id;
@@ -688,13 +690,6 @@ meta_renderer_native_queue_power_save_page_flip (MetaRendererNative *renderer_na
                     g_object_ref (onscreen));
 }
 
-static void
-clear_kept_alive_onscreens (MetaRendererNative *renderer_native)
-{
-  g_clear_list (&renderer_native->kept_alive_onscreens,
-                g_object_unref);
-}
-
 static gboolean
 is_gpu_unused (gpointer key,
                gpointer value,
@@ -728,11 +723,70 @@ free_unused_gpu_datas (MetaRendererNative *renderer_native)
       g_hash_table_add (used_gpus, gpu);
     }
 
+  for (l = renderer_native->lingering_onscreens; l; l = l->next)
+    {
+      MetaOnscreenNative *onscreen_native = META_ONSCREEN_NATIVE (l->data);
+      MetaCrtc *crtc = meta_onscreen_native_get_crtc (onscreen_native);
+
+      g_hash_table_add (used_gpus, meta_crtc_get_gpu (crtc));
+    }
+
   g_hash_table_foreach_remove (renderer_native->gpu_datas,
                                is_gpu_unused,
                                used_gpus);
 }
 
+static gboolean
+release_unused_gpus_idle (gpointer user_data)
+{
+  MetaRendererNative *renderer_native = META_RENDERER_NATIVE (user_data);
+
+  renderer_native->release_unused_gpus_idle_id = 0;
+  free_unused_gpu_datas (renderer_native);
+
+  return G_SOURCE_REMOVE;
+}
+
+static void
+old_onscreen_freed (gpointer  user_data,
+                    GObject  *freed_onscreen)
+{
+  MetaRendererNative *renderer_native = META_RENDERER_NATIVE (user_data);
+
+  renderer_native->lingering_onscreens =
+    g_list_remove (renderer_native->lingering_onscreens, freed_onscreen);
+
+  if (!renderer_native->release_unused_gpus_idle_id)
+    {
+      renderer_native->release_unused_gpus_idle_id =
+        g_idle_add (release_unused_gpus_idle, renderer_native);
+    }
+}
+
+static void
+clear_kept_alive_onscreens (MetaRendererNative *renderer_native)
+{
+  GList *l;
+
+  for (l = renderer_native->kept_alive_onscreens; l; l = l->next)
+    {
+      CoglOnscreen *onscreen;
+
+      if (!COGL_IS_ONSCREEN (l->data))
+        continue;
+
+      onscreen = COGL_ONSCREEN (l->data);
+      g_object_weak_ref (G_OBJECT (onscreen),
+                         old_onscreen_freed,
+                         renderer_native);
+      renderer_native->lingering_onscreens =
+        g_list_prepend (renderer_native->lingering_onscreens, onscreen);
+    }
+
+  g_clear_list (&renderer_native->kept_alive_onscreens,
+                g_object_unref);
+}
+
 void
 meta_renderer_native_post_mode_set_updates (MetaRendererNative *renderer_native)
 {
@@ -2061,8 +2115,6 @@ meta_renderer_native_finalize (GObject *object)
 {
   MetaRendererNative *renderer_native = META_RENDERER_NATIVE (object);
 
-  clear_kept_alive_onscreens (renderer_native);
-
   g_clear_list (&renderer_native->power_save_page_flip_onscreens,
                 g_object_unref);
   g_clear_handle_id (&renderer_native->power_save_page_flip_source_id,
@@ -2070,6 +2122,10 @@ meta_renderer_native_finalize (GObject *object)
 
   g_list_free (renderer_native->pending_mode_set_views);
 
+  g_clear_handle_id (&renderer_native->release_unused_gpus_idle_id,
+                     g_source_remove);
+  clear_kept_alive_onscreens (renderer_native);
+
   g_hash_table_destroy (renderer_native->gpu_datas);
   g_clear_object (&renderer_native->gles3);