diff --git a/src/backends/meta-monitor-config-manager.c b/src/backends/meta-monitor-config-manager.c
index d84f7c509..f1404c02b 100644
--- a/src/backends/meta-monitor-config-manager.c
+++ b/src/backends/meta-monitor-config-manager.c
@@ -28,6 +28,8 @@
 #include "backends/meta-monitor-manager-private.h"
 #include "core/boxes-private.h"
 
+#define CONFIG_HISTORY_MAX_SIZE 3
+
 struct _MetaMonitorConfigManager
 {
   GObject parent;
@@ -37,7 +39,7 @@ struct _MetaMonitorConfigManager
   MetaMonitorConfigStore *config_store;
 
   MetaMonitorsConfig *current_config;
-  MetaMonitorsConfig *previous_config;
+  GQueue config_history;
 };
 
 G_DEFINE_TYPE (MetaMonitorConfigManager, meta_monitor_config_manager,
@@ -988,8 +990,15 @@ void
 meta_monitor_config_manager_set_current (MetaMonitorConfigManager *config_manager,
                                          MetaMonitorsConfig       *config)
 {
-  g_set_object (&config_manager->previous_config,
-                config_manager->current_config);
+  if (config_manager->current_config)
+    {
+      g_queue_push_head (&config_manager->config_history,
+                         g_object_ref (config_manager->current_config));
+      if (g_queue_get_length (&config_manager->config_history) >
+          CONFIG_HISTORY_MAX_SIZE)
+        g_object_unref (g_queue_pop_tail (&config_manager->config_history));
+    }
+
   g_set_object (&config_manager->current_config, config);
 }
 
@@ -1008,10 +1017,23 @@ meta_monitor_config_manager_get_current (MetaMonitorConfigManager *config_manage
   return config_manager->current_config;
 }
 
+MetaMonitorsConfig *
+meta_monitor_config_manager_pop_previous (MetaMonitorConfigManager *config_manager)
+{
+  return g_queue_pop_head (&config_manager->config_history);
+}
+
 MetaMonitorsConfig *
 meta_monitor_config_manager_get_previous (MetaMonitorConfigManager *config_manager)
 {
-  return config_manager->previous_config;
+  return g_queue_peek_head (&config_manager->config_history);
+}
+
+void
+meta_monitor_config_manager_clear_history (MetaMonitorConfigManager *config_manager)
+{
+  g_queue_foreach (&config_manager->config_history, (GFunc) g_object_unref, NULL);
+  g_queue_clear (&config_manager->config_history);
 }
 
 static void
@@ -1021,7 +1043,7 @@ meta_monitor_config_manager_dispose (GObject *object)
     META_MONITOR_CONFIG_MANAGER (object);
 
   g_clear_object (&config_manager->current_config);
-  g_clear_object (&config_manager->previous_config);
+  meta_monitor_config_manager_clear_history (config_manager);
 
   G_OBJECT_CLASS (meta_monitor_config_manager_parent_class)->dispose (object);
 }
@@ -1029,6 +1051,7 @@ meta_monitor_config_manager_dispose (GObject *object)
 static void
 meta_monitor_config_manager_init (MetaMonitorConfigManager *config_manager)
 {
+  g_queue_init (&config_manager->config_history);
 }
 
 static void
diff --git a/src/backends/meta-monitor-config-manager.h b/src/backends/meta-monitor-config-manager.h
index aa2646ebd..04501eebd 100644
--- a/src/backends/meta-monitor-config-manager.h
+++ b/src/backends/meta-monitor-config-manager.h
@@ -104,8 +104,12 @@ void meta_monitor_config_manager_set_current (MetaMonitorConfigManager *config_m
 
 MetaMonitorsConfig * meta_monitor_config_manager_get_current (MetaMonitorConfigManager *config_manager);
 
+MetaMonitorsConfig * meta_monitor_config_manager_pop_previous (MetaMonitorConfigManager *config_manager);
+
 MetaMonitorsConfig * meta_monitor_config_manager_get_previous (MetaMonitorConfigManager *config_manager);
 
+void meta_monitor_config_manager_clear_history (MetaMonitorConfigManager *config_manager);
+
 void meta_monitor_config_manager_save_current (MetaMonitorConfigManager *config_manager);
 
 MetaMonitorsConfig * meta_monitors_config_new (GList                       *logical_monitor_configs,
diff --git a/src/backends/meta-monitor-manager.c b/src/backends/meta-monitor-manager.c
index 4258f71fa..b13926299 100644
--- a/src/backends/meta-monitor-manager.c
+++ b/src/backends/meta-monitor-manager.c
@@ -1111,14 +1111,13 @@ restore_previous_config (MetaMonitorManager *manager)
   GError *error = NULL;
 
   previous_config =
-    meta_monitor_config_manager_get_previous (manager->config_manager);
+    meta_monitor_config_manager_pop_previous (manager->config_manager);
 
   if (previous_config)
     {
       MetaMonitorsConfigMethod method;
 
       method = META_MONITORS_CONFIG_METHOD_TEMPORARY;
-      g_object_ref (previous_config);
       if (meta_monitor_manager_apply_monitors_config (manager,
                                                       previous_config,
                                                       method,