diff --git a/src/backends/meta-monitor-config-manager.c b/src/backends/meta-monitor-config-manager.c
index 601d4c36c..4da1e8357 100644
--- a/src/backends/meta-monitor-config-manager.c
+++ b/src/backends/meta-monitor-config-manager.c
@@ -180,7 +180,8 @@ assign_monitor_crtc (MetaMonitor         *monitor,
    * as such.
    */
   first_monitor_config = data->logical_monitor_config->monitor_configs->data;
-  if (data->monitor_config == first_monitor_config &&
+  if (data->logical_monitor_config->is_primary &&
+      data->monitor_config == first_monitor_config &&
       meta_monitor_get_main_output (monitor) == output)
     assign_output_as_primary = TRUE;
   else