diff --git a/js/ui/main.js b/js/ui/main.js
index 4ba3faa8c..23e36aba5 100644
--- a/js/ui/main.js
+++ b/js/ui/main.js
@@ -755,7 +755,8 @@ export function popModal(grab, timestamp) {
         if (record.prevFocus)
             record.prevFocus.disconnect(record.prevFocusDestroyId);
         actionMode = record.actionMode;
-        global.stage.set_key_focus(record.prevFocus);
+        if (global.stage.key_focus === record.actor)
+            global.stage.set_key_focus(record.prevFocus);
     } else {
         // If we have:
         //     global.stage.set_focus(a);