diff --git a/src/core/display.c b/src/core/display.c
index 8f1fe3cc0..676cd566f 100644
--- a/src/core/display.c
+++ b/src/core/display.c
@@ -1799,6 +1799,8 @@ event_callback (XEvent   *event,
       if (display->grab_op == META_GRAB_OP_COMPOSITOR)
         break;
 
+      display->overlay_key_only_pressed = FALSE;
+
       if (event->xbutton.button == 4 || event->xbutton.button == 5)
         /* Scrollwheel event, do nothing and deliver event to compositor below */
         break;
@@ -1996,6 +1998,8 @@ event_callback (XEvent   *event,
       if (display->grab_op == META_GRAB_OP_COMPOSITOR)
         break;
 
+      display->overlay_key_only_pressed = FALSE;
+
       if (display->grab_window == window &&
           grab_op_is_mouse (display->grab_op))
         meta_window_handle_mouse_grab_op_event (window, event);