diff --git a/src/core/display.c b/src/core/display.c
index 9a0d80998..c7f1a349a 100644
--- a/src/core/display.c
+++ b/src/core/display.c
@@ -2411,6 +2411,7 @@ meta_display_handle_event (MetaDisplay *display,
 
           if ((window &&
                meta_grab_op_is_mouse (display->grab_op) &&
+               (device_event->mods.effective & display->window_grab_modifiers) &&
                display->grab_button != device_event->detail &&
                display->grab_window == window) ||
               grab_op_is_keyboard (display->grab_op))