diff --git a/src/core/window.c b/src/core/window.c
index 76a6c1a9b..3385be8fd 100644
--- a/src/core/window.c
+++ b/src/core/window.c
@@ -6300,21 +6300,13 @@ meta_window_handle_mouse_grab_op_event  (MetaWindow         *window,
 
     case CLUTTER_BUTTON_PRESS:
       {
-        ClutterModifierType grab_mods = meta_display_get_compositor_modifiers (window->display);
-
         /* This is the keybinding or menu case where we've
          * been dragging around the window without the button
-         * pressed. */
-
-        if ((meta_grab_op_is_mouse (window->display->grab_op) &&
-             (event->button.modifier_state & grab_mods) == grab_mods &&
-             window->display->grab_button != (int) event->button.button) ||
-            meta_grab_op_is_keyboard (window->display->grab_op))
-          {
-            end_grab_op (window, event);
-            return FALSE;
-          }
-        return TRUE;
+         * pressed, or the case of pressing extra mouse buttons
+         * while a grab op is ongoing.
+         */
+        end_grab_op (window, event);
+        return FALSE;
       }
 
     case CLUTTER_TOUCH_END: