diff --git a/src/compositor/compositor.c b/src/compositor/compositor.c
index 52c0db8c8..4c6f0e21d 100644
--- a/src/compositor/compositor.c
+++ b/src/compositor/compositor.c
@@ -991,6 +991,9 @@ event_is_passive_button_grab (MetaDisplay   *display,
   if (device_event->evtype != XI_ButtonPress)
     return FALSE;
 
+  if (display->window_grab_modifiers == 0)
+    return FALSE;
+
   if ((device_event->mods.effective & display->window_grab_modifiers) !=
       display->window_grab_modifiers)
     return FALSE;
diff --git a/src/core/display.c b/src/core/display.c
index 7f59a8413..acd26c68c 100644
--- a/src/core/display.c
+++ b/src/core/display.c
@@ -2108,7 +2108,7 @@ meta_display_handle_event (MetaDisplay        *display,
            * care about. Just let the event through.
            */
           unmodified = (event->button.modifier_state & grab_mask) == 0;
-          fully_modified = (event->button.modifier_state & grab_mask) == grab_mask;
+          fully_modified = grab_mask && (event->button.modifier_state & grab_mask) == grab_mask;
 
           if (unmodified && window && window->have_focus_click_grab)
             {