diff --git a/src/x11/events.c b/src/x11/events.c
index f60f650a6..291b7114a 100644
--- a/src/x11/events.c
+++ b/src/x11/events.c
@@ -830,6 +830,16 @@ crossing_serial_is_ignored (MetaDisplay  *display,
   return FALSE;
 }
 
+static gboolean
+event_has_button_mask (XIEnterEvent *enter_event)
+{
+  int i;
+  for (i = 0; i < enter_event->buttons.mask_len; i++)
+    if (enter_event->buttons.mask[i] != '\0')
+      return TRUE;
+  return FALSE;
+}
+
 static gboolean
 handle_input_xevent (MetaDisplay  *display,
                      XIEvent      *input_event,
@@ -871,6 +881,7 @@ handle_input_xevent (MetaDisplay  *display,
        * avoid races.
        */
       if (window && !crossing_serial_is_ignored (display, serial) &&
+          !event_has_button_mask (enter_event) &&
           enter_event->mode != XINotifyGrab &&
           enter_event->mode != XINotifyUngrab &&
           enter_event->detail != XINotifyInferior &&