diff --git a/src/core/events.c b/src/core/events.c
index e456be2e6..6fd141f3e 100644
--- a/src/core/events.c
+++ b/src/core/events.c
@@ -251,26 +251,33 @@ meta_display_handle_event (MetaDisplay        *display,
           bypass_clutter = TRUE;
         }
 
-      /* Under X11, we have a Sync grab and in order to send it back to
-       * clients, we have to explicitly replay it.
-       *
-       * Under Wayland, we retrieve all events and we have to make sure
-       * to filter them out from Wayland clients.
-       */
-      if (meta_window_handle_ungrabbed_event (window, event))
+      meta_window_handle_ungrabbed_event (window, event);
+
+      /* This might start a grab op. If it does, then filter out the
+       * event, and if it doesn't, replay the event to release our
+       * own sync grab. */
+
+      if (display->grab_window == window &&
+          meta_grab_op_is_moving_or_resizing (display->grab_op))
         {
+          bypass_clutter = TRUE;
           bypass_wayland = TRUE;
         }
-      else if (!clutter_event_get_event_sequence (event))
+      else
         {
-          MetaBackend *backend = meta_get_backend ();
-          if (META_IS_BACKEND_X11 (backend))
+          /* Only replay button press events, since that's where we
+           * have the synchronous grab. */
+          if (event->type == CLUTTER_BUTTON_PRESS)
             {
-              Display *xdisplay = meta_backend_x11_get_xdisplay (META_BACKEND_X11 (backend));
-              meta_verbose ("Allowing events time %u\n",
-                            (unsigned int)event->button.time);
-              XIAllowEvents (xdisplay, clutter_event_get_device_id (event),
-                             XIReplayDevice, event->button.time);
+              MetaBackend *backend = meta_get_backend ();
+              if (META_IS_BACKEND_X11 (backend))
+                {
+                  Display *xdisplay = meta_backend_x11_get_xdisplay (META_BACKEND_X11 (backend));
+                  meta_verbose ("Allowing events time %u\n",
+                                (unsigned int)event->button.time);
+                  XIAllowEvents (xdisplay, clutter_event_get_device_id (event),
+                                 XIReplayDevice, event->button.time);
+                }
             }
         }
 
diff --git a/src/core/window-private.h b/src/core/window-private.h
index ee014aca2..2e998f731 100644
--- a/src/core/window-private.h
+++ b/src/core/window-private.h
@@ -662,8 +662,8 @@ void meta_window_handle_enter (MetaWindow  *window,
                                guint        root_y);
 void meta_window_handle_leave (MetaWindow  *window);
 
-gboolean meta_window_handle_ungrabbed_event (MetaWindow         *window,
-                                             const ClutterEvent *event);
+void meta_window_handle_ungrabbed_event (MetaWindow         *window,
+                                         const ClutterEvent *event);
 
 void meta_window_get_client_area_rect (const MetaWindow      *window,
                                        cairo_rectangle_int_t *rect);
diff --git a/src/core/window.c b/src/core/window.c
index 43cc71e90..2fd48d2ee 100644
--- a/src/core/window.c
+++ b/src/core/window.c
@@ -7871,7 +7871,7 @@ meta_window_handle_leave (MetaWindow *window)
     meta_window_lower (window);
 }
 
-gboolean
+void
 meta_window_handle_ungrabbed_event (MetaWindow         *window,
                                     const ClutterEvent *event)
 {
@@ -7880,17 +7880,17 @@ meta_window_handle_ungrabbed_event (MetaWindow         *window,
   gboolean is_window_grab;
 
   if (event->type != CLUTTER_BUTTON_PRESS)
-    return FALSE;
+    return;
 
   if (display->grab_op != META_GRAB_OP_NONE)
-    return FALSE;
+    return;
 
   /* Some windows might not ask for input, in which case we might be here
    * because we selected for ButtonPress on the root window. In that case,
    * we have to take special care not to act for an override-redirect window.
    */
   if (window->override_redirect)
-    return FALSE;
+    return;
 
   /* We have three passive button grabs:
    * - on any button, without modifiers => focuses and maybe raises the window
@@ -7941,7 +7941,6 @@ meta_window_handle_ungrabbed_event (MetaWindow         *window,
 
       meta_verbose ("Allowing events time %u\n",
                     (unsigned int)event->button.time);
-      return FALSE;
     }
   else if (is_window_grab && (int) event->button.button == meta_prefs_get_mouse_button_resize ())
     {
@@ -7991,7 +7990,6 @@ meta_window_handle_ungrabbed_event (MetaWindow         *window,
                                         event->button.x,
                                         event->button.y);
         }
-      return TRUE;
     }
   else if (is_window_grab && (int) event->button.button == meta_prefs_get_mouse_button_menu ())
     {
@@ -8001,7 +7999,6 @@ meta_window_handle_ungrabbed_event (MetaWindow         *window,
                              META_WINDOW_MENU_WM,
                              event->button.x,
                              event->button.y);
-      return TRUE;
     }
   else if (is_window_grab && (int) event->button.button == 1)
     {
@@ -8019,10 +8016,7 @@ meta_window_handle_ungrabbed_event (MetaWindow         *window,
                                       event->button.x,
                                       event->button.y);
         }
-      return TRUE;
     }
-
-  return FALSE;
 }
 
 void