diff --git a/src/wayland/meta-wayland-pointer-constraints.c b/src/wayland/meta-wayland-pointer-constraints.c
index 6d99c73f2..5cf74fcd4 100644
--- a/src/wayland/meta-wayland-pointer-constraints.c
+++ b/src/wayland/meta-wayland-pointer-constraints.c
@@ -279,6 +279,15 @@ static void
 pointer_focus_surface_changed (MetaWaylandPointer           *pointer,
                                MetaWaylandPointerConstraint *constraint)
 {
+  MetaWindow *window = constraint->surface->window;
+
+  if (window)
+    {
+      MetaWaylandSeat *seat = meta_wayland_pointer_get_seat (pointer);
+
+      meta_wayland_pointer_constraint_maybe_remove_for_seat (seat, window);
+    }
+
   meta_wayland_pointer_constraint_maybe_enable (constraint);
 }