diff --git a/src/wayland/meta-wayland-pointer-constraints.c b/src/wayland/meta-wayland-pointer-constraints.c
index 8442dc4e2..2c288ea39 100644
--- a/src/wayland/meta-wayland-pointer-constraints.c
+++ b/src/wayland/meta-wayland-pointer-constraints.c
@@ -44,6 +44,7 @@
 #include "pointer-constraints-unstable-v1-server-protocol.h"
 
 static GQuark quark_pending_constraint_state = 0;
+static GQuark quark_surface_pointer_constraints_data = 0;
 
 struct _MetaWaylandPointerConstraint
 {
@@ -64,6 +65,11 @@ struct _MetaWaylandPointerConstraint
   MetaPointerConstraint *constraint;
 };
 
+typedef struct _MetaWaylandSurfacePointerConstraintsData
+{
+  GList *pointer_constraints;
+} MetaWaylandSurfacePointerConstraintsData;
+
 typedef struct
 {
   MetaWaylandPointerConstraint *constraint;
@@ -84,6 +90,71 @@ static const struct zwp_confined_pointer_v1_interface confined_pointer_interface
 static const MetaWaylandPointerGrabInterface locked_pointer_grab_interface;
 static const MetaWaylandPointerGrabInterface confined_pointer_grab_interface;
 
+static void
+meta_wayland_pointer_constraint_destroy (MetaWaylandPointerConstraint *constraint);
+
+static MetaWaylandSurfacePointerConstraintsData *
+get_surface_constraints_data (MetaWaylandSurface *surface)
+{
+  return g_object_get_qdata (G_OBJECT (surface),
+                             quark_surface_pointer_constraints_data);
+}
+
+static void
+surface_constraint_data_free (MetaWaylandSurfacePointerConstraintsData *data)
+{
+  g_list_free_full (data->pointer_constraints,
+                    (GDestroyNotify) meta_wayland_pointer_constraint_destroy);
+  g_free (data);
+}
+
+static MetaWaylandSurfacePointerConstraintsData *
+ensure_surface_constraints_data (MetaWaylandSurface *surface)
+{
+  MetaWaylandSurfacePointerConstraintsData *data;
+
+  data = get_surface_constraints_data (surface);
+  if (!data)
+    {
+      data = g_new0 (MetaWaylandSurfacePointerConstraintsData, 1);
+      g_object_set_qdata_full (G_OBJECT (surface),
+                               quark_surface_pointer_constraints_data,
+                               data,
+                               (GDestroyNotify) surface_constraint_data_free);
+    }
+
+  return data;
+}
+
+static void
+surface_add_pointer_constraint (MetaWaylandSurface           *surface,
+                                MetaWaylandPointerConstraint *constraint)
+{
+  MetaWaylandSurfacePointerConstraintsData *data;
+
+  data = ensure_surface_constraints_data (surface);
+  data->pointer_constraints = g_list_append (data->pointer_constraints,
+                                             constraint);
+}
+
+static void
+surface_remove_pointer_constraints (MetaWaylandSurface           *surface,
+                                    MetaWaylandPointerConstraint *constraint)
+{
+  MetaWaylandSurfacePointerConstraintsData *data;
+
+  data = get_surface_constraints_data (surface);
+  data->pointer_constraints =
+    g_list_remove (data->pointer_constraints, constraint);
+
+  if (!data->pointer_constraints)
+    {
+      g_object_set_qdata (G_OBJECT (surface),
+                          quark_surface_pointer_constraints_data,
+                          NULL);
+    }
+}
+
 static MetaWaylandPointerConstraint *
 meta_wayland_pointer_constraint_new (MetaWaylandSurface                      *surface,
                                      MetaWaylandSeat                         *seat,
@@ -260,7 +331,7 @@ meta_wayland_pointer_constraint_remove (MetaWaylandPointerConstraint *constraint
 {
   MetaWaylandSurface *surface = constraint->surface;
 
-  meta_wayland_surface_remove_pointer_constraint (surface, constraint);
+  surface_remove_pointer_constraints (surface, constraint);
   meta_wayland_pointer_constraint_destroy (constraint);
 }
 
@@ -298,11 +369,16 @@ void
 meta_wayland_pointer_constraint_maybe_enable_for_window (MetaWindow *window)
 {
   MetaWaylandSurface *surface = window->surface;
-  GList *it;
+  MetaWaylandSurfacePointerConstraintsData *surface_data;
+  GList *l;
 
-  for (it = surface->pointer_constraints; it; it = it->next)
+  surface_data = get_surface_constraints_data (surface);
+  if (!surface_data)
+    return;
+
+  for (l = surface_data->pointer_constraints; l; l = l->next)
     {
-      MetaWaylandPointerConstraint *constraint = it->data;
+      MetaWaylandPointerConstraint *constraint = l->data;
 
       meta_wayland_pointer_constraint_maybe_enable (constraint);
     }
@@ -500,6 +576,28 @@ meta_wayland_pointer_constraint_set_pending_region (MetaWaylandPointerConstraint
     }
 }
 
+static MetaWaylandPointerConstraint *
+get_pointer_constraint_for_seat (MetaWaylandSurface *surface,
+                                 MetaWaylandSeat    *seat)
+{
+  MetaWaylandSurfacePointerConstraintsData *surface_data;
+  GList *l;
+
+  surface_data = get_surface_constraints_data (surface);
+  if (!surface_data)
+    return NULL;
+
+  for (l = surface_data->pointer_constraints; l; l = l->next)
+    {
+      MetaWaylandPointerConstraint *constraint = l->data;
+
+      if (seat == constraint->seat)
+        return constraint;
+    }
+
+  return NULL;
+}
+
 static void
 init_pointer_constraint (struct wl_resource                      *resource,
                          uint32_t                                 id,
@@ -515,7 +613,7 @@ init_pointer_constraint (struct wl_resource                      *resource,
   struct wl_resource *cr;
   MetaWaylandPointerConstraint *constraint;
 
-  if (meta_wayland_surface_get_pointer_constraint_for_seat (surface, seat))
+  if (get_pointer_constraint_for_seat (surface, seat))
     {
       wl_resource_post_error (resource,
                               WL_DISPLAY_ERROR_INVALID_OBJECT,
@@ -543,7 +641,7 @@ init_pointer_constraint (struct wl_resource                      *resource,
       return;
     }
 
-  meta_wayland_surface_add_pointer_constraint (surface, constraint);
+  surface_add_pointer_constraint (surface, constraint);
 
   wl_resource_set_implementation (cr, implementation, constraint,
                                   pointer_constraint_resource_destroyed);
@@ -805,4 +903,6 @@ meta_wayland_pointer_constraint_class_init (MetaWaylandPointerConstraintClass *k
 {
   quark_pending_constraint_state =
     g_quark_from_static_string ("-meta-wayland-pointer-constraint-pending_state");
+  quark_surface_pointer_constraints_data =
+    g_quark_from_static_string ("-meta-wayland-surface-constraints-data");
 }
diff --git a/src/wayland/meta-wayland-pointer-constraints.h b/src/wayland/meta-wayland-pointer-constraints.h
index 4ed56f8bd..2a6609135 100644
--- a/src/wayland/meta-wayland-pointer-constraints.h
+++ b/src/wayland/meta-wayland-pointer-constraints.h
@@ -38,8 +38,6 @@ G_DECLARE_FINAL_TYPE (MetaWaylandPointerConstraint,
 
 void meta_wayland_pointer_constraints_init (MetaWaylandCompositor *compositor);
 
-void meta_wayland_pointer_constraint_destroy (MetaWaylandPointerConstraint *constraint);
-
 MetaWaylandSeat * meta_wayland_pointer_constraint_get_seat (MetaWaylandPointerConstraint *constraint);
 
 cairo_region_t * meta_wayland_pointer_constraint_calculate_effective_region (MetaWaylandPointerConstraint *constraint);
diff --git a/src/wayland/meta-wayland-surface.c b/src/wayland/meta-wayland-surface.c
index 78c4735a1..43c3483b1 100644
--- a/src/wayland/meta-wayland-surface.c
+++ b/src/wayland/meta-wayland-surface.c
@@ -1109,9 +1109,6 @@ wl_surface_destructor (struct wl_resource *resource)
   if (surface->window)
     destroy_window (surface);
 
-  g_list_free_full (surface->pointer_constraints,
-                    (GDestroyNotify) meta_wayland_pointer_constraint_destroy);
-
   surface_set_buffer (surface, NULL);
   g_clear_object (&surface->pending);
 
@@ -2520,39 +2517,6 @@ meta_wayland_surface_get_toplevel_window (MetaWaylandSurface *surface)
   return NULL;
 }
 
-void
-meta_wayland_surface_add_pointer_constraint (MetaWaylandSurface           *surface,
-                                             MetaWaylandPointerConstraint *constraint)
-{
-  surface->pointer_constraints = g_list_append (surface->pointer_constraints,
-                                                constraint);
-}
-
-void
-meta_wayland_surface_remove_pointer_constraint (MetaWaylandSurface           *surface,
-                                                MetaWaylandPointerConstraint *constraint)
-{
-  surface->pointer_constraints = g_list_remove (surface->pointer_constraints,
-                                                constraint);
-}
-
-MetaWaylandPointerConstraint *
-meta_wayland_surface_get_pointer_constraint_for_seat (MetaWaylandSurface *surface,
-                                                      MetaWaylandSeat    *seat)
-{
-  GList *iter;
-
-  for (iter = surface->pointer_constraints; iter; iter = iter->next)
-    {
-      MetaWaylandPointerConstraint *constraint = iter->data;
-
-      if (seat == meta_wayland_pointer_constraint_get_seat (constraint))
-        return constraint;
-    }
-
-  return NULL;
-}
-
 void
 meta_wayland_surface_get_relative_coordinates (MetaWaylandSurface *surface,
                                                float               abs_x,
diff --git a/src/wayland/meta-wayland-surface.h b/src/wayland/meta-wayland-surface.h
index bc1027646..ff3835f9e 100644
--- a/src/wayland/meta-wayland-surface.h
+++ b/src/wayland/meta-wayland-surface.h
@@ -218,8 +218,6 @@ struct _MetaWaylandSurface
     gboolean pending_pos;
     GSList *pending_placement_ops;
   } sub;
-
-  GList *pointer_constraints;
 };
 
 void                meta_wayland_shell_init     (MetaWaylandCompositor *compositor);
@@ -264,15 +262,6 @@ void                meta_wayland_surface_queue_pending_frame_callbacks (MetaWayl
 void                meta_wayland_surface_queue_pending_state_frame_callbacks (MetaWaylandSurface      *surface,
                                                                               MetaWaylandPendingState *pending);
 
-void                meta_wayland_surface_add_pointer_constraint (MetaWaylandSurface           *surface,
-                                                                 MetaWaylandPointerConstraint *constraint);
-
-void                meta_wayland_surface_remove_pointer_constraint (MetaWaylandSurface           *surface,
-                                                                    MetaWaylandPointerConstraint *constraint);
-MetaWaylandPointerConstraint *
-                    meta_wayland_surface_get_pointer_constraint_for_seat (MetaWaylandSurface *surface,
-                                                                          MetaWaylandSeat    *seat);
-
 void                meta_wayland_surface_get_relative_coordinates (MetaWaylandSurface *surface,
                                                                    float               abs_x,
                                                                    float               abs_y,