diff --git a/src/core/window-private.h b/src/core/window-private.h
index 267f163e8..c0f71212a 100644
--- a/src/core/window-private.h
+++ b/src/core/window-private.h
@@ -421,6 +421,15 @@ struct _MetaWindow
    */
   MetaRectangle unconstrained_rect;
 
+  /* The rectangle of the "server-side" geometry of the buffer,
+   * in root coordinates.
+   *
+   * For X11 windows, this matches XGetGeometry of the toplevel.
+   *
+   * For Wayland windows, this matches the buffer size and where
+   * the surface actor is positioned. */
+  MetaRectangle buffer_rect;
+
   /* Cached net_wm_icon_geometry */
   MetaRectangle icon_geometry;
 
diff --git a/src/core/window.c b/src/core/window.c
index e1598b612..59efbba6d 100644
--- a/src/core/window.c
+++ b/src/core/window.c
@@ -4052,24 +4052,7 @@ void
 meta_window_get_buffer_rect (const MetaWindow *window,
                              MetaRectangle    *rect)
 {
-  if (window->frame)
-    *rect = window->frame->rect;
-  else if (window->xwindow != None)
-    {
-      XWindowAttributes xwa;
-
-      XGetWindowAttributes (window->display->xdisplay, window->xwindow, &xwa);
-
-      rect->x = xwa.x;
-      rect->y = xwa.y;
-      rect->width = xwa.width;
-      rect->height = xwa.height;
-    }
-  else
-    {
-      *rect = window->rect;
-      meta_window_frame_rect_to_client_rect ((MetaWindow *) window, rect, rect);
-    }
+  *rect = window->buffer_rect;
 }
 
 /**
diff --git a/src/wayland/window-wayland.c b/src/wayland/window-wayland.c
index 7ee6f07d3..e78a35682 100644
--- a/src/wayland/window-wayland.c
+++ b/src/wayland/window-wayland.c
@@ -361,6 +361,8 @@ meta_window_wayland_move_resize (MetaWindow *window,
   rect.width = width;
   rect.height = height;
 
+  window->buffer_rect = rect;
+
   if (rect.width != window->rect.width || rect.height != window->rect.height)
     flags |= META_IS_RESIZE_ACTION;
 
diff --git a/src/x11/window-x11.c b/src/x11/window-x11.c
index ca91dab24..5422b542a 100644
--- a/src/x11/window-x11.c
+++ b/src/x11/window-x11.c
@@ -550,6 +550,7 @@ meta_window_x11_manage (MetaWindow *window)
       MetaWindowX11Private *priv = meta_window_x11_get_instance_private (window_x11);
 
       priv->client_rect = window->rect;
+      window->buffer_rect = window->rect;
     }
   else
     {
@@ -1257,6 +1258,11 @@ meta_window_x11_move_resize_internal (MetaWindow                *window,
                                                      gravity,
                                                      need_move_frame, need_resize_frame);
 
+  if (window->frame)
+    window->buffer_rect = window->frame->rect;
+  else
+    window->buffer_rect = client_rect;
+
   if (need_configure_notify)
     send_configure_notify (window);
 
@@ -3265,6 +3271,7 @@ meta_window_x11_configure_notify (MetaWindow      *window,
   window->rect.height = event->height;
 
   priv->client_rect = window->rect;
+  window->buffer_rect = window->rect;
 
   meta_window_update_monitor (window);