window: Assume CSD frame extents are 0 in maximized and tiling modes

When the user maximized a CSD window, this is the order of operations
that happens:

  1. The CSD application sends us a _NET_WM_STATE client message telling
     us to add the _NET_WM_MAXIMIZED_VERT/HORZ state.

  2. We respond by setting _NET_WM_STATE on the window and resizing the
     window. Windows are positioned according to the visible borders of
     the window, so since _GTK_FRAME_EXTENTS has not been updated yet to
     reflect the border/padding of the window state, the borders are wrong.

  3. GTK+ gets the _NET_WM_STATE event, updates its state, and then sets
     new theme properties, including _GTK_FRAME_EXTENTS.

  4. mutter sees that _GTK_FRAME_EXTENTS has been updated, and repositions
     the window to where it should be.

This means that the user sees the borders "flicker" for a half-second while
the maximization state is in limbo. Trying to make this resize operation
atomic is a foregone conclusion under X11, so to solve this, we're going to
make two changes to our _GTK_FRAME_EXTENTS protocol:

  1. _GTK_FRAME_EXTENTS now reflects the "normal" borders, always. This
     means that _GTK_FRAME_EXTENTS should not change at runtime unless the
     theme changes.

  2. mutter will only respect _GTK_FRAME_EXTENTS in places where the user
     expects to see a shadow. This means that mutter will hardcode the
     extents to 0 when the window is maximized. If the window is tiled-left,
     then all extents will be 0, except for the shadow on the right hand
     side, and vice versa for the tiled-right state.

src/core/window.c

@@ -5727,6 +5727,32 @@ meta_window_get_input_rect (const MetaWindow *window,
     *rect = window->rect;
 }
 
+static void
+get_custom_frame_extents (MetaWindow *window,
+                          GtkBorder  *extents)
+{
+  if (!window->has_custom_frame_extents)
+    return;
+
+  *extents = window->custom_frame_extents;
+
+  if (META_WINDOW_MAXIMIZED_HORIZONTALLY (window))
+    {
+      extents->left = 0;
+      extents->right = 0;
+    }
+  else if (META_WINDOW_TILED_LEFT (window))
+    extents->left = 0;
+  else if (META_WINDOW_TILED_RIGHT (window))
+    extents->right = 0;
+
+  if (META_WINDOW_MAXIMIZED_VERTICALLY (window))
+    {
+      extents->top = 0;
+      extents->bottom = 0;
+    }
+}
+
 /**
  * meta_window_client_rect_to_frame_rect:
  * @window: a #MetaWindow
@@ -5765,16 +5791,15 @@ meta_window_client_rect_to_frame_rect (MetaWindow    *window,
     }
   else
     {
-      if (window->has_custom_frame_extents)
+      GtkBorder extents = { 0 };
-        {
+      get_custom_frame_extents (window, &extents);
-          const GtkBorder *extents = &window->custom_frame_extents;
+
-          frame_rect->x += extents->left;
+      frame_rect->x += extents.left;
-          frame_rect->y += extents->top;
+      frame_rect->y += extents.top;
-          if (frame_rect->width != G_MAXINT)
+      if (frame_rect->width != G_MAXINT)
-            frame_rect->width -= extents->left + extents->right;
+        frame_rect->width -= extents.left + extents.right;
-          if (frame_rect->height != G_MAXINT)
+      if (frame_rect->height != G_MAXINT)
-            frame_rect->height -= extents->top + extents->bottom;
+        frame_rect->height -= extents.top + extents.bottom;
-        }
     }
 }
 
@@ -5809,14 +5834,13 @@ meta_window_frame_rect_to_client_rect (MetaWindow    *window,
     }
   else
     {
-      if (window->has_custom_frame_extents)
+      GtkBorder extents = { 0 };
-        {
+      get_custom_frame_extents (window, &extents);
-          const GtkBorder *extents = &window->custom_frame_extents;
+
-          client_rect->x -= extents->left;
+      client_rect->x -= extents.left;
-          client_rect->y -= extents->top;
+      client_rect->y -= extents.top;
-          client_rect->width += extents->left + extents->right;
+      client_rect->width += extents.left + extents.right;
-          client_rect->height += extents->top + extents->bottom;
+      client_rect->height += extents.top + extents.bottom;
-        }
     }
 }