x11: Adopt X11 error trap infrastructure from GDK

From reading the comment in the top of the file, not for the first
time. Keep our own error handler and maintain our own list of
failable x11 sequences in MetaX11Display, so we can move away from
GTK's.

Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/2864>

src/x11/meta-x11-display-private.h

@@ -144,6 +144,8 @@ struct _MetaX11Display
   GSubprocess *frames_client;
   GCancellable *frames_client_cancellable;
 
+  GList *error_traps;
+
   struct {
     Window xwindow;
     guint timeout_id;
@@ -298,4 +300,6 @@ gboolean meta_x11_display_xwindow_is_a_no_focus_window (MetaX11Display *x11_disp
 
 void meta_x11_display_clear_stage_input_region (MetaX11Display *x11_display);
 
+void meta_x11_display_init_error_traps (MetaX11Display *x11_display);
+
 #endif /* META_X11_DISPLAY_PRIVATE_H */

src/x11/meta-x11-display.c

@@ -288,6 +288,8 @@ meta_x11_display_dispose (GObject *object)
   g_free (x11_display->screen_name);
   x11_display->screen_name = NULL;
 
+  g_clear_list (&x11_display->error_traps, g_free);
+
   G_OBJECT_CLASS (meta_x11_display_parent_class)->dispose (object);
 }
 
@@ -1320,6 +1322,8 @@ meta_x11_display_new (MetaDisplay  *display,
 #include "x11/atomnames.h"
 #undef item
 
+  meta_x11_display_init_error_traps (x11_display);
+
   query_xsync_extension (x11_display);
   query_xshape_extension (x11_display);
   query_xcomposite_extension (x11_display);

src/x11/meta-x11-errors.c

@@ -1,5 +1,4 @@
 /* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- */
-
 /*
  * Copyright (C) 2001 Havoc Pennington, error trapping inspired by GDK
  * code copyrighted by the GTK team.
@@ -30,36 +29,274 @@
 
 #include <errno.h>
 #include <stdlib.h>
-#include <gdk/gdkx.h>
+#include <X11/Xlibint.h>
 
 #include "x11/meta-x11-display-private.h"
 
-/* In GTK+-3.0, the error trapping code was significantly rewritten. The new code
- * has some neat features (like knowing automatically if a sync is needed or not
- * and handling errors asynchronously when the error code isn't needed immediately),
- * but it's basically incompatible with the hacks we played with GTK+-2.0 to
- * use a custom error handler along with gdk_error_trap_push().
- *
- * Since the main point of our custom error trap was to get the error logged
- * to the right place, with GTK+-3.0 we simply omit our own error handler and
- * use the GTK+ handling straight-up.
- * (See https://bugzilla.gnome.org/show_bug.cgi?id=630216 for restoring logging.)
+/* compare X sequence numbers handling wraparound */
+#define SEQUENCE_COMPARE(a,op,b) (((long) (a) - (long) (b)) op 0)
+
+typedef struct _MetaErrorTrap
+{
+  /* Next sequence when trap was pushed, i.e. first sequence to
+   * ignore
+   */
+  unsigned long start_sequence;
+
+  /* Next sequence when trap was popped, i.e. first sequence
+   * to not ignore. 0 if trap is still active.
+   */
+  unsigned long end_sequence;
+
+  /* Most recent error code within the sequence */
+  int error_code;
+} MetaErrorTrap;
+
+/* Previously existing error handler */
+typedef int (* MetaXErrorHandler) (Display *, XErrorEvent *);
+static MetaXErrorHandler old_error_handler = NULL;
+/* number of times we've pushed the error handler */
+static int error_handler_push_count = 0;
+static MetaX11Display *error_x11_display = NULL;
+
+/* look up the extension name for a given major opcode.  grubs around in
+ * xlib to do it since a) it’s already cached there b) XQueryExtension
+ * emits protocol so we can’t use it in an error handler.
  */
+static const char *
+decode_request_code (Display *dpy,
+                     int      code)
+{
+  _XExtension *ext;
+
+  if (code < 128)
+    return "core protocol";
+
+  for (ext = dpy->ext_procs; ext; ext = ext->next)
+    {
+      if (ext->codes.major_opcode == code)
+        return ext->name;
+    }
+
+  return "unknown";
+}
+
+static void
+display_error_event (MetaX11Display *x11_display,
+                     XErrorEvent    *error)
+{
+  GList *l;
+  gboolean ignore = FALSE;
+
+  for (l = x11_display->error_traps; l; l = l->next)
+    {
+      MetaErrorTrap *trap;
+
+      trap = l->data;
+
+      if (SEQUENCE_COMPARE (trap->start_sequence, <=, error->serial) &&
+          (trap->end_sequence == 0 ||
+           SEQUENCE_COMPARE (trap->end_sequence, >, error->serial)))
+        {
+          ignore = TRUE;
+          trap->error_code = error->error_code;
+          break; /* only innermost trap gets the error code */
+        }
+    }
+
+  if (!ignore)
+    {
+      char buf[64];
+
+      XGetErrorText (x11_display->xdisplay, error->error_code, buf, 63);
+
+      g_error ("Received an X Window System error.\n"
+               "This probably reflects a bug in the program.\n"
+               "The error was '%s'.\n"
+               "  (Details: serial %ld error_code %d request_code %d (%s) minor_code %d)\n"
+               "  (Note to programmers: normally, X errors are reported asynchronously;\n"
+               "   that is, you will receive the error a while after causing it.\n"
+               "   To debug your program, run it with the MUTTER_SYNC environment\n"
+               "   variable to change this behavior. You can then get a meaningful\n"
+               "   backtrace from your debugger if you break on the meta_x_error() function.)",
+               buf,
+               error->serial,
+               error->error_code,
+               error->request_code,
+               decode_request_code (x11_display->xdisplay, error->request_code),
+               error->minor_code);
+    }
+}
+
+static int
+meta_x_error (Display     *xdisplay,
+              XErrorEvent *error)
+{
+  if (error->error_code)
+    display_error_event (error_x11_display, error);
+
+  return 0;
+}
+
+static void
+error_handler_push (void)
+{
+  MetaXErrorHandler previous_handler;
+
+  previous_handler = XSetErrorHandler (meta_x_error);
+
+  if (error_handler_push_count > 0)
+    {
+      if (previous_handler != meta_x_error)
+        g_warning ("XSetErrorHandler() called with a Mutter X11 error trap pushed. Don't do that.");
+    }
+  else
+    {
+      old_error_handler = previous_handler;
+    }
+
+  error_handler_push_count += 1;
+}
+
+static void
+error_handler_pop (void)
+{
+  g_return_if_fail (error_handler_push_count > 0);
+
+  error_handler_push_count -= 1;
+
+  if (error_handler_push_count == 0)
+    {
+      XSetErrorHandler (old_error_handler);
+      old_error_handler = NULL;
+    }
+}
+
+static void
+delete_outdated_error_traps (MetaX11Display *x11_display)
+{
+  GList *l;
+  unsigned long processed_sequence;
+
+  processed_sequence = XLastKnownRequestProcessed (x11_display->xdisplay);
+  l = x11_display->error_traps;
+
+  while (l != NULL)
+    {
+      MetaErrorTrap *trap = l->data;
+
+      if (trap->end_sequence != 0 &&
+          SEQUENCE_COMPARE (trap->end_sequence, <=, processed_sequence))
+        {
+          GList *link = l;
+
+          l = l->next;
+          x11_display->error_traps =
+            g_list_delete_link (x11_display->error_traps, link);
+          g_free (trap);
+        }
+      else
+        {
+          l = l->next;
+        }
+    }
+}
+
+void
+meta_x11_display_init_error_traps (MetaX11Display *x11_display)
+{
+  error_x11_display = x11_display;
+  XSetErrorHandler (meta_x_error);
+}
 
 void
 meta_x11_error_trap_push (MetaX11Display *x11_display)
 {
-  gdk_x11_display_error_trap_push (x11_display->gdk_display);
+  MetaErrorTrap *trap;
+
+  delete_outdated_error_traps (x11_display);
+
+  /* set up the Xlib callback to tell us about errors */
+  error_handler_push ();
+
+  trap = g_new0 (MetaErrorTrap, 1);
+  trap->start_sequence = XNextRequest (x11_display->xdisplay);
+  trap->error_code = Success;
+
+  x11_display->error_traps =
+    g_list_prepend (x11_display->error_traps, trap);
+}
+
+static int
+meta_x11_error_trap_pop_internal (MetaX11Display *x11_display,
+                                  gboolean        need_code)
+{
+  MetaErrorTrap *trap = NULL;
+  GList *l;
+  int result;
+
+  g_return_val_if_fail (x11_display->error_traps != NULL, Success);
+
+  /* Find the first trap that hasn't been popped already */
+  for (l = x11_display->error_traps; l; l = l->next)
+    {
+      trap = l->data;
+
+      if (trap->end_sequence == 0)
+        break;
+    }
+
+  g_return_val_if_fail (trap != NULL, Success);
+  g_assert (trap->end_sequence == 0);
+
+  /* May need to sync to fill in trap->error_code if we care about
+   * getting an error code.
+   */
+  if (need_code)
+    {
+      unsigned long processed_sequence, next_sequence;
+
+      next_sequence = XNextRequest (x11_display->xdisplay);
+      processed_sequence = XLastKnownRequestProcessed (x11_display->xdisplay);
+
+      /* If our last request was already processed, there is no point
+       * in syncing. i.e. if last request was a round trip (or even if
+       * we got an event with the serial of a non-round-trip)
+       */
+      if ((next_sequence - 1) != processed_sequence)
+        {
+          XSync (x11_display->xdisplay, False);
+        }
+
+      result = trap->error_code;
+    }
+  else
+    {
+      result = Success;
+    }
+
+  /* record end of trap, giving us a range of
+   * error sequences we'll ignore.
+   */
+  trap->end_sequence = XNextRequest (x11_display->xdisplay);
+
+  /* remove the Xlib callback */
+  error_handler_pop ();
+
+  /* we may already be outdated */
+  delete_outdated_error_traps (x11_display);
+
+  return result;
 }
 
 void
 meta_x11_error_trap_pop (MetaX11Display *x11_display)
 {
-  gdk_x11_display_error_trap_pop_ignored (x11_display->gdk_display);
+  meta_x11_error_trap_pop_internal (x11_display, FALSE);
 }
 
 int
 meta_x11_error_trap_pop_with_return (MetaX11Display *x11_display)
 {
-  return gdk_x11_display_error_trap_pop (x11_display->gdk_display);
+  return meta_x11_error_trap_pop_internal (x11_display, TRUE);
 }