wayland: Adds basic hybrid X + Wayland support

This adds support for running mutter as a hybrid X and Wayland
compositor. It runs a headless XWayland server for X applications
that presents wayland surfaces back to mutter which mutter can then
composite.

This aims to not break Mutter's existing support for the traditional X
compositing model which means a single build of Mutter can be
distributed supporting the traditional model and the new Wayland based
compositing model.

TODO: although building with --disable-wayland has at least been tested,
I still haven't actually verified that running as a traditional
compositor isn't broken currently.

Note: At this point no input is supported

Note: multiple authors have contributed to this patch:
Authored-by: Robert Bragg <robert@linux.intel.com>
Authored-by: Neil Roberts <neil@linux.intel.com>
Authored-by: Rico Tzschichholz.
Authored-by: Giovanni Campagna <gcampagna@src.gnome.org>

src/wayland/meta-wayland-private.h

@@ -0,0 +1,171 @@
+/*
+ * Copyright (C) 2012 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
+ * 02111-1307, USA.
+ */
+
+#ifndef META_WAYLAND_PRIVATE_H
+#define META_WAYLAND_PRIVATE_H
+
+#include <wayland-server.h>
+
+#include <clutter/clutter.h>
+
+#include <glib.h>
+#include <cairo.h>
+
+#include "window-private.h"
+
+typedef struct _MetaWaylandCompositor MetaWaylandCompositor;
+
+typedef struct
+{
+  struct wl_resource *resource;
+  struct wl_signal destroy_signal;
+  struct wl_listener destroy_listener;
+
+  union
+  {
+    struct wl_shm_buffer *shm_buffer;
+    struct wl_buffer *legacy_buffer;
+  };
+
+  int32_t width, height;
+  uint32_t busy_count;
+} MetaWaylandBuffer;
+
+typedef struct
+{
+  MetaWaylandBuffer *buffer;
+  struct wl_listener destroy_listener;
+} MetaWaylandBufferReference;
+
+typedef struct
+{
+  struct wl_resource *resource;
+  cairo_region_t *region;
+} MetaWaylandRegion;
+
+struct _MetaWaylandSurface
+{
+  struct wl_resource *resource;
+  MetaWaylandCompositor *compositor;
+  guint32 xid;
+  int x;
+  int y;
+  MetaWaylandBufferReference buffer_ref;
+  MetaWindow *window;
+  gboolean has_shell_surface;
+
+  /* All the pending state, that wl_surface.commit will apply. */
+  struct
+  {
+    /* wl_surface.attach */
+    gboolean newly_attached;
+    MetaWaylandBuffer *buffer;
+    struct wl_listener buffer_destroy_listener;
+    int32_t sx;
+    int32_t sy;
+
+    /* wl_surface.damage */
+    cairo_region_t *damage;
+
+    /* wl_surface.frame */
+    struct wl_list frame_callback_list;
+  } pending;
+};
+
+#ifndef HAVE_META_WAYLAND_SURFACE_TYPE
+typedef struct _MetaWaylandSurface MetaWaylandSurface;
+#endif
+
+typedef struct
+{
+  MetaWaylandSurface *surface;
+  struct wl_resource *resource;
+  struct wl_listener surface_destroy_listener;
+} MetaWaylandShellSurface;
+
+typedef struct
+{
+  guint32 flags;
+  int width;
+  int height;
+  int refresh;
+} MetaWaylandMode;
+
+typedef struct
+{
+  struct wl_object wayland_output;
+  int x;
+  int y;
+  int width_mm;
+  int height_mm;
+  /* XXX: with sliced stages we'd reference a CoglFramebuffer here. */
+
+  GList *modes;
+} MetaWaylandOutput;
+
+typedef struct
+{
+  GSource source;
+  GPollFD pfd;
+  struct wl_display *display;
+} WaylandEventSource;
+
+typedef struct
+{
+  struct wl_list link;
+
+  /* Pointer back to the compositor */
+  MetaWaylandCompositor *compositor;
+
+  struct wl_resource *resource;
+} MetaWaylandFrameCallback;
+
+struct _MetaWaylandCompositor
+{
+  struct wl_display *wayland_display;
+  struct wl_event_loop *wayland_loop;
+  GMainLoop *init_loop;
+  ClutterActor *stage;
+  GList *outputs;
+  GSource *wayland_event_source;
+  GList *surfaces;
+  struct wl_list frame_callbacks;
+
+  int xwayland_display_index;
+  char *xwayland_lockfile;
+  int xwayland_abstract_fd;
+  int xwayland_unix_fd;
+  pid_t xwayland_pid;
+  struct wl_client *xwayland_client;
+  struct wl_resource *xserver_resource;
+  GHashTable *window_surfaces;
+};
+
+void                    meta_wayland_init                   (void);
+void                    meta_wayland_finalize               (void);
+
+/* We maintain a singleton MetaWaylandCompositor which can be got at via this
+ * API after meta_wayland_init() has been called. */
+MetaWaylandCompositor  *meta_wayland_compositor_get_default (void);
+
+void                    meta_wayland_handle_sig_child       (void);
+
+MetaWaylandSurface     *meta_wayland_lookup_surface_for_xid (guint32 xid);
+
+#endif /* META_WAYLAND_PRIVATE_H */

src/wayland/meta-xwayland-private.h

@@ -0,0 +1,33 @@
+/*
+ * Copyright (C) 2013 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
+ * 02111-1307, USA.
+ */
+
+#ifndef META_XWAYLAND_PRIVATE_H
+#define META_XWAYLAND_PRIVATE_H
+
+#include "meta-wayland-private.h"
+
+#include <glib.h>
+
+gboolean
+meta_xwayland_start (MetaWaylandCompositor *compositor);
+
+void
+meta_xwayland_stop (MetaWaylandCompositor *compositor);
+
+#endif /* META_XWAYLAND_PRIVATE_H */

src/wayland/meta-xwayland.c

@@ -0,0 +1,310 @@
+/*
+ * X Wayland Support
+ *
+ * Copyright (C) 2013 Intel Corporation
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA
+ * 02111-1307, USA.
+ */
+
+#include "meta-xwayland-private.h"
+
+#include <glib.h>
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+static char *
+create_lockfile (int display, int *display_out)
+{
+  char *filename;
+  int size;
+  char pid[11];
+  int fd;
+
+  do
+    {
+      char *end;
+      pid_t other;
+
+      filename = g_strdup_printf ("/tmp/.X%d-lock", display);
+      fd = open (filename, O_WRONLY | O_CLOEXEC | O_CREAT | O_EXCL, 0444);
+
+      if (fd < 0 && errno == EEXIST)
+        {
+          fd = open (filename, O_CLOEXEC, O_RDONLY);
+          if (fd < 0 || read (fd, pid, 11) != 11)
+            {
+              const char *msg = strerror (errno);
+              g_warning ("can't read lock file %s: %s", filename, msg);
+              g_free (filename);
+
+              /* ignore error and try the next display number */
+              display++;
+              continue;
+          }
+          close (fd);
+
+          other = strtol (pid, &end, 0);
+          if (end != pid + 10)
+            {
+              g_warning ("can't parse lock file %s", filename);
+              g_free (filename);
+
+              /* ignore error and try the next display number */
+              display++;
+              continue;
+          }
+
+          if (kill (other, 0) < 0 && errno == ESRCH)
+            {
+              g_warning ("unlinking stale lock file %s", filename);
+              if (unlink (filename) < 0)
+                {
+                  const char *msg = strerror (errno);
+                  g_warning ("failed to unlink stale lock file: %s", msg);
+                  display++;
+                }
+              g_free (filename);
+              continue;
+          }
+
+          g_free (filename);
+          display++;
+          continue;
+        }
+      else if (fd < 0)
+        {
+          const char *msg = strerror (errno);
+          g_warning ("failed to create lock file %s: %s", filename , msg);
+          g_free (filename);
+          return NULL;
+        }
+
+      break;
+    }
+  while (1);
+
+  /* Subtle detail: we use the pid of the wayland compositor, not the xserver
+   * in the lock file. */
+  size = snprintf (pid, 11, "%10d\n", getpid ());
+  if (size != 11 || write (fd, pid, 11) != 11)
+    {
+      unlink (filename);
+      close (fd);
+      g_warning ("failed to write pid to lock file %s", filename);
+      g_free (filename);
+      return NULL;
+    }
+
+  close (fd);
+
+  *display_out = display;
+  return filename;
+}
+
+static int
+bind_to_abstract_socket (int display)
+{
+  struct sockaddr_un addr;
+  socklen_t size, name_size;
+  int fd;
+
+  fd = socket (PF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0);
+  if (fd < 0)
+    return -1;
+
+  addr.sun_family = AF_LOCAL;
+  name_size = snprintf (addr.sun_path, sizeof addr.sun_path,
+                        "%c/tmp/.X11-unix/X%d", 0, display);
+  size = offsetof (struct sockaddr_un, sun_path) + name_size;
+  if (bind (fd, (struct sockaddr *) &addr, size) < 0)
+    {
+      g_warning ("failed to bind to @%s: %s\n",
+                 addr.sun_path + 1, strerror (errno));
+      close (fd);
+      return -1;
+    }
+
+  if (listen (fd, 1) < 0)
+    {
+      close (fd);
+      return -1;
+    }
+
+  return fd;
+}
+
+static int
+bind_to_unix_socket (int display)
+{
+  struct sockaddr_un addr;
+  socklen_t size, name_size;
+  int fd;
+
+  fd = socket (PF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0);
+  if (fd < 0)
+    return -1;
+
+  addr.sun_family = AF_LOCAL;
+  name_size = snprintf (addr.sun_path, sizeof addr.sun_path,
+                        "/tmp/.X11-unix/X%d", display) + 1;
+  size = offsetof (struct sockaddr_un, sun_path) + name_size;
+  unlink (addr.sun_path);
+  if (bind (fd, (struct sockaddr *) &addr, size) < 0)
+    {
+      char *msg = strerror (errno);
+      g_warning ("failed to bind to %s (%s)\n", addr.sun_path, msg);
+      close (fd);
+      return -1;
+    }
+
+  if (listen (fd, 1) < 0) {
+      unlink (addr.sun_path);
+      close (fd);
+      return -1;
+  }
+
+  return fd;
+}
+
+gboolean
+meta_xwayland_start (MetaWaylandCompositor *compositor)
+{
+  int display = 0;
+  char *lockfile = NULL;
+  int sp[2];
+  pid_t pid;
+
+  do
+    {
+      lockfile = create_lockfile (display, &display);
+      if (!lockfile)
+        {
+         g_warning ("Failed to create an X lock file");
+         return FALSE;
+        }
+
+      compositor->xwayland_abstract_fd = bind_to_abstract_socket (display);
+      if (compositor->xwayland_abstract_fd < 0)
+        {
+          unlink (lockfile);
+
+          if (errno == EADDRINUSE)
+            {
+              display++;
+              continue;
+            }
+          else
+            return FALSE;
+        }
+
+      compositor->xwayland_unix_fd = bind_to_unix_socket (display);
+      if (compositor->xwayland_abstract_fd < 0)
+        {
+          unlink (lockfile);
+          close (compositor->xwayland_abstract_fd);
+          return FALSE;
+        }
+
+      break;
+    }
+  while (1);
+
+  compositor->xwayland_display_index = display;
+  compositor->xwayland_lockfile = lockfile;
+
+  /* We want xwayland to be a wayland client so we make a socketpair to setup a
+   * wayland protocol connection. */
+  if (socketpair (AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sp) < 0)
+    {
+      g_warning ("socketpair failed\n");
+      unlink (lockfile);
+      return 1;
+    }
+
+  switch ((pid = fork()))
+    {
+    case 0:
+        {
+          char *fd_string;
+          char *display_name;
+          /* Make sure the client end of the socket pair doesn't get closed
+           * when we exec xwayland. */
+          int flags = fcntl (sp[1], F_GETFD);
+          if (flags != -1)
+            fcntl (sp[1], F_SETFD, flags & ~FD_CLOEXEC);
+
+          fd_string = g_strdup_printf ("%d", sp[1]);
+          setenv ("WAYLAND_SOCKET", fd_string, 1);
+          g_free (fd_string);
+
+          display_name = g_strdup_printf (":%d",
+                                          compositor->xwayland_display_index);
+
+          if (execl (XWAYLAND_PATH,
+                     XWAYLAND_PATH,
+                     display_name,
+                     "-wayland",
+                     "-rootless",
+                     "-retro",
+                     "-noreset",
+                     /* FIXME: does it make sense to log to the filesystem by
+                      * default? */
+                     "-logfile", "/tmp/xwayland.log",
+                     "-nolisten", "all",
+                     NULL) < 0)
+            {
+              char *msg = strerror (errno);
+              g_warning ("xwayland exec failed: %s", msg);
+            }
+          exit (-1);
+          return FALSE;
+        }
+    default:
+      g_message ("forked X server, pid %d\n", pid);
+
+      close (sp[1]);
+      compositor->xwayland_client =
+        wl_client_create (compositor->wayland_display, sp[0]);
+
+      compositor->xwayland_pid = pid;
+      break;
+
+    case -1:
+      g_error ("Failed to fork for xwayland server");
+      return FALSE;
+    }
+
+  return TRUE;
+}
+
+void
+meta_xwayland_stop (MetaWaylandCompositor *compositor)
+{
+  char path[256];
+
+  snprintf (path, sizeof path, "/tmp/.X%d-lock",
+            compositor->xwayland_display_index);
+  unlink (path);
+  snprintf (path, sizeof path, "/tmp/.X11-unix/X%d",
+            compositor->xwayland_display_index);
+  unlink (path);
+
+  unlink (compositor->xwayland_lockfile);
+}