xwayland: Do not rely on X-lock files

Some X11 servers may not always create a lock file, yet mutter uses the
lock file to find a possible display number and then tries to bind to
the socket corresponding to that display number.

If it fails to bind, it will simply bail out. As a result, if an X11
server is already listening on that display but hadn't created a lock
file, mutter won't be able to start Xwayland.

To avoid that possible issue, make mutter retry with another display
for a given number of tries when binding fails even though the display
was supposed to be available based on the lock file presence.

Closes: https://gitlab.gnome.org/GNOME/mutter/-/issues/1604
Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1669>

src/wayland/meta-xwayland.c

@@ -440,7 +440,6 @@ create_lock_file (int      display,
 
 static int
 bind_to_abstract_socket (int      display,
-                         gboolean  *fatal,
                          GError **error)
 {
   struct sockaddr_un addr;
@@ -450,7 +449,6 @@ bind_to_abstract_socket (int        display,
   fd = socket (PF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0);
   if (fd < 0)
     {
-      *fatal = TRUE;
       g_set_error (error, G_IO_ERROR, g_io_error_from_errno (errno),
                    "Failed to create socket: %s", g_strerror (errno));
       return -1;
@@ -462,7 +460,6 @@ bind_to_abstract_socket (int        display,
   size = offsetof (struct sockaddr_un, sun_path) + name_size;
   if (bind (fd, (struct sockaddr *) &addr, size) < 0)
     {
-      *fatal = errno != EADDRINUSE;
       g_set_error (error, G_IO_ERROR, g_io_error_from_errno (errno),
                    "Failed to bind to %s: %s",
                    addr.sun_path + 1, g_strerror (errno));
@@ -472,7 +469,6 @@ bind_to_abstract_socket (int        display,
 
   if (listen (fd, 1) < 0)
     {
-      *fatal = errno != EADDRINUSE;
       g_set_error (error, G_IO_ERROR, g_io_error_from_errno (errno),
                    "Failed to listen to %s: %s",
                    addr.sun_path + 1, g_strerror (errno));
@@ -644,19 +640,17 @@ open_display_sockets (MetaXWaylandManager  *manager,
                       int                   display_index,
                       int                  *abstract_fd_out,
                       int                  *unix_fd_out,
-                      gboolean             *fatal,
                       GError              **error)
 {
   int abstract_fd, unix_fd;
 
-  abstract_fd = bind_to_abstract_socket (display_index, fatal, error);
+  abstract_fd = bind_to_abstract_socket (display_index, error);
   if (abstract_fd < 0)
     return FALSE;
 
   unix_fd = bind_to_unix_socket (display_index, error);
   if (unix_fd < 0)
     {
-      *fatal = FALSE;
       close (abstract_fd);
       return FALSE;
     }
@@ -673,8 +667,8 @@ choose_xdisplay (MetaXWaylandManager     *manager,
                  GError                 **error)
 {
   int display = 0;
+  int number_of_tries = 0;
   char *lock_file = NULL;
-  gboolean fatal = FALSE;
 
   if (display_number_override != -1)
     display = display_number_override;
@@ -699,22 +693,19 @@ choose_xdisplay (MetaXWaylandManager     *manager,
       if (!open_display_sockets (manager, display,
                                  &connection->abstract_fd,
                                  &connection->unix_fd,
-                                 &fatal,
                                  &local_error))
         {
           unlink (lock_file);
 
-          if (!fatal)
+          if (++number_of_tries >= 50)
-            {
-              display++;
-              continue;
-            }
-          else
             {
               g_prefix_error (&local_error, "Failed to bind X11 socket: ");
               g_propagate_error (error, g_steal_pointer (&local_error));
               return FALSE;
             }
+
+          display++;
+          continue;
         }
 
       break;
@@ -1064,7 +1055,6 @@ meta_xwayland_init (MetaXWaylandManager  *manager,
                     GError              **error)
 {
   MetaDisplayPolicy policy;
-  gboolean fatal;
 
   if (!manager->public_connection.name)
     {
@@ -1082,7 +1072,6 @@ meta_xwayland_init (MetaXWaylandManager  *manager,
                                  manager->public_connection.display_index,
                                  &manager->public_connection.abstract_fd,
                                  &manager->public_connection.unix_fd,
-                                 &fatal,
                                  error))
         return FALSE;
 
@@ -1090,7 +1079,6 @@ meta_xwayland_init (MetaXWaylandManager  *manager,
                                  manager->private_connection.display_index,
                                  &manager->private_connection.abstract_fd,
                                  &manager->private_connection.unix_fd,
-                                 &fatal,
                                  error))
         return FALSE;
     }