[mutter] xwayland: Do not rely on X-lock files

[Marge Bot <marge-bot src gnome org>]

commit eb06d9e16514e452e05acbd2173552820075588f
Author: Olivier Fourdan <ofourdan redhat com>
Date:   Thu Jan 14 09:56:13 2021 +0100

    xwayland: Do not rely on X-lock files
    
    Some X11 servers may not always create a lock file, yet mutter uses the
    lock file to find a possible display number and then tries to bind to
    the socket corresponding to that display number.
    
    If it fails to bind, it will simply bail out. As a result, if an X11
    server is already listening on that display but hadn't created a lock
    file, mutter won't be able to start Xwayland.
    
    To avoid that possible issue, make mutter retry with another display
    for a given number of tries when binding fails even though the display
    was supposed to be available based on the lock file presence.
    
    Closes: https://gitlab.gnome.org/GNOME/mutter/-/issues/1604
    Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/1669>

 src/wayland/meta-xwayland.c | 28 ++++++++--------------------
 1 file changed, 8 insertions(+), 20 deletions(-)
---
diff --git a/src/wayland/meta-xwayland.c b/src/wayland/meta-xwayland.c
index 7dc7077f63..9d2e608aa3 100644
--- a/src/wayland/meta-xwayland.c
+++ b/src/wayland/meta-xwayland.c
@@ -439,9 +439,8 @@ create_lock_file (int      display,
 }
 
 static int
-bind_to_abstract_socket (int        display,
-                         gboolean  *fatal,
-                         GError   **error)
+bind_to_abstract_socket (int      display,
+                         GError **error)
 {
   struct sockaddr_un addr;
   socklen_t size, name_size;
@@ -450,7 +449,6 @@ bind_to_abstract_socket (int        display,
   fd = socket (PF_LOCAL, SOCK_STREAM | SOCK_CLOEXEC, 0);
   if (fd < 0)
     {
-      *fatal = TRUE;
       g_set_error (error, G_IO_ERROR, g_io_error_from_errno (errno),
                    "Failed to create socket: %s", g_strerror (errno));
       return -1;
@@ -462,7 +460,6 @@ bind_to_abstract_socket (int        display,
   size = offsetof (struct sockaddr_un, sun_path) + name_size;
   if (bind (fd, (struct sockaddr *) &addr, size) < 0)
     {
-      *fatal = errno != EADDRINUSE;
       g_set_error (error, G_IO_ERROR, g_io_error_from_errno (errno),
                    "Failed to bind to %s: %s",
                    addr.sun_path + 1, g_strerror (errno));
@@ -472,7 +469,6 @@ bind_to_abstract_socket (int        display,
 
   if (listen (fd, 1) < 0)
     {
-      *fatal = errno != EADDRINUSE;
       g_set_error (error, G_IO_ERROR, g_io_error_from_errno (errno),
                    "Failed to listen to %s: %s",
                    addr.sun_path + 1, g_strerror (errno));
@@ -644,19 +640,17 @@ open_display_sockets (MetaXWaylandManager  *manager,
                       int                   display_index,
                       int                  *abstract_fd_out,
                       int                  *unix_fd_out,
-                      gboolean             *fatal,
                       GError              **error)
 {
   int abstract_fd, unix_fd;
 
-  abstract_fd = bind_to_abstract_socket (display_index, fatal, error);
+  abstract_fd = bind_to_abstract_socket (display_index, error);
   if (abstract_fd < 0)
     return FALSE;
 
   unix_fd = bind_to_unix_socket (display_index, error);
   if (unix_fd < 0)
     {
-      *fatal = FALSE;
       close (abstract_fd);
       return FALSE;
     }
@@ -673,8 +667,8 @@ choose_xdisplay (MetaXWaylandManager     *manager,
                  GError                 **error)
 {
   int display = 0;
+  int number_of_tries = 0;
   char *lock_file = NULL;
-  gboolean fatal = FALSE;
 
   if (display_number_override != -1)
     display = display_number_override;
@@ -699,22 +693,19 @@ choose_xdisplay (MetaXWaylandManager     *manager,
       if (!open_display_sockets (manager, display,
                                  &connection->abstract_fd,
                                  &connection->unix_fd,
-                                 &fatal,
                                  &local_error))
         {
           unlink (lock_file);
 
-          if (!fatal)
-            {
-              display++;
-              continue;
-            }
-          else
+          if (++number_of_tries >= 50)
             {
               g_prefix_error (&local_error, "Failed to bind X11 socket: ");
               g_propagate_error (error, g_steal_pointer (&local_error));
               return FALSE;
             }
+
+          display++;
+          continue;
         }
 
       break;
@@ -1064,7 +1055,6 @@ meta_xwayland_init (MetaXWaylandManager  *manager,
                     GError              **error)
 {
   MetaDisplayPolicy policy;
-  gboolean fatal;
 
   if (!manager->public_connection.name)
     {
@@ -1082,7 +1072,6 @@ meta_xwayland_init (MetaXWaylandManager  *manager,
                                  manager->public_connection.display_index,
                                  &manager->public_connection.abstract_fd,
                                  &manager->public_connection.unix_fd,
-                                 &fatal,
                                  error))
         return FALSE;
 
@@ -1090,7 +1079,6 @@ meta_xwayland_init (MetaXWaylandManager  *manager,
                                  manager->private_connection.display_index,
                                  &manager->private_connection.abstract_fd,
                                  &manager->private_connection.unix_fd,
-                                 &fatal,
                                  error))
         return FALSE;
     }