[mutter] context: Add a new API to raise and restore NOFILE limits

[Marge Bot <marge-bot src gnome org>]

commit 8b9341131a4418d59c268b5fbcda488129d6b11a
Author: Olivier Fourdan <ofourdan redhat com>
Date:   Fri Jan 14 11:33:22 2022 +0100

    context: Add a new API to raise and restore NOFILE limits
    
    Wayland, especially when using SHM buffers, can use a lot of file
    descriptors and can possibly reach the NOFILE soft limit.
    
    Add a new API to MetaContext to raise and restore the NOFILE limit so
    that the Wayland compositor can use as many file descriptors as the hard
    limit allows.
    
    See-also: https://gitlab.freedesktop.org/xorg/xserver/-/issues/1283
    Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/2235>

 src/core/meta-context.c | 113 ++++++++++++++++++++++++++++++++++++++++++++++++
 src/meta/meta-context.h |   8 ++++
 2 files changed, 121 insertions(+)
---
diff --git a/src/core/meta-context.c b/src/core/meta-context.c
index f38f94d439..4363e14e11 100644
--- a/src/core/meta-context.c
+++ b/src/core/meta-context.c
@@ -23,6 +23,7 @@
 #include "core/meta-context-private.h"
 
 #include <locale.h>
+#include <sys/resource.h>
 
 #include "backends/meta-backend-private.h"
 #include "compositor/meta-plugin-manager.h"
@@ -77,6 +78,9 @@ typedef struct _MetaContextPrivate
 
   GMainLoop *main_loop;
   GError *termination_error;
+#ifdef RLIMIT_NOFILE
+  struct rlimit saved_rlimit_nofile;
+#endif
 } MetaContextPrivate;
 
 G_DEFINE_TYPE_WITH_PRIVATE (MetaContext, meta_context, G_TYPE_OBJECT)
@@ -492,6 +496,108 @@ meta_context_set_unsafe_mode (MetaContext *context,
   g_object_notify_by_pspec (G_OBJECT (context), obj_props[PROP_UNSAFE_MODE]);
 }
 
+static gboolean
+meta_context_save_rlimit_nofile (MetaContext  *context,
+                                 GError      **error)
+{
+#ifdef RLIMIT_NOFILE
+  MetaContextPrivate *priv = meta_context_get_instance_private (context);
+
+  if (getrlimit (RLIMIT_NOFILE, &priv->saved_rlimit_nofile) != 0)
+    {
+      priv->saved_rlimit_nofile.rlim_cur = 0;
+      priv->saved_rlimit_nofile.rlim_max = 0;
+      g_set_error (error, G_FILE_ERROR, g_file_error_from_errno (errno),
+                   "getrlimit failed: %s", g_strerror (errno));
+      return FALSE;
+    }
+
+  return TRUE;
+#else
+  g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_NOSYS,
+               "Missing support for RLIMIT_NOFILE");
+
+  return FALSE;
+#endif
+}
+
+/**
+ * meta_context_raise_rlimit_nofile:
+ * @context: a #MetaContext
+ * @error: a return location for errors
+ *
+ * Raises the RLIMIT_NOFILE limit value to the hard limit.
+ */
+gboolean
+meta_context_raise_rlimit_nofile (MetaContext  *context,
+                                  GError      **error)
+{
+#ifdef RLIMIT_NOFILE
+  struct rlimit new_rlimit;
+
+  if (getrlimit (RLIMIT_NOFILE, &new_rlimit) != 0)
+    {
+      g_set_error (error, G_FILE_ERROR, g_file_error_from_errno (errno),
+                   "getrlimit failed: %s", g_strerror (errno));
+      return FALSE;
+    }
+
+  /* Raise the rlimit_nofile value to the hard limit */
+  new_rlimit.rlim_cur = new_rlimit.rlim_max;
+
+  if (setrlimit (RLIMIT_NOFILE, &new_rlimit) != 0)
+    {
+      g_set_error (error, G_FILE_ERROR, g_file_error_from_errno (errno),
+                   "setrlimit failed: %s", g_strerror (errno));
+      return FALSE;
+    }
+
+  return TRUE;
+#else
+  g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_NOSYS,
+               "Missing support for RLIMIT_NOFILE");
+
+  return FALSE;
+#endif
+}
+
+/**
+ * meta_context_restore_rlimit_nofile:
+ * @context: a #MetaContext
+ * @error: a return location for errors
+ *
+ * Restores the RLIMIT_NOFILE limits from when the #MetaContext was created.
+ */
+gboolean
+meta_context_restore_rlimit_nofile (MetaContext  *context,
+                                    GError      **error)
+{
+#ifdef RLIMIT_NOFILE
+  MetaContextPrivate *priv = meta_context_get_instance_private (context);
+
+  if (priv->saved_rlimit_nofile.rlim_cur == 0)
+    {
+      g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_NOENT,
+                   "RLIMIT_NOFILE not saved");
+      return FALSE;
+    }
+
+  if (setrlimit (RLIMIT_NOFILE, &priv->saved_rlimit_nofile) != 0)
+    {
+      g_set_error (error, G_FILE_ERROR, g_file_error_from_errno (errno),
+                   "setrlimit failed: %s", g_strerror (errno));
+      return FALSE;
+    }
+
+  return TRUE;
+#else
+  g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_NOSYS,
+               "Missing support for RLIMIT_NOFILE");
+
+  return FALSE;
+#endif
+}
+
 static void
 meta_context_get_property (GObject    *object,
                            guint       prop_id,
@@ -618,6 +724,7 @@ static void
 meta_context_init (MetaContext *context)
 {
   MetaContextPrivate *priv = meta_context_get_instance_private (context);
+  g_autoptr (GError) error = NULL;
 
   priv->plugin_gtype = G_TYPE_NONE;
   priv->gnome_wm_keybindings = g_strdup ("Mutter");
@@ -631,4 +738,10 @@ meta_context_init (MetaContext *context)
   g_option_context_set_main_group (priv->option_context,
                                    g_option_group_new (NULL, NULL, NULL,
                                                        context, NULL));
+
+  if (!meta_context_save_rlimit_nofile (context, &error))
+    {
+      if (!g_error_matches (error, G_FILE_ERROR, G_FILE_ERROR_NOSYS))
+        g_warning ("Failed to save the nofile limit: %s", error->message);
+    }
 }
diff --git a/src/meta/meta-context.h b/src/meta/meta-context.h
index 50b4bd5f94..6c7d6d5b95 100644
--- a/src/meta/meta-context.h
+++ b/src/meta/meta-context.h
@@ -98,4 +98,12 @@ MetaBackend * meta_context_get_backend (MetaContext *context);
 META_EXPORT
 MetaDisplay * meta_context_get_display (MetaContext *context);
 
+META_EXPORT
+gboolean meta_context_raise_rlimit_nofile (MetaContext  *context,
+                                           GError      **error);
+
+META_EXPORT
+gboolean meta_context_restore_rlimit_nofile (MetaContext  *context,
+                                             GError      **error);
+
 #endif /* META_CONTEXT_H */