From 4b522a02c31d8a3010bdfd1924a38a8f6855457a Mon Sep 17 00:00:00 2001
From: Daniel Drake <drake@endlessm.com>
Date: Wed, 6 Jun 2018 07:07:17 -0600
Subject: [PATCH] shell-app: remove child_setup from app launching

When the amount of free memory on the system is somewhat low, gnome-shell
will sometimes fail to launch apps, reporting the error:
  fork(): Cannot allocate memory

fork() is failing here because while cloning the process virtual address
space, Linux worries that the thread being forked may end up COWing the
entire address space of the parent process (gnome-shell, which is
memory-hungry), and there is not enough free memory to permit that to
happen. This check is somewhat irrelevant because we are only forking
to immediately exec(), which will discard the whole virtual address
space anyway.

This issue can be avoided by using a new optimized gspawn codepath in
the latest glib development version, which uses posix_spawn() internally.

For the optimized codepath to be used, we must not pass a child_setup
function, so the the file descriptor management is reimplemented here
using new glib API to pass fds to the child process. The old API will
continue to be used on older glib versions.

We must also change the spawn flags for this code path to be hit.
I checked that gnome-shell's open file descriptors are all CLOEXEC
so using G_SPAWN_LEAVE_DESCRIPTORS_OPEN should be safe.

This will result in more resilient app launching when memory is low,
since the optimized spawn path avoids cloning the virtual address
space of the parent process (gnome-shell) and avoids the irrelevant
memory overcommit check.

https://gitlab.gnome.org/GNOME/gnome-shell/merge_requests/132
---
 config.h.meson  |  3 +++
 meson.build     |  5 +++++
 src/shell-app.c | 36 ++++++++++++++++++++++++++++++++++--
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/config.h.meson b/config.h.meson
index 10cadf98b..0dac17b94 100644
--- a/config.h.meson
+++ b/config.h.meson
@@ -24,3 +24,6 @@
 
 /* Define if _NL_TIME_FIRST_WEEKDATE is available */
 #mesondefine HAVE__NL_TIME_FIRST_WEEKDAY
+
+/* Define if you have the `g_desktop_app_info_launch_uris_as_manager_with_fds` function */
+#mesondefine HAVE_GIO_DESKTOP_LAUNCH_URIS_WITH_FDS
diff --git a/meson.build b/meson.build
index 65f6c7f1f..5750b4fec 100644
--- a/meson.build
+++ b/meson.build
@@ -161,6 +161,11 @@ cdata.set_quoted('PACKAGE_VERSION', meson.project_version())
 cdata.set('HAVE_NETWORKMANAGER', have_networkmanager)
 cdata.set('HAVE_SYSTEMD', have_systemd)
 
+# New API added in glib-2.57.2
+cdata.set('HAVE_GIO_DESKTOP_LAUNCH_URIS_WITH_FDS',
+          cc.has_function('g_desktop_app_info_launch_uris_as_manager_with_fds',
+                          dependencies : gio_dep)
+)
 cdata.set('HAVE_FDWALK', cc.has_function('fdwalk'))
 cdata.set('HAVE_MALLINFO', cc.has_function('mallinfo'))
 cdata.set('HAVE_SYS_RESOURCE_H', cc.has_header('sys/resource.h'))
diff --git a/src/shell-app.c b/src/shell-app.c
index 6b726514a..bb3f0b171 100644
--- a/src/shell-app.c
+++ b/src/shell-app.c
@@ -1207,7 +1207,7 @@ shell_app_request_quit (ShellApp   *app)
   return TRUE;
 }
 
-#ifdef HAVE_SYSTEMD
+#if !defined(HAVE_GIO_DESKTOP_LAUNCH_URIS_WITH_FDS) && defined(HAVE_SYSTEMD)
 /* This sets up the launched application to log to the journal
  * using its own identifier, instead of just "gnome-session".
  */
@@ -1255,6 +1255,7 @@ shell_app_launch (ShellApp     *app,
   ShellGlobal *global;
   GAppLaunchContext *context;
   gboolean ret;
+  GSpawnFlags flags;
 
   if (app->info == NULL)
     {
@@ -1274,9 +1275,39 @@ shell_app_launch (ShellApp     *app,
   if (discrete_gpu)
     g_app_launch_context_setenv (context, "DRI_PRIME", "1");
 
+  /* Set LEAVE_DESCRIPTORS_OPEN in order to use an optimized gspawn
+   * codepath. The shell's open file descriptors should be marked CLOEXEC
+   * so that they are automatically closed even with this flag set.
+   */
+  flags = G_SPAWN_SEARCH_PATH | G_SPAWN_DO_NOT_REAP_CHILD |
+          G_SPAWN_LEAVE_DESCRIPTORS_OPEN;
+
+#ifdef HAVE_GIO_DESKTOP_LAUNCH_URIS_WITH_FDS
+  /* Optimized spawn path, avoiding a child_setup function */
+  {
+    int journalfd = -1;
+
+#ifdef HAVE_SYSTEMD
+    journalfd = sd_journal_stream_fd (shell_app_get_id (app), LOG_INFO, FALSE);
+#endif /* HAVE_SYSTEMD */
+
+    ret = g_desktop_app_info_launch_uris_as_manager_with_fds (app->info, NULL,
+                                                              context,
+                                                              flags,
+                                                              NULL, NULL,
+                                                              wait_pid, NULL,
+                                                              -1,
+                                                              journalfd,
+                                                              journalfd,
+                                                              error);
+
+    if (journalfd >= 0)
+      (void) close (journalfd);
+  }
+#else /* !HAVE_GIO_DESKTOP_LAUNCH_URIS_WITH_FDS */
   ret = g_desktop_app_info_launch_uris_as_manager (app->info, NULL,
                                                    context,
-                                                   G_SPAWN_SEARCH_PATH | G_SPAWN_DO_NOT_REAP_CHILD,
+                                                   flags,
 #ifdef HAVE_SYSTEMD
                                                    app_child_setup, (gpointer)shell_app_get_id (app),
 #else
@@ -1284,6 +1315,7 @@ shell_app_launch (ShellApp     *app,
 #endif
                                                    wait_pid, NULL,
                                                    error);
+#endif /* HAVE_GIO_DESKTOP_LAUNCH_URIS_WITH_FDS */
   g_object_unref (context);
 
   return ret;