community/vte3: fix deadlock by using alloca

use alloca instead of malloc.

upstream bugs:
https://gitlab.gnome.org/GNOME/vte/-/issues/271
https://gitlab.gnome.org/GNOME/vte/-/issues/263

fixes #11819

2 files changed, 143 insertions, 2 deletions

diff --git a/community/vte3/APKBUILD b/community/vte3/APKBUILD
index ade191d96ad..550109c2935 100644
--- a/community/vte3/APKBUILD
+++ b/community/vte3/APKBUILD
@@ -2,7 +2,7 @@
 # Maintainer: Rasmus Thomsen <oss@cogitri.dev>
 pkgname=vte3
 pkgver=0.60.3
-pkgrel=1
+pkgrel=2
 pkgdesc="Virtual Terminal Emulator library"
 url="https://developer.gnome.org/vte/"
 arch="all"
@@ -42,4 +42,4 @@ package() {
 sha512sums="3694fe711e0b3eb9d6ba37ad8036f5d3cca4265635ed7afcde750a8445b17f820d1c55b557d0ea1c8a5a45e5408915d8da2ffd65b4d397c6582f288812ae1f18  vte-0.60.3.tar.xz
 94de7160b71f1c41aa19e8f9dc4538cd493da1a33dda6482c332f8e8932213631ccc2355506a616772dae451d653f035c66c721bd839570d6cb55e4d2b12e8a3  fix-W_EXITCODE.patch
 09caf90f7c280e484c40b4205d17b3921848dfb211b15f4126cca980eb7598c498dc0fd81fb38ca30c9f0bdab44fcb532819eab09b54d96badcffbf80695bb9b  vte-ng.patch
-1e82a3efcca13314eb99b653703ac44c3c12c4657cf671ae0b9636685fdc5343cc8ccd230cc10299dc08a87fd03aea8fad3a1c03feedab58f00fb68d44449575  no-setenv-after-fork.patch"
+3e9e6394ac7e7aff8ecd6060dcf0cd0d1bf772a1417b5a1d49ad4685465d4c056a0316e54ec13a5891ea0462e5815da02e3791246c1420ac3e456d9df6f1a70f  no-setenv-after-fork.patch"
diff --git a/community/vte3/no-setenv-after-fork.patch b/community/vte3/no-setenv-after-fork.patch
index 06c609e9138..1c6dc615707 100644
--- a/community/vte3/no-setenv-after-fork.patch
+++ b/community/vte3/no-setenv-after-fork.patch
@@ -20,3 +20,144 @@ index 3db1a7c..3c97f6c 100644
  	/* Finally call an extra child setup */
  	if (m_extra_child_setup.func) {
  		m_extra_child_setup.func(m_extra_child_setup.data);
+diff --git a/src/vtespawn.cc b/src/vtespawn.cc
+index df7ff7f0..23a7fd29 100644
+--- a/src/vtespawn.cc
++++ b/src/vtespawn.cc
+@@ -445,7 +445,7 @@ filename_to_fd (const char *p)
+ 
+   while ((c = *p++) != '\0')
+     {
+-      if (!g_ascii_isdigit (c))
++      if (c < '0' || c > '9')
+         return -1;
+       c -= '0';
+ 
+@@ -460,6 +460,53 @@ filename_to_fd (const char *p)
+ }
+ #endif
+ 
++/* This function is called between fork and execve/_exit and so must be
++ * async-signal-safe; see man:signal-safety(7).
++ */
++static int
++getrlimit_NOFILE_max(void)
++{
++#ifdef HAVE_SYS_RESOURCE_H
++        struct rlimit rlim;
++
++#ifdef __linux__
++        if (prlimit(0 /* this PID */, RLIMIT_NOFILE, nullptr, &rlim) == 0 &&
++            rlim.rlim_max != RLIM_INFINITY)
++                return rlim.rlim_max;
++
++        /* fallback */
++#endif /* __linux__ */
++
++#ifdef __GLIBC__
++        /* Use getrlimit() function provided by the system if it is known to be
++         * async-signal safe.
++         *
++         * According to the glibc manual, getrlimit is AS-safe.
++         */
++        if (getrlimit(RLIMIT_NOFILE, &rlim) == 0 &&
++            rlim.rlim_max != RLIM_INFINITY)
++                return rlim.rlim_max;
++
++        /* fallback */
++#endif
++
++#endif /* HAVE_SYS_RESOURCE_H */
++
++#if defined(__FreeBSD__) || defined(__OpenBSD__)
++        /* Use sysconf() function provided by the system if it is known to be
++         * async-signal safe.
++         */
++        auto const r = sysconf(_SC_OPEN_MAX);
++        if (r != -1)
++                return r;
++
++        /* fallback */
++#endif
++
++        /* Hardcoded fallback: the default process hard limit in Linux as of 2020 */
++        return 4096;
++}
++
+ #ifndef HAVE_FDWALK
+ static int
+ fdwalk (int (*cb)(void *data, int fd), void *data)
+@@ -468,14 +515,9 @@ fdwalk (int (*cb)(void *data, int fd), void *data)
+    * may be slow on non-Linux operating systems, especially on systems allowing
+    * very high number of open file descriptors.
+    */
+-  gint open_max;
+   gint fd;
+   gint res = 0;
+ 
+-#ifdef HAVE_SYS_RESOURCE_H
+-  struct rlimit rl;
+-#endif
+-
+ #ifdef __linux__
+   /* Avoid use of opendir/closedir since these are not async-signal-safe. */
+   int dir_fd = open ("/proc/self/fd", O_RDONLY | O_DIRECTORY);
+@@ -509,13 +551,7 @@ fdwalk (int (*cb)(void *data, int fd), void *data)
+ 
+ #endif
+ 
+-#ifdef HAVE_SYS_RESOURCE_H
+-
+-  if (getrlimit(RLIMIT_NOFILE, &rl) == 0 && rl.rlim_max != RLIM_INFINITY)
+-      open_max = rl.rlim_max;
+-  else
+-#endif
+-      open_max = sysconf (_SC_OPEN_MAX);
++  auto const open_max = getrlimit_NOFILE_max();
+ 
+   for (fd = 0; fd < open_max; fd++)
+       if ((res = cb (data, fd)) != 0)
+@@ -1071,7 +1107,7 @@ script_execute (const gchar *file,
+   {
+     gchar **new_argv;
+ 
+-    new_argv = g_new0 (gchar*, argc + 2); /* /bin/sh and NULL */
++    new_argv = g_newa (gchar*, argc + 2); /* /bin/sh and NULL */
+     
+     new_argv[0] = (char *) "/bin/sh";
+     new_argv[1] = (char *) file;
+@@ -1086,8 +1122,6 @@ script_execute (const gchar *file,
+       execve (new_argv[0], new_argv, envp);
+     else
+       execv (new_argv[0], new_argv);
+-    
+-    g_free (new_argv);
+   }
+ }
+ 
+@@ -1147,7 +1181,7 @@ g_execute (const gchar *file,
+ 
+       len = strlen (file) + 1;
+       pathlen = strlen (path);
+-      freeme = name = (char*)g_malloc (pathlen + len + 1);
++      name = (char*)g_alloca (pathlen + len + 1);
+       
+       /* Copy the file name at the top, including '\0'  */
+       memcpy (name + pathlen + 1, file, len);
+@@ -1217,7 +1251,6 @@ g_execute (const gchar *file,
+                * something went wrong executing it; return the error to our
+                * caller.
+                */
+-              g_free (freeme);
+ 	      return -1;
+ 	    }
+ 	}
+@@ -1229,8 +1262,6 @@ g_execute (const gchar *file,
+          * error.
+          */
+         errno = EACCES;
+-
+-      g_free (freeme);
+     }
+ 
+   /* Return the error from the last attempt (probably ENOENT).  */