overhaul system() and popen() to use vfork; fix various related bugs

since we target systems without overcommit, special care should be
taken that system() and popen(), like posix_spawn(), do not fail in
processes whose commit charges are too high to allow ordinary forking.

this in turn requires special precautions to ensure that the parent
process's signal handlers do not end up running in the shared-memory
child, where they could corrupt the state of the parent process.

popen has also been updated to use pipe2, so it does not have a
fd-leak race in multi-threaded programs. since pipe2 is missing on
older kernels, (non-atomic) emulation has been added.

some silly bugs in the old code should be gone too.

4 files changed, 110 insertions, 56 deletions

diff --git a/src/process/posix_spawn.c b/src/process/posix_spawn.c
index 8a6ff6d..e855748 100644
--- a/src/process/posix_spawn.c
+++ b/src/process/posix_spawn.c
@@ -4,6 +4,7 @@
 #include <stdint.h>
 #include <fcntl.h>
 #include "syscall.h"
+#include "pthread_impl.h"
 #include "fdop.h"
 #include "libc.h"
 
@@ -30,7 +31,7 @@ int __posix_spawnx(pid_t *restrict res, const char *restrict path,
 
 	if (!attr) attr = &dummy_attr;
 
-	sigprocmask(SIG_BLOCK, (void *)(uint64_t []){-1}, &oldmask);
+	sigprocmask(SIG_BLOCK, SIGALL_SET, &oldmask);
 
 	__acquire_ptc();
 	pid = __vfork();
@@ -43,14 +44,14 @@ int __posix_spawnx(pid_t *restrict res, const char *restrict path,
 		return 0;
 	}
 
-	for (i=1; i<=64; i++) {
+	for (i=1; i<=8*__SYSCALL_SSLEN; i++) {
 		struct sigaction sa;
-		sigaction(i, 0, &sa);
-		if (sa.sa_handler!=SIG_IGN ||
+		__libc_sigaction(i, 0, &sa);
+		if (sa.sa_handler!=SIG_DFL && (sa.sa_handler!=SIG_IGN ||
 		    ((attr->__flags & POSIX_SPAWN_SETSIGDEF)
-		     && sigismember(&attr->__def, i) )) {
+		     && sigismember(&attr->__def, i) ))) {
 			sa.sa_handler = SIG_DFL;
-			sigaction(i, &sa, 0);
+			__libc_sigaction(i, &sa, 0);
 		}
 	}
 
diff --git a/src/process/system.c b/src/process/system.c
index 0f1c07b..c8f2600 100644
--- a/src/process/system.c
+++ b/src/process/system.c
@@ -3,43 +3,62 @@
 #include <signal.h>
 #include <sys/wait.h>
 #include <errno.h>
+#include "pthread_impl.h"
+#include "libc.h"
+
+static void dummy_0()
+{
+}
+weak_alias(dummy_0, __acquire_ptc);
+weak_alias(dummy_0, __release_ptc);
+
+pid_t __vfork(void);
 
 int system(const char *cmd)
 {
 	pid_t pid;
-	sigset_t old, new;
-	struct sigaction sa, oldint, oldquit;
-	int status;
+	sigset_t old;
+	struct sigaction sa = { .sa_handler = SIG_IGN }, oldint, oldquit;
+	int status = -1, i;
 
 	if (!cmd) return 1;
 
-	sa.sa_handler = SIG_IGN;
-	sigemptyset(&sa.sa_mask);
-	sa.sa_flags = 0;
-
 	sigaction(SIGINT, &sa, &oldint);
 	sigaction(SIGQUIT, &sa, &oldquit);
-	sigaddset(&sa.sa_mask, SIGCHLD);
-	sigprocmask(SIG_BLOCK, &new, &old);
+	sigprocmask(SIG_BLOCK, SIGALL_SET, &old);
+
+	__acquire_ptc();
+	pid = __vfork();
+	__release_ptc();
 
-	pid = fork();
-	if (pid <= 0) {
+	if (pid > 0) {
+		sigset_t new = old;
+		sigaddset(&new, SIGCHLD);
+		sigprocmask(SIG_BLOCK, &new, 0);
+		while (waitpid(pid, &status, 0) && errno == EINTR);
+	}
+
+	if (pid) {
 		sigaction(SIGINT, &oldint, NULL);
 		sigaction(SIGQUIT, &oldquit, NULL);
 		sigprocmask(SIG_SETMASK, &old, NULL);
-		if (pid == 0) {
-			execl("/bin/sh", "sh", "-c", cmd, (char *)0);
-			_exit(127);
-		}
-		return -1;
+		return status;
 	}
-	while (waitpid(pid, &status, 0) == -1)
-		if (errno != EINTR) {
-			status = -1;
-			break;
+
+	/* Before we can unblock signals in the child, all signal
+	 * handlers must be eliminated -- even implementation-internal
+	 * ones. Otherwise, a signal handler could run in the child
+	 * and clobber the parent's memory (due to vfork). */
+	for (i=1; i<=8*__SYSCALL_SSLEN; i++) {
+		struct sigaction sa;
+		__libc_sigaction(i, 0, &sa);
+		if (sa.sa_handler!=SIG_IGN && sa.sa_handler!=SIG_DFL) {
+			sa.sa_handler = SIG_DFL;
+			__libc_sigaction(i, &sa, 0);
 		}
-	sigaction(SIGINT, &oldint, NULL);
-	sigaction(SIGQUIT, &oldquit, NULL);
+	}
+
 	sigprocmask(SIG_SETMASK, &old, NULL);
-	return status;
+	execl("/bin/sh", "sh", "-c", cmd, (char *)0);
+	_exit(127);
 }
diff --git a/src/stdio/popen.c b/src/stdio/popen.c
index 4f9d6e9..0c9f24e 100644
--- a/src/stdio/popen.c
+++ b/src/stdio/popen.c
@@ -1,18 +1,22 @@
+#include <fcntl.h>
 #include "stdio_impl.h"
+#include "pthread_impl.h"
 #include "syscall.h"
 
-static inline void nc_close(int fd)
+static void dummy_0()
 {
-	__syscall(SYS_close, fd);
 }
-#define close(x) nc_close(x)
+weak_alias(dummy_0, __acquire_ptc);
+weak_alias(dummy_0, __release_ptc);
+
+pid_t __vfork(void);
 
 FILE *popen(const char *cmd, const char *mode)
 {
-	int p[2];
-	int op;
+	int p[2], op, i;
 	pid_t pid;
 	FILE *f;
+	sigset_t old;
 	const char *modes = "rw", *mi = strchr(modes, *mode);
 
 	if (mi) {
@@ -22,29 +26,45 @@ FILE *popen(const char *cmd, const char *mode)
 		return 0;
 	}
 	
-	if (pipe(p)) return NULL;
+	if (pipe2(p, O_CLOEXEC)) return NULL;
 	f = fdopen(p[op], mode);
 	if (!f) {
-		close(p[0]);
-		close(p[1]);
+		__syscall(SYS_close, p[0]);
+		__syscall(SYS_close, p[1]);
 		return NULL;
 	}
+
+	sigprocmask(SIG_BLOCK, SIGALL_SET, &old);
 	
-	pid = fork();
-	switch (pid) {
-	case -1:
-		fclose(f);
-		close(p[0]);
-		close(p[1]);
-		return NULL;
-	case 0:
-		if (dup2(p[1-op], 1-op) < 0) _exit(127);
-		if (p[0] != 1-op) close(p[0]);
-		if (p[1] != 1-op) close(p[1]);
-		execl("/bin/sh", "sh", "-c", cmd, (char *)0);
-		_exit(127);
+	__acquire_ptc();
+	pid = __vfork();
+	__release_ptc();
+
+	if (pid) {
+		__syscall(SYS_close, p[1-op]);
+		sigprocmask(SIG_BLOCK, SIGALL_SET, &old);
+		if (pid < 0) {
+			fclose(f);
+			return 0;
+		}
+		f->pipe_pid = pid;
+		return f;
+	}
+
+	/* See notes in system.c for why this is needed. */
+	for (i=1; i<=8*__SYSCALL_SSLEN; i++) {
+		struct sigaction sa;
+		__libc_sigaction(i, 0, &sa);
+		if (sa.sa_handler!=SIG_IGN && sa.sa_handler!=SIG_DFL) {
+			sa.sa_handler = SIG_DFL;
+			__libc_sigaction(i, &sa, 0);
+		}
 	}
-	close(p[1-op]);
-	f->pipe_pid = pid;
-	return f;
+	if (dup2(p[1-op], 1-op) < 0) _exit(127);
+	fcntl(1-op, F_SETFD, 0);
+	if (p[0] != 1-op) __syscall(SYS_close, p[0]);
+	if (p[1] != 1-op) __syscall(SYS_close, p[1]);
+	sigprocmask(SIG_SETMASK, &old, 0);
+	execl("/bin/sh", "sh", "-c", cmd, (char *)0);
+	_exit(127);
 }
diff --git a/src/unistd/pipe2.c b/src/unistd/pipe2.c
index 83282bb..04e0c12 100644
--- a/src/unistd/pipe2.c
+++ b/src/unistd/pipe2.c
@@ -1,8 +1,22 @@
-#define _GNU_SOURCE
 #include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
 #include "syscall.h"
 
-int pipe2(int fd[2], int flg)
+int pipe2(int fd[2], int flag)
 {
-	return syscall(SYS_pipe2, fd, flg);
+	if (!flag) return syscall(SYS_pipe, fd);
+	int ret = __syscall(SYS_pipe2, fd, flag);
+	if (ret != -ENOSYS) return __syscall_ret(ret);
+	ret = syscall(SYS_pipe, fd);
+	if (ret) return __syscall_ret(ret);
+	if (flag & O_CLOEXEC) {
+		fcntl(fd[0], F_SETFD, FD_CLOEXEC);
+		fcntl(fd[1], F_SETFD, FD_CLOEXEC);
+	}
+	if (flag & O_NONBLOCK) {
+		fcntl(fd[0], F_SETFL, fcntl(fd[0], F_GETFL) | O_NONBLOCK);
+		fcntl(fd[1], F_SETFL, fcntl(fd[1], F_GETFL) | O_NONBLOCK);
+	}
+	return 0;
 }