- (dtucker) Bug #536: Test for and work around openpty/controlling tty

problem on Linux (fixes "could not set controlling tty" errors).

Also renames STREAMS_PUSH_ACQUIRES_CTTY to the more generic SSHD_ACQUIRES_CTTY
and moves the Solaris-specific comments to configure.ac.

ChangeLog

@@ -1,6 +1,8 @@
 20030510
  - (dtucker) Bug #318: Create ssh_prng_cmds.out during "make" rather than
    "make install".  Patch by roth@feep.net.
+ - (dtucker) Bug #536: Test for and work around openpty/controlling tty
+   problem on Linux (fixes "could not set controlling tty" errors).
 
 20030504
  - (dtucker) Bug #497: Move #include of bsd-cygwin_util.h to openbsd-compat.h.
@@ -1374,4 +1376,4 @@
      save auth method before monitor_reset_key_state(); bugzilla bug #284;
      ok provos@
 
-$Id: ChangeLog,v 1.2671 2003/05/10 06:48:23 dtucker Exp $
+$Id: ChangeLog,v 1.2672 2003/05/10 07:05:46 dtucker Exp $

acconfig.h

@@ -1,4 +1,4 @@
-/* $Id: acconfig.h,v 1.150 2003/05/02 10:48:21 dtucker Exp $ */
+/* $Id: acconfig.h,v 1.151 2003/05/10 07:05:46 dtucker Exp $ */
 
 #ifndef _CONFIG_H
 #define _CONFIG_H
@@ -374,8 +374,8 @@
 /* Some systems put this outside of libc */
 #undef HAVE_NANOSLEEP
 
-/* Pushing STREAMS modules incorrectly acquires a controlling TTY */
-#undef STREAMS_PUSH_ACQUIRES_CTTY
+/* Define if sshd somehow reacquires a controlling TTY after setsid() */
+#undef SSHD_ACQUIRES_CTTY
 
 /* Define if cmsg_type is not passed correctly */
 #undef BROKEN_CMSG_TYPE

configure.ac

@@ -1,4 +1,4 @@
-# $Id: configure.ac,v 1.115 2003/05/02 10:48:21 dtucker Exp $
+# $Id: configure.ac,v 1.116 2003/05/10 07:05:46 dtucker Exp $
 
 AC_INIT
 AC_CONFIG_SRCDIR([ssh.c])
@@ -188,15 +188,56 @@ main() { if (NSVersionOfRunTimeLibrary("System") >= (60 << 16))
 	AC_DEFINE(SETPROCTITLE_STRATEGY,PS_USE_CLOBBER_ARGV)
 	AC_DEFINE(SETPROCTITLE_PS_PADDING, '\0')
 	inet6_default_4in6=yes
-	AC_MSG_CHECKING(for broken cmsg_type)
+	# openpty can be in libutil, needed for controlling tty test
+	AC_SEARCH_LIBS(openpty, util)
+	# make sure that openpty does not reacquire controlling terminal
+	AC_MSG_CHECKING(if openpty correctly handles controlling tty)
+	AC_TRY_RUN(
+		[
+#include <stdio.h>
+#include <sys/fcntl.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+
+int
+main()
+{
+	pid_t pid;
+	int fd, ptyfd, ttyfd, status;
+
+	pid = fork();
+	if (pid < 0) {		/* failed */
+		exit(1);
+	} else if (pid > 0) {	/* parent */
+		waitpid(pid, &status, 0);
+		if (WIFEXITED(status)) 
+			exit(WEXITSTATUS(status));
+		else
+			exit(2);
+	} else {		/* child */
+		close(0); close(1); close(2);
+		setsid();
+		openpty(&ptyfd, &ttyfd, NULL, NULL, NULL);
+		fd = open("/dev/tty", O_RDWR | O_NOCTTY);
+		if (fd >= 0)
+			exit(3);	/* Acquired ctty: broken */
+		else
+			exit(0);	/* Did not acquire ctty: OK */
+	}
+}
+		],
+		[
+			AC_MSG_RESULT(yes)
+		],
+		[
+			AC_MSG_RESULT(no)
+			AC_DEFINE(SSHD_ACQUIRES_CTTY)
+		]
+	)
 	case `uname -r` in
-	2.0.*)
-		AC_MSG_RESULT(yes)
+	1.*|2.0.*)
 		AC_DEFINE(BROKEN_CMSG_TYPE)
 		;;
-	*)
-		AC_MSG_RESULT(no)
-		;;
 	esac
 	;;
 mips-sony-bsd|mips-sony-newsos4)
@@ -230,7 +271,8 @@ mips-sony-bsd|mips-sony-newsos4)
 	AC_DEFINE(LOGIN_NEEDS_UTMPX)
 	AC_DEFINE(LOGIN_NEEDS_TERM)
 	AC_DEFINE(PAM_TTY_KLUDGE)
-	AC_DEFINE(STREAMS_PUSH_ACQUIRES_CTTY)
+	# Pushing STREAMS modules will cause sshd to acquire a controlling tty.
+	AC_DEFINE(SSHD_ACQUIRES_CTTY)
 	# hardwire lastlog location (can't detect it on some versions)
 	conf_lastlog_location="/var/adm/lastlog"
 	AC_MSG_CHECKING(for obsolete utmp and wtmp in solaris2.x)

sshd.c

@@ -1400,11 +1400,11 @@ main(int ac, char **av)
 	 * setlogin() affects the entire process group.  We don't
 	 * want the child to be able to affect the parent.
 	 */
-#if !defined(STREAMS_PUSH_ACQUIRES_CTTY)
+#if !defined(SSHD_ACQUIRES_CTTY)
 	/*
-	 * If setsid is called on Solaris, sshd will acquire the controlling
-	 * terminal while pushing STREAMS modules. This will prevent the
-	 * shell from acquiring it later.
+	 * If setsid is called, on some platforms sshd will later acquire a
+	 * controlling terminal which will result in "could not set
+	 * controlling tty" errors.
 	 */
 	if (!debug_flag && !inetd_flag && setsid() < 0)
 		error("setsid: %.100s", strerror(errno));