* fhandler.h (fhandler_pipe::fixup_in_child): Declare new function.

(fhandler_console::invisible_console): Declare new variable.
(fhandler_console::need_invisible): Ditto.
(fhandler_console::has_a): Ditto.
* fhandler_console.cc (set_console_state_for_spawn): Eliminate return value.
Set up an invisible console if necessary prior to spawning.
(fhandler_console::invisible_console): Define.
* fhandler_tty.cc (fhandler_tty_slave::open): Use
fhandler_console::invisible_console to setup an invisible console.
* pipe.cc (fhandler_pipe::fixup_in_child): Define new function from
fixup_after_exec.
(fhandler_pipe::fixup_after_exec): Use fixup_in_child when appropriate.
(fhandler_pipe::fixup_after_fork): Ditto.
* spawn.cc (handle): Reorganize and modernize a little.
(spawn_guts): Rely on set_console_state_for_spawn to set the console into the
right state but don't create the process with "detached" flag if we have no
controlling tty since that confuses 'cmd'.
* dtable.cc (dtable::stdio_init): Don't set console as controlling terminal if
we have an invisible console.
* sigproc.cc (child_info::sync): Use correct name in ForceCloseHandle1.

8 files changed, 120 insertions, 65 deletions

diff --git a/winsup/cygwin/ChangeLog b/winsup/cygwin/ChangeLog
index f29d97e..14d22a8 100644
--- a/winsup/cygwin/ChangeLog
+++ b/winsup/cygwin/ChangeLog
@@ -1,5 +1,29 @@
 2005-12-18  Christopher Faylor  <cgf@timesys.com>
 
+	* fhandler.h (fhandler_pipe::fixup_in_child): Declare new function.
+	(fhandler_console::invisible_console): Declare new variable.
+	(fhandler_console::need_invisible): Ditto.
+	(fhandler_console::has_a): Ditto.
+	* fhandler_console.cc (set_console_state_for_spawn): Eliminate return
+	value.  Set up an invisible console if necessary prior to spawning.
+	(fhandler_console::invisible_console): Define.
+	* fhandler_tty.cc (fhandler_tty_slave::open): Use
+	fhandler_console::invisible_console to setup an invisible console.
+	* pipe.cc (fhandler_pipe::fixup_in_child): Define new function from
+	fixup_after_exec.
+	(fhandler_pipe::fixup_after_exec): Use fixup_in_child when appropriate.
+	(fhandler_pipe::fixup_after_fork): Ditto.
+	* spawn.cc (handle): Reorganize and modernize a little.
+	(spawn_guts): Rely on set_console_state_for_spawn to set the console
+	into the right state but don't create the process with "detached" flag
+	if we have no controlling tty since that confuses 'cmd'.
+	* dtable.cc (dtable::stdio_init): Don't set console as controlling
+	terminal if we have an invisible console.
+
+	* sigproc.cc (child_info::sync): Use correct name in ForceCloseHandle1.
+
+2005-12-18  Christopher Faylor  <cgf@timesys.com>
+
 	* include/sys/dirent.h: Change __deprecated_d_ino to __invalid_d_ino
 	throughout to make things a little clearer.
 	* dir.cc (readdir_worker): Reflect above change.
diff --git a/winsup/cygwin/dtable.cc b/winsup/cygwin/dtable.cc
index 08832a6..31b1198 100644
--- a/winsup/cygwin/dtable.cc
+++ b/winsup/cygwin/dtable.cc
@@ -160,9 +160,10 @@ dtable::stdio_init ()
 
   init_std_file_from_handle (1, out);
   init_std_file_from_handle (2, err);
+
   /* Assign the console as the controlling tty for this process if we actually
      have a console and no other controlling tty has been assigned. */
-  if (myself->ctty < 0 && GetConsoleCP () > 0)
+  if (!fhandler_console::need_invisible () && myself->ctty < 0)
     set_console_ctty ();
 }
 
diff --git a/winsup/cygwin/fhandler.h b/winsup/cygwin/fhandler.h
index defb288..e948403 100644
--- a/winsup/cygwin/fhandler.h
+++ b/winsup/cygwin/fhandler.h
@@ -515,6 +515,7 @@ public:
   void create_guard (SECURITY_ATTRIBUTES *sa) {guard = CreateMutex (sa, FALSE, NULL);}
   int dup (fhandler_base *child);
   int ioctl (unsigned int cmd, void *);
+  void fixup_in_child ();
   virtual void fixup_after_fork (HANDLE);
   void fixup_after_exec ();
   bool hit_eof ();
@@ -864,6 +865,7 @@ class fhandler_console: public fhandler_termios
 {
  private:
   static dev_console *dev_state;
+  static bool invisible_console;
 
 /* Output calls */
   void set_default_attr ();
@@ -917,6 +919,8 @@ class fhandler_console: public fhandler_termios
   void send_winch_maybe ();
   static tty_min *get_tty_stuff (int);
   bool is_slow () {return 1;}
+  static bool need_invisible ();
+  static bool fhandler_console::has_a () {return !invisible_console;}
 };
 
 class fhandler_tty_common: public fhandler_termios
@@ -1360,6 +1364,6 @@ class select_stuff
 		   device_specific_mailslot (0) {}
 };
 
-int __stdcall set_console_state_for_spawn ();
+void __stdcall set_console_state_for_spawn ();
 
 #endif /* _FHANDLER_H_ */
diff --git a/winsup/cygwin/fhandler_console.cc b/winsup/cygwin/fhandler_console.cc
index 496abfd..460b0dd 100644
--- a/winsup/cygwin/fhandler_console.cc
+++ b/winsup/cygwin/fhandler_console.cc
@@ -142,15 +142,18 @@ tty_list::get_tty (int n)
 /* Determine if a console is associated with this process prior to a spawn.
    If it is, then we'll return 1.  If the console has been initialized, then
    set it into a more friendly state for non-cygwin apps. */
-int __stdcall
+void __stdcall
 set_console_state_for_spawn ()
 {
+  if (fhandler_console::need_invisible ())
+    return;
+
   HANDLE h = CreateFile ("CONIN$", GENERIC_READ, FILE_SHARE_WRITE,
 			 &sec_none_nih, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
 			 NULL);
 
   if (h == INVALID_HANDLE_VALUE)
-    return 0;
+    return;
 
   if (shared_console_info != NULL)
     {
@@ -160,7 +163,8 @@ set_console_state_for_spawn ()
     }
 
   CloseHandle (h);
-  return 1;
+
+  return;
 }
 
 /* The results of GetConsoleCP() and GetConsoleOutputCP() cannot be
@@ -1800,3 +1804,57 @@ fhandler_console::fixup_after_fork_exec (bool execing)
       CloseHandle (oh);
     }
 }
+
+bool NO_COPY fhandler_console::invisible_console;
+
+bool
+fhandler_console::need_invisible ()
+{
+  BOOL b = false;
+  if (GetConsoleCP () || !wincap.pty_needs_alloc_console ())
+    invisible_console = false;
+  else
+    {
+      HWINSTA h, horig;
+      /* The intent here is to allocate an "invisible" console if we have no
+	 controlling tty or to reuse the existing console if we already have
+	 a tty.  So, first get the old windows station.  If there is no controlling
+	 terminal, create a new windows station and then set it as the current
+	 windows station.  The subsequent AllocConsole will then be allocated
+	 invisibly.  But, after doing that we have to restore any existing windows
+	 station or, strangely, characters will not be displayed in any windows
+	 drawn on the current screen.  We only do this if we have changed to
+	 a new windows station and if we had an existing windows station previously.
+	 We also close the previously opened work station even though AllocConsole
+	 is now "using" it.  This doesn't seem to cause any problems.
+
+	 Things to watch out for if you make changes in this code:
+
+	 - Flashing, black consoles showing up when you start, e.g., ssh in
+	   an xterm.
+	 - Non-displaying of characters in rxvt or xemacs if you start a
+	   process using setsid: bash -lc "setsid rxvt".  */
+
+      h = horig = GetProcessWindowStation ();
+      if (myself->ctty == -1)
+	{
+	  h = CreateWindowStation (NULL, 0, WINSTA_ALL_ACCESS, &sec_none_nih);
+	  termios_printf ("CreateWindowStation %p, %E", h);
+	  if (h)
+	    {
+	      b = SetProcessWindowStation (h);
+	      termios_printf ("SetProcessWindowStation %d, %E", b);
+	    }
+	}
+      b = AllocConsole ();	// will cause flashing if workstation
+				// stuff fails
+      debug_printf ("h (%p), horig (%p)", h, horig);
+      if (0 && horig && h && h != horig && SetProcessWindowStation (horig))
+	CloseHandle (h);
+      termios_printf ("%d = AllocConsole (), %E", b);
+      invisible_console = true;
+    }
+
+  debug_printf ("invisible_console %d", invisible_console);
+  return b;
+}
diff --git a/winsup/cygwin/fhandler_tty.cc b/winsup/cygwin/fhandler_tty.cc
index 8b7c2d5..f7afce0 100644
--- a/winsup/cygwin/fhandler_tty.cc
+++ b/winsup/cygwin/fhandler_tty.cc
@@ -576,52 +576,8 @@ fhandler_tty_slave::open (int flags, mode_t)
 
   set_open_status ();
   if (cygheap->manage_console_count ("fhandler_tty_slave::open", 1) == 1
-      && !GetConsoleCP () && !output_done_event
-      && wincap.pty_needs_alloc_console ())
-    {
-      BOOL b;
-      HWINSTA h, horig;
-      /* The intent here is to allocate an "invisible" console if we have no
-	 controlling tty or to reuse the existing console if we already have
-	 a tty.  So, first get the old windows station.  If there is no controlling
-	 terminal, create a new windows station and then set it as the current
-	 windows station.  The subsequent AllocConsole will then be allocated
-	 invisibly.  But, after doing that we have to restore any existing windows
-	 station or, strangely, characters will not be displayed in any windows
-	 drawn on the current screen.  We only do this if we have changed to
-	 a new windows station and if we had an existing windows station previously.
-	 We also close the previously opened work station even though AllocConsole
-	 is now "using" it.  This doesn't seem to cause any problems.
-
-	 Things to watch out for if you make changes in this code:
-
-	 - Flashing, black consoles showing up when you start, e.g., ssh in
-	   an xterm.
-	 - Non-displaying of characters in rxvt or xemacs if you start a
-	   process using setsid: bash -lc "setsid rxvt".  */
-
-      h = horig = GetProcessWindowStation ();
-      if (myself->ctty == -1)
-        {
-	  h = CreateWindowStation (NULL, 0, WINSTA_ALL_ACCESS, &sec_none_nih);
-	  termios_printf ("CreateWindowStation %p, %E", h);
-	  if (h)
-	    {
-	      b = SetProcessWindowStation (h);
-	      termios_printf ("SetProcessWindowStation %d, %E", b);
-	    }
-	}
-      b = AllocConsole ();	// will cause flashing if workstation
-				// stuff fails
-      if (horig && h && h != horig)
-	{
-	  SetProcessWindowStation (horig);
-	  CloseHandle (h);
-	}
-      termios_printf ("%d = AllocConsole (), %E", b);
-      if (b)
-	init_console_handler (TRUE);
-    }
+      && !output_done_event && fhandler_console::need_invisible ())
+    init_console_handler (TRUE);
 
   // FIXME: Do this better someday
   arch = (fhandler_tty_slave *) cmalloc (HEAP_ARCHETYPES, sizeof (*this));
diff --git a/winsup/cygwin/pipe.cc b/winsup/cygwin/pipe.cc
index 440e703..a01a2dc 100644
--- a/winsup/cygwin/pipe.cc
+++ b/winsup/cygwin/pipe.cc
@@ -224,7 +224,7 @@ fhandler_pipe::hit_eof ()
 }
 
 void
-fhandler_pipe::fixup_after_exec ()
+fhandler_pipe::fixup_in_child ()
 {
   if (read_state)
     {
@@ -234,6 +234,13 @@ fhandler_pipe::fixup_after_exec ()
 }
 
 void
+fhandler_pipe::fixup_after_exec ()
+{
+  if (!close_on_exec ())
+      fixup_in_child ();
+}
+
+void
 fhandler_pipe::fixup_after_fork (HANDLE parent)
 {
   fhandler_base::fixup_after_fork (parent);
@@ -241,7 +248,7 @@ fhandler_pipe::fixup_after_fork (HANDLE parent)
     fork_fixup (parent, guard, "guard");
   if (writepipe_exists)
     fork_fixup (parent, writepipe_exists, "guard");
-  fixup_after_exec ();
+  fixup_in_child ();
 }
 
 int
diff --git a/winsup/cygwin/sigproc.cc b/winsup/cygwin/sigproc.cc
index d719b33..2e65429 100644
--- a/winsup/cygwin/sigproc.cc
+++ b/winsup/cygwin/sigproc.cc
@@ -845,7 +845,7 @@ child_info::sync (pid_t pid, HANDLE& hProcess, DWORD howlong)
     {
       if (type == _PROC_EXEC && x == nsubproc_ready && myself->wr_proc_pipe)
 	{
-	  ForceCloseHandle1 (hProcess, childhProcess);
+	  ForceCloseHandle1 (hProcess, childhProc);
 	  hProcess = NULL;
 	}
       sigproc_printf ("process %d synchronized, WFMO returned %d", pid, x);
diff --git a/winsup/cygwin/spawn.cc b/winsup/cygwin/spawn.cc
index 463ec48..df5cde1 100644
--- a/winsup/cygwin/spawn.cc
+++ b/winsup/cygwin/spawn.cc
@@ -200,17 +200,20 @@ find_exec (const char *name, path_conv& buf, const char *mywinenv,
 /* Utility for spawn_guts.  */
 
 static HANDLE
-handle (int n, int direction)
+handle (int fd, int direction)
 {
-  fhandler_base *fh = cygheap->fdtab[n];
-
-  if (!fh)
-    return INVALID_HANDLE_VALUE;
-  if (fh->close_on_exec ())
-    return INVALID_HANDLE_VALUE;
-  if (direction == 0)
-    return fh->get_handle ();
-  return fh->get_output_handle ();
+  HANDLE h;
+  cygheap_fdget cfd (fd);
+
+  if (cfd < 0)
+    h = INVALID_HANDLE_VALUE;
+  else if (cfd->close_on_exec ())
+    h = INVALID_HANDLE_VALUE;
+  else if (direction == 0)
+    h = cfd->get_handle ();
+  else
+    h = cfd->get_output_handle ();
+  return h;
 }
 
 int
@@ -580,7 +583,9 @@ spawn_guts (const char * prog_arg, const char *const *argv,
   sigproc_printf ("priority class %d", flags);
   flags |= CREATE_DEFAULT_ERROR_MODE | CREATE_SEPARATE_WOW_VDM;
 
-  if (mode == _P_DETACH || !set_console_state_for_spawn ())
+  set_console_state_for_spawn ();
+
+  if (mode == _P_DETACH)
     flags |= DETACHED_PROCESS;
 
   if (mode != _P_OVERLAY)