* cygheap.h (cygheap_fdnew): Avoid setting errno directly since it will have

been set by a previous function.
* dtable.h (dtable::extend): Accept second size_t argument.
* dtable.cc (dtable::extend): Accept second "min" argument which allows
checking for OPEN_MAX_MAX boundary conditions.
(dtable_init): Accommodate second argument to dtable::extend.
(dtable::find_unused_handle): Ditto.
* syscalls.cc (setdtablesize): Ditto.
(dup): Return any error passed by cygheap_fdnew() directly.
(getdtablesize): Just return dtable size directly.

5 files changed, 33 insertions, 10 deletions

diff --git a/winsup/cygwin/ChangeLog b/winsup/cygwin/ChangeLog
index 490a7d6..68f05c4 100644
--- a/winsup/cygwin/ChangeLog
+++ b/winsup/cygwin/ChangeLog
@@ -1,3 +1,16 @@
+2013-12-05  Christopher Faylor  <me.cygwin2013@cgf.cx>
+
+	* cygheap.h (cygheap_fdnew): Avoid setting errno directly since it will
+	have been set by a previous function.
+	* dtable.h (dtable::extend): Accept second size_t argument.
+	* dtable.cc (dtable::extend): Accept second "min" argument which allows
+	checking for OPEN_MAX_MAX boundary conditions.
+	(dtable_init): Accommodate second argument to dtable::extend.
+	(dtable::find_unused_handle): Ditto.
+	* syscalls.cc (setdtablesize): Ditto.
+	(dup): Return any error passed by cygheap_fdnew() directly.
+	(getdtablesize): Just return dtable size directly.
+
 2013-12-04  Christopher Faylor  <me.cygwin2013@cgf.cx>
 
 	* dtable.cc (dtable::find_unused_handle): When extending, always make
diff --git a/winsup/cygwin/cygheap.h b/winsup/cygwin/cygheap.h
index b4c478f..164a870 100644
--- a/winsup/cygwin/cygheap.h
+++ b/winsup/cygwin/cygheap.h
@@ -457,7 +457,7 @@ class cygheap_fdnew : public cygheap_fdmanip
       locked = lockit;
     else
       {
-	set_errno (EMFILE);
+	/* errno set by find_unused_handle */
 	if (lockit)
 	  cygheap->fdtab.unlock ();
 	locked = false;
diff --git a/winsup/cygwin/dtable.cc b/winsup/cygwin/dtable.cc
index 4547984..2e9ee54 100644
--- a/winsup/cygwin/dtable.cc
+++ b/winsup/cygwin/dtable.cc
@@ -58,7 +58,7 @@ void
 dtable_init ()
 {
   if (!cygheap->fdtab.size)
-    cygheap->fdtab.extend (NOFILE_INCR);
+    cygheap->fdtab.extend (NOFILE_INCR, 0);
 }
 
 void __stdcall
@@ -72,12 +72,16 @@ set_std_handle (int fd)
 }
 
 int
-dtable::extend (size_t howmuch)
+dtable::extend (size_t howmuch, size_t min)
 {
   size_t new_size = size + howmuch;
   fhandler_base **newfds;
 
-  if (new_size > OPEN_MAX_MAX)
+  if (new_size <= OPEN_MAX_MAX)
+    /* ok */;
+  else if (size < OPEN_MAX_MAX && min < OPEN_MAX_MAX)
+    new_size = OPEN_MAX_MAX;
+  else
     {
       set_errno (EMFILE);
       return 0;
@@ -223,7 +227,7 @@ dtable::delete_archetype (fhandler_base *fh)
 int
 dtable::find_unused_handle (size_t start)
 {
-  /* When extending, always make sure that there is a NOFILE_INCR chunk
+  /* When extending, try to allocate a NOFILE_INCR chunk
      following the empty fd.  */
   size_t extendby = NOFILE_INCR + ((start >= size) ? 1 + start - size : 0);
 
@@ -238,7 +242,7 @@ dtable::find_unused_handle (size_t start)
 	    goto out;
 	  }
     }
-  while (extend (extendby));
+  while (extend (extendby, start));
 out:
   return res;
 }
diff --git a/winsup/cygwin/dtable.h b/winsup/cygwin/dtable.h
index 0ec7b3a..53ffca3 100644
--- a/winsup/cygwin/dtable.h
+++ b/winsup/cygwin/dtable.h
@@ -52,7 +52,7 @@ public:
   void vfork_parent_restore ();
   void vfork_child_fixup ();
   fhandler_base *dup_worker (fhandler_base *oldfh, int flags);
-  int extend (size_t howmuch);
+  int extend (size_t, size_t) __reg3;
   void fixup_after_fork (HANDLE);
   void fixup_close (size_t, fhandler_base *);
 
diff --git a/winsup/cygwin/syscalls.cc b/winsup/cygwin/syscalls.cc
index 8769eb5..cb97091 100644
--- a/winsup/cygwin/syscalls.cc
+++ b/winsup/cygwin/syscalls.cc
@@ -120,7 +120,12 @@ close_all_files (bool norelease)
 extern "C" int
 dup (int fd)
 {
-  int res = cygheap->fdtab.dup3 (fd, cygheap_fdnew (), 0);
+  int res;
+  int newfd = cygheap_fdnew ();
+  if (newfd < 0)
+    res = -1;
+  else
+    res = cygheap->fdtab.dup3 (fd, newfd, 0);
   syscall_printf ("%R = dup(%d)", res, fd);
   return res;
 }
@@ -2611,7 +2616,8 @@ setdtablesize (int size)
       return -1;
     }
 
-  if (size <= (int)cygheap->fdtab.size || cygheap->fdtab.extend (size - cygheap->fdtab.size))
+  if (size <= (int) cygheap->fdtab.size
+      || cygheap->fdtab.extend (size - cygheap->fdtab.size, OPEN_MAX_MAX))
     return 0;
 
   return -1;
@@ -2620,7 +2626,7 @@ setdtablesize (int size)
 extern "C" int
 getdtablesize ()
 {
-  return cygheap->fdtab.size > OPEN_MAX ? cygheap->fdtab.size : OPEN_MAX;
+  return cygheap->fdtab.size;
 }
 
 extern "C" int