* elf/dl-close.c: Include stddef.h.

	(_dl_close): If called recursively, just remember GC needs to be rerun
	and decrease l_direct_opencount.  Avoid GC if l_direct_opencount
	decreased to 1.  Rerun GC at the end if any destructor unloaded some
	additional libraries.
	* elf/Makefile: Add rules to build and run unload6 test.
	* elf/unload6.c: New test.
	* elf/unload6mod1.c: New file.
	* elf/unload6mod2.c: New file.
	* elf/unload6mod3.c: New file.

	* malloc/hooks.c (mem2chunk_check): Add magic_p argument, set *magic_p
	if magic_p is not NULL.
	(top_check): Invoke MALLOC_FAILURE_ACTION if MORECORE failed.
	(malloc_check): Fail if sz == -1.
	(free_check): Adjust mem2chunk_check caller.
	(realloc_check): Likewise.  Fail if bytes == -1.  If bytes == 0 and
	oldmem != NULL, call free_check and return NULL.  If reallocating
	and returning NULL, invert magic byte again to make oldmem valid
	region for further checking.
	(memalign_check): Fail if bytes == -1.
	* malloc/Makefile: Add rules to build and run tst-mcheck.
	* malloc/tst-mcheck.c: New test.

10 files changed, 302 insertions, 26 deletions

diff --git a/ChangeLog b/ChangeLog
index 7440d01..2062804 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,29 @@
+2005-04-26  Jakub Jelinek  <jakub@redhat.com>
+
+	* elf/dl-close.c: Include stddef.h.
+	(_dl_close): If called recursively, just remember GC needs to be rerun
+	and decrease l_direct_opencount.  Avoid GC if l_direct_opencount
+	decreased to 1.  Rerun GC at the end if any destructor unloaded some
+	additional libraries.
+	* elf/Makefile: Add rules to build and run unload6 test.
+	* elf/unload6.c: New test.
+	* elf/unload6mod1.c: New file.
+	* elf/unload6mod2.c: New file.
+	* elf/unload6mod3.c: New file.
+
+	* malloc/hooks.c (mem2chunk_check): Add magic_p argument, set *magic_p
+	if magic_p is not NULL.
+	(top_check): Invoke MALLOC_FAILURE_ACTION if MORECORE failed.
+	(malloc_check): Fail if sz == -1.
+	(free_check): Adjust mem2chunk_check caller.
+	(realloc_check): Likewise.  Fail if bytes == -1.  If bytes == 0 and
+	oldmem != NULL, call free_check and return NULL.  If reallocating
+	and returning NULL, invert magic byte again to make oldmem valid
+	region for further checking.
+	(memalign_check): Fail if bytes == -1.
+	* malloc/Makefile: Add rules to build and run tst-mcheck.
+	* malloc/tst-mcheck.c: New test.
+
 2005-04-26  Ulrich Drepper  <drepper@redhat.com>
 
 	* stdio-common/vfscanf.c: Correctly account for characters of
diff --git a/elf/Makefile b/elf/Makefile
index c034b2b..d988bac 100644
--- a/elf/Makefile
+++ b/elf/Makefile
@@ -86,7 +86,7 @@ distribute	:= rtld-Rules \
 		   tst-deep1mod1.c tst-deep1mod2.c tst-deep1mod3.c \
 		   unload3mod1.c unload3mod2.c unload3mod3.c unload3mod4.c \
 		   unload4mod1.c unload4mod2.c unload4mod3.c unload4mod4.c \
-		   tst-auditmod1.c \
+		   unload6mod1.c unload6mod2.c unload6mod3.c tst-auditmod1.c \
 		   order2mod1.c order2mod2.c order2mod3.c order2mod4.c
 
 CFLAGS-dl-runtime.c = -fexceptions -fasynchronous-unwind-tables
@@ -162,7 +162,7 @@ tests += loadtest restest1 preloadtest loadfail multiload origtest resolvfail \
 	 tst-tls10 tst-tls11 tst-tls12 tst-tls13 tst-tls14 tst-tls15 tst-align \
 	 tst-align2 $(tests-execstack-$(have-z-execstack)) tst-dlmodcount \
 	 tst-dlopenrpath tst-deep1 tst-dlmopen1 tst-dlmopen2 tst-dlmopen3 \
-	 unload3 unload4 unload5 tst-audit1 tst-global1 order2
+	 unload3 unload4 unload5 unload6 tst-audit1 tst-global1 order2
 #	 reldep9
 test-srcs = tst-pathopt
 tests-vis-yes = vismain
@@ -201,6 +201,7 @@ modules-names = testobj1 testobj2 testobj3 testobj4 testobj5 testobj6 \
 		tst-dlmopen1mod tst-auditmod1 \
 		unload3mod1 unload3mod2 unload3mod3 unload3mod4 \
 		unload4mod1 unload4mod2 unload4mod3 unload4mod4 \
+		unload6mod1 unload6mod2 unload6mod3 \
 		order2mod1 order2mod2 order2mod3 order2mod4
 ifeq (yes,$(have-initfini-array))
 modules-names += tst-array2dep
@@ -438,6 +439,9 @@ $(objpfx)unload3mod2.so: $(objpfx)unload3mod3.so
 $(objpfx)unload3mod3.so: $(objpfx)unload3mod4.so
 $(objpfx)unload4mod1.so: $(objpfx)unload4mod2.so $(objpfx)unload4mod3.so
 $(objpfx)unload4mod2.so: $(objpfx)unload4mod4.so $(objpfx)unload4mod3.so
+$(objpfx)unload6mod1.so: $(libdl)
+$(objpfx)unload6mod2.so: $(libdl)
+$(objpfx)unload6mod3.so: $(libdl)
 
 LDFLAGS-tst-tlsmod5.so = -nostdlib
 LDFLAGS-tst-tlsmod6.so = -nostdlib
@@ -710,6 +714,10 @@ $(objpfx)unload5: $(libdl)
 $(objpfx)unload5.out: $(objpfx)unload3mod1.so $(objpfx)unload3mod2.so \
 		      $(objpfx)unload3mod3.so $(objpfx)unload3mod4.so
 
+$(objpfx)unload6: $(libdl)
+$(objpfx)unload6.out: $(objpfx)unload6mod1.so $(objpfx)unload6mod2.so \
+		      $(objpfx)unload6mod3.so
+
 ifdef libdl
 $(objpfx)tst-tls9-static: $(common-objpfx)dlfcn/libdl.a
 $(objpfx)tst-tls9-static.out: $(objpfx)tst-tlsmod5.so $(objpfx)tst-tlsmod6.so
diff --git a/elf/dl-close.c b/elf/dl-close.c
index cd4fa7c..754dd67 100644
--- a/elf/dl-close.c
+++ b/elf/dl-close.c
@@ -20,6 +20,7 @@
 #include <assert.h>
 #include <dlfcn.h>
 #include <libintl.h>
+#include <stddef.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -105,10 +106,6 @@ _dl_close (void *_map)
   struct link_map *map = _map;
   Lmid_t ns = map->l_ns;
   unsigned int i;
-#ifdef USE_TLS
-  bool any_tls = false;
-#endif
-
   /* First see whether we can remove the object at all.  */
   if (__builtin_expect (map->l_flags_1 & DF_1_NODELETE, 0)
       && map->l_init_called)
@@ -124,9 +121,17 @@ _dl_close (void *_map)
   /* One less direct use.  */
   --map->l_direct_opencount;
 
-  /* Decrement the reference count.  */
-  if (map->l_direct_opencount > 1 || map->l_type != lt_loaded)
+  /* If _dl_close is called recursively (some destructor call dlclose),
+     just record that the parent _dl_close will need to do garbage collection
+     again and return.  */
+  static enum { not_pending, pending, rerun } dl_close_state;
+
+  if (map->l_direct_opencount > 0 || map->l_type != lt_loaded
+      || dl_close_state != not_pending)
     {
+      if (map->l_direct_opencount == 0 && map->l_type == lt_loaded)
+	dl_close_state = rerun;
+
       /* There are still references to this object.  Do nothing more.  */
       if (__builtin_expect (GLRO(dl_debug_mask) & DL_DEBUG_FILES, 0))
 	_dl_debug_printf ("\nclosing file=%s; direct_opencount=%u\n",
@@ -136,12 +141,18 @@ _dl_close (void *_map)
       return;
     }
 
+ retry:
+  dl_close_state = pending;
+
+#ifdef USE_TLS
+  bool any_tls = false;
+#endif
   const unsigned int nloaded = GL(dl_ns)[ns]._ns_nloaded;
   char used[nloaded];
   char done[nloaded];
   struct link_map *maps[nloaded];
 
-  /* Run over the list and assign indeces to the link maps and enter
+  /* Run over the list and assign indexes to the link maps and enter
      them into the MAPS array.  */
   int idx = 0;
   for (struct link_map *l = GL(dl_ns)[ns]._ns_loaded; l != NULL; l = l->l_next)
@@ -302,7 +313,7 @@ _dl_close (void *_map)
 	  if (imap->l_searchlist.r_list == NULL
 	      && imap->l_initfini != NULL)
 	    {
-	      /* The object is still used.  But the object we are
+	      /* The object is still used.  But one of the objects we are
 		 unloading right now is responsible for loading it.  If
 		 the current object does not have it's own scope yet we
 		 have to create one.  This has to be done before running
@@ -318,15 +329,27 @@ _dl_close (void *_map)
 	      imap->l_searchlist.r_nlist = cnt;
 
 	      for (cnt = 0; imap->l_scope[cnt] != NULL; ++cnt)
-		if (imap->l_scope[cnt] == &map->l_searchlist)
+		/* This relies on l_scope[] entries being always set either
+		   to its own l_symbolic_searchlist address, or some other map's
+		   l_searchlist address.  */
+		if (imap->l_scope[cnt] != &imap->l_symbolic_searchlist)
 		  {
-		    imap->l_scope[cnt] = &imap->l_searchlist;
-		    break;
+		    struct link_map *tmap;
+
+		    tmap = (struct link_map *) ((char *) imap->l_scope[cnt]
+						- offsetof (struct link_map,
+							    l_searchlist));
+		    assert (tmap->l_ns == ns);
+		    if (tmap->l_idx != -1)
+		      {
+			imap->l_scope[cnt] = &imap->l_searchlist;
+			break;
+		      }
 		  }
 	    }
 
 	  /* The loader is gone, so mark the object as not having one.
-	     Note: l_idx == -1 -> object will be removed.  */
+	     Note: l_idx != -1 -> object will be removed.  */
 	  if (imap->l_loader != NULL && imap->l_loader->l_idx != -1)
 	    imap->l_loader = NULL;
 
@@ -583,8 +606,12 @@ _dl_close (void *_map)
   r->r_state = RT_CONSISTENT;
   _dl_debug_state ();
 
-  /* Release the lock.  */
+  /* Recheck if we need to retry, release the lock.  */
  out:
+  if (dl_close_state == rerun)
+    goto retry;
+
+  dl_close_state = not_pending;
   __rtld_lock_unlock_recursive (GL(dl_load_lock));
 }
 
@@ -654,7 +681,7 @@ libc_freeres_fn (free_mem)
 	free_slotinfo (&GL(dl_tls_dtv_slotinfo_list));
       else
 # endif
-        /* The first element of the list does not have to be deallocated.
+	/* The first element of the list does not have to be deallocated.
 	   It was allocated in the dynamic linker (i.e., with a different
 	   malloc), and in the static library it's in .bss space.  */
 	free_slotinfo (&GL(dl_tls_dtv_slotinfo_list)->next);
diff --git a/elf/unload6.c b/elf/unload6.c
new file mode 100644
index 0000000..1efc7eb
--- /dev/null
+++ b/elf/unload6.c
@@ -0,0 +1,30 @@
+#include <dlfcn.h>
+#include <stdio.h>
+
+int
+main (void)
+{
+  void *h = dlopen ("unload6mod1.so", RTLD_LAZY);
+  if (h == NULL)
+    {
+      puts ("dlopen unload6mod1.so failed");
+      return 1;
+    }
+
+  int (*fn) (int);
+  fn = dlsym (h, "foo");
+  if (fn == NULL)
+    {
+      puts ("dlsym failed");
+      return 1;
+    }
+
+  int val = fn (16);
+  if (val != 24)
+    {
+      printf ("foo returned %d != 24\n", val);
+      return 1;
+    }
+
+  return 0;
+}
diff --git a/elf/unload6mod1.c b/elf/unload6mod1.c
new file mode 100644
index 0000000..24f2e5a
--- /dev/null
+++ b/elf/unload6mod1.c
@@ -0,0 +1,16 @@
+#include <dlfcn.h>
+#include <stdio.h>
+
+int
+foo (int i)
+{
+  void *h = dlopen ("unload6mod2.so", RTLD_LAZY);
+  if (h == NULL)
+    {
+      puts ("dlopen unload6mod2.so failed");
+      return 1;
+    }
+
+  dlclose (h);
+  return i + 8;
+}
diff --git a/elf/unload6mod2.c b/elf/unload6mod2.c
new file mode 100644
index 0000000..980efa4
--- /dev/null
+++ b/elf/unload6mod2.c
@@ -0,0 +1,23 @@
+#include <dlfcn.h>
+#include <stdio.h>
+#include <unistd.h>
+
+static void *h;
+
+static void __attribute__((constructor))
+mod2init (void)
+{
+  h = dlopen ("unload6mod3.so", RTLD_LAZY);
+  if (h == NULL)
+    {
+      puts ("dlopen unload6mod3.so failed");
+      fflush (stdout);
+      _exit (1);
+    }
+}
+
+static void __attribute__((destructor))
+mod2fini (void)
+{
+  dlclose (h);
+}
diff --git a/elf/unload6mod3.c b/elf/unload6mod3.c
new file mode 100644
index 0000000..7b29e1d
--- /dev/null
+++ b/elf/unload6mod3.c
@@ -0,0 +1,23 @@
+#include <dlfcn.h>
+#include <stdio.h>
+#include <unistd.h>
+
+static void *h;
+
+static void __attribute__((constructor))
+mod3init (void)
+{
+  h = dlopen ("unload6mod1.so", RTLD_LAZY);
+  if (h == NULL)
+    {
+      puts ("dlopen unload6mod1.so failed");
+      fflush (stdout);
+      _exit (1);
+    }
+}
+
+static void __attribute__((destructor))
+mod3fini (void)
+{
+  dlclose (h);
+}
diff --git a/malloc/Makefile b/malloc/Makefile
index 45e8f59..398dd2b 100644
--- a/malloc/Makefile
+++ b/malloc/Makefile
@@ -1,4 +1,5 @@
-# Copyright (C) 1991-1999,2000,2001,2002,2003 Free Software Foundation, Inc.
+# Copyright (C) 1991-1999, 2000, 2001, 2002, 2003, 2005
+# Free Software Foundation, Inc.
 # This file is part of the GNU C Library.
 
 # The GNU C Library is free software; you can redistribute it and/or
@@ -26,7 +27,7 @@ all:
 dist-headers := malloc.h
 headers := $(dist-headers) obstack.h mcheck.h
 tests := mallocbug tst-malloc tst-valloc tst-calloc tst-obstack \
-	 tst-mallocstate
+	 tst-mallocstate tst-mcheck
 test-srcs = tst-mtrace
 
 distribute = thread-m.h mtrace.pl mcheck-init.c stackinfo.h memusage.h \
@@ -120,6 +121,8 @@ endif
 endif
 endif
 
+tst-mcheck-ENV = MALLOC_CHECK_=3
+
 # Uncomment this for test releases.  For public releases it is too expensive.
 #CPPFLAGS-malloc.o += -DMALLOC_DEBUG=1
 
diff --git a/malloc/hooks.c b/malloc/hooks.c
index a5c97f3..0f8f274 100644
--- a/malloc/hooks.c
+++ b/malloc/hooks.c
@@ -1,5 +1,5 @@
 /* Malloc implementation for multiple threads without lock contention.
-   Copyright (C) 2001, 2002, 2003, 2004 Free Software Foundation, Inc.
+   Copyright (C) 2001, 2002, 2003, 2004, 2005 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Wolfram Gloger <wg@malloc.de>, 2001.
 
@@ -146,9 +146,9 @@ mem2mem_check(ptr, sz) Void_t *ptr; size_t sz;
 static mchunkptr
 internal_function
 #if __STD_C
-mem2chunk_check(Void_t* mem)
+mem2chunk_check(Void_t* mem, unsigned char **magic_p)
 #else
-mem2chunk_check(mem) Void_t* mem;
+mem2chunk_check(mem, magic_p) Void_t* mem; unsigned char **magic_p;
 #endif
 {
   mchunkptr p;
@@ -173,7 +173,6 @@ mem2chunk_check(mem) Void_t* mem;
     for(sz += SIZE_SZ-1; (c = ((unsigned char*)p)[sz]) != magic; sz -= c) {
       if(c<=0 || sz<(c+2*SIZE_SZ)) return NULL;
     }
-    ((unsigned char*)p)[sz] ^= 0xFF;
   } else {
     unsigned long offset, page_mask = malloc_getpagesize-1;
 
@@ -193,8 +192,10 @@ mem2chunk_check(mem) Void_t* mem;
     for(sz -= 1; (c = ((unsigned char*)p)[sz]) != magic; sz -= c) {
       if(c<=0 || sz<(c+2*SIZE_SZ)) return NULL;
     }
-    ((unsigned char*)p)[sz] ^= 0xFF;
   }
+  ((unsigned char*)p)[sz] ^= 0xFF;
+  if (magic_p)
+    *magic_p = (unsigned char *)p + sz;
   return p;
 }
 
@@ -232,7 +233,11 @@ top_check()
   sbrk_size = front_misalign + mp_.top_pad + MINSIZE;
   sbrk_size += pagesz - ((unsigned long)(brk + sbrk_size) & (pagesz - 1));
   new_brk = (char*)(MORECORE (sbrk_size));
-  if (new_brk == (char*)(MORECORE_FAILURE)) return -1;
+  if (new_brk == (char*)(MORECORE_FAILURE))
+    {
+      MALLOC_FAILURE_ACTION;
+      return -1;
+    }
   /* Call the `morecore' hook if necessary.  */
   if (__after_morecore_hook)
     (*__after_morecore_hook) ();
@@ -253,6 +258,11 @@ malloc_check(sz, caller) size_t sz; const Void_t *caller;
 {
   Void_t *victim;
 
+  if (sz+1 == 0) {
+    MALLOC_FAILURE_ACTION;
+    return NULL;
+  }
+
   (void)mutex_lock(&main_arena.mutex);
   victim = (top_check() >= 0) ? _int_malloc(&main_arena, sz+1) : NULL;
   (void)mutex_unlock(&main_arena.mutex);
@@ -270,7 +280,7 @@ free_check(mem, caller) Void_t* mem; const Void_t *caller;
 
   if(!mem) return;
   (void)mutex_lock(&main_arena.mutex);
-  p = mem2chunk_check(mem);
+  p = mem2chunk_check(mem, NULL);
   if(!p) {
     (void)mutex_unlock(&main_arena.mutex);
 
@@ -302,10 +312,19 @@ realloc_check(oldmem, bytes, caller)
   mchunkptr oldp;
   INTERNAL_SIZE_T nb, oldsize;
   Void_t* newmem = 0;
+  unsigned char *magic_p;
 
+  if (bytes+1 == 0) {
+    MALLOC_FAILURE_ACTION;
+    return NULL;
+  }
   if (oldmem == 0) return malloc_check(bytes, NULL);
+  if (bytes == 0) {
+    free_check (oldmem, NULL);
+    return NULL;
+  }
   (void)mutex_lock(&main_arena.mutex);
-  oldp = mem2chunk_check(oldmem);
+  oldp = mem2chunk_check(oldmem, &magic_p);
   (void)mutex_unlock(&main_arena.mutex);
   if(!oldp) {
     malloc_printerr(check_action, "realloc(): invalid pointer", oldmem);
@@ -357,6 +376,12 @@ realloc_check(oldmem, bytes, caller)
 #if HAVE_MMAP
   }
 #endif
+
+  /* mem2chunk_check changed the magic byte in the old chunk.
+     If newmem is NULL, then the old chunk will still be used though,
+     so we need to invert that change here.  */
+  if (newmem == NULL) *magic_p ^= 0xFF;
+
   (void)mutex_unlock(&main_arena.mutex);
 
   return mem2mem_check(newmem, bytes);
@@ -376,6 +401,10 @@ memalign_check(alignment, bytes, caller)
   if (alignment <= MALLOC_ALIGNMENT) return malloc_check(bytes, NULL);
   if (alignment <  MINSIZE) alignment = MINSIZE;
 
+  if (bytes+1 == 0) {
+    MALLOC_FAILURE_ACTION;
+    return NULL;
+  }
   checked_request2size(bytes+1, nb);
   (void)mutex_lock(&main_arena.mutex);
   mem = (top_check() >= 0) ? _int_memalign(&main_arena, alignment, bytes+1) :
diff --git a/malloc/tst-mcheck.c b/malloc/tst-mcheck.c
new file mode 100644
index 0000000..1678491
--- /dev/null
+++ b/malloc/tst-mcheck.c
@@ -0,0 +1,91 @@
+/* Copyright (C) 2005 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Jakub Jelinek <jakub@redhat.com>, 2005.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static int errors = 0;
+
+static void
+merror (const char *msg)
+{
+  ++errors;
+  printf ("Error: %s\n", msg);
+}
+
+int
+main (void)
+{
+  void *p, *q;
+
+  errno = 0;
+
+  p = malloc (-1);
+
+  if (p != NULL)
+    merror ("malloc (-1) succeeded.");
+  else if (errno != ENOMEM)
+    merror ("errno is not set correctly.");
+
+  p = malloc (10);
+  if (p == NULL)
+    merror ("malloc (10) failed.");
+
+  p = realloc (p, 0);
+  if (p != NULL)
+    merror ("realloc (p, 0) failed.");
+
+  p = malloc (0);
+  if (p == NULL)
+    merror ("malloc (0) failed.");
+
+  p = realloc (p, 0);
+  if (p != NULL)
+    merror ("realloc (p, 0) failed.");
+
+  q = malloc (256);
+  if (q == NULL)
+    merror ("malloc (256) failed.");
+
+  p = malloc (512);
+  if (p == NULL)
+    merror ("malloc (512) failed.");
+
+  if (realloc (p, -256) != NULL)
+    merror ("realloc (p, -256) succeeded.");
+  else if (errno != ENOMEM)
+    merror ("errno is not set correctly.");
+
+  free (p);
+
+  p = malloc (512);
+  if (p == NULL)
+    merror ("malloc (512) failed.");
+
+  if (realloc (p, -1) != NULL)
+    merror ("realloc (p, -1) succeeded.");
+  else if (errno != ENOMEM)
+    merror ("errno is not set correctly.");
+
+  free (p);
+  free (q);
+
+  return errors != 0;
+}