Fix BZ #15089: malloc_trim always trim for large padding.

3 files changed, 50 insertions, 43 deletions

diff --git a/ChangeLog b/ChangeLog
index 7c8164a..c03e056 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,8 @@
+2013-12-06  Fernando J. V. da Silva  <fernandojvdasilva@gmail.com>
+
+	[BZ #15089]
+	* malloc/malloc.c: Exit systrim() if pad is bigger than heap top size.
+
 2013-12-06  Adhemerval Zanella  <azanella@linux.vnet.ibm.com>
 
 	* NEWS: Mention ppc32/power4+ STT_GNU_IFUNC support.
diff --git a/NEWS b/NEWS
index 2e80cbe..59ae1c2 100644
--- a/NEWS
+++ b/NEWS
@@ -12,16 +12,16 @@ Version 2.19
   156, 387, 431, 832, 926, 2801, 4772, 6786, 6787, 6807, 6810, 7003, 9954,
   10253, 10278, 11087, 11157, 11214, 12486, 13028, 13982, 13985, 14029,
   14032, 14143, 14155, 14547, 14699, 14752, 14876, 14910, 15004, 15048,
-  15218, 15268, 15277, 15308, 15362, 15374, 15400, 15425, 15427, 15483,
-  15522, 15531, 15532, 15601, 15608, 15609, 15610, 15632, 15640, 15670,
-  15672, 15680, 15681, 15723, 15734, 15735, 15736, 15748, 15749, 15754,
-  15760, 15763, 15764, 15797, 15799, 15825, 15844, 15847, 15849, 15855,
-  15856, 15857, 15859, 15867, 15886, 15887, 15890, 15892, 15893, 15895,
-  15897, 15901, 15905, 15909, 15915, 15917, 15919, 15921, 15923, 15939,
-  15941, 15948, 15963, 15966, 15985, 15988, 15997, 16032, 16034, 16036,
-  16037, 16038, 16041, 16055, 16071, 16072, 16074, 16077, 16078, 16103,
-  16112, 16143, 16144, 16146, 16150, 16151, 16153, 16167, 16172, 16195,
-  16214, 16245, 16271, 16274, 16283, 16289.
+  15089, 15218, 15268, 15277, 15308, 15362, 15374, 15400, 15425, 15427,
+  15483, 15522, 15531, 15532, 15601, 15608, 15609, 15610, 15632, 15640,
+  15670, 15672, 15680, 15681, 15723, 15734, 15735, 15736, 15748, 15749,
+  15754, 15760, 15763, 15764, 15797, 15799, 15825, 15844, 15847, 15849,
+  15855, 15856, 15857, 15859, 15867, 15886, 15887, 15890, 15892, 15893,
+  15895, 15897, 15901, 15905, 15909, 15915, 15917, 15919, 15921, 15923,
+  15939, 15941, 15948, 15963, 15966, 15985, 15988, 15997, 16032, 16034,
+  16036, 16037, 16038, 16041, 16055, 16071, 16072, 16074, 16077, 16078,
+  16103, 16112, 16143, 16144, 16146, 16150, 16151, 16153, 16167, 16172,
+  16195, 16214, 16245, 16271, 16274, 16283, 16289.
 
 * The public headers no longer use __unused nor __block.  This change is to
   support compiling programs that are derived from BSD sources and use
diff --git a/malloc/malloc.c b/malloc/malloc.c
index 8977687..b3353bd 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -2709,52 +2709,54 @@ static int systrim(size_t pad, mstate av)
   char* current_brk;     /* address returned by pre-check sbrk call */
   char* new_brk;         /* address returned by post-check sbrk call */
   size_t pagesz;
+  long  top_area;
 
   pagesz = GLRO(dl_pagesize);
   top_size = chunksize(av->top);
 
+  top_area = top_size - MINSIZE - 1;
+  if (top_area <= pad)
+    return 0;
+
   /* Release in pagesize units, keeping at least one page */
-  extra = (top_size - pad - MINSIZE - 1) & ~(pagesz - 1);
+  extra = (top_area - pad) & ~(pagesz - 1);
 
-  if (extra > 0) {
+  /*
+    Only proceed if end of memory is where we last set it.
+    This avoids problems if there were foreign sbrk calls.
+  */
+  current_brk = (char*)(MORECORE(0));
+  if (current_brk == (char*)(av->top) + top_size) {
 
     /*
-      Only proceed if end of memory is where we last set it.
-      This avoids problems if there were foreign sbrk calls.
+      Attempt to release memory. We ignore MORECORE return value,
+      and instead call again to find out where new end of memory is.
+      This avoids problems if first call releases less than we asked,
+      of if failure somehow altered brk value. (We could still
+      encounter problems if it altered brk in some very bad way,
+      but the only thing we can do is adjust anyway, which will cause
+      some downstream failure.)
     */
-    current_brk = (char*)(MORECORE(0));
-    if (current_brk == (char*)(av->top) + top_size) {
-
-      /*
-	Attempt to release memory. We ignore MORECORE return value,
-	and instead call again to find out where new end of memory is.
-	This avoids problems if first call releases less than we asked,
-	of if failure somehow altered brk value. (We could still
-	encounter problems if it altered brk in some very bad way,
-	but the only thing we can do is adjust anyway, which will cause
-	some downstream failure.)
-      */
 
-      MORECORE(-extra);
-      /* Call the `morecore' hook if necessary.  */
-      void (*hook) (void) = force_reg (__after_morecore_hook);
-      if (__builtin_expect (hook != NULL, 0))
-	(*hook) ();
-      new_brk = (char*)(MORECORE(0));
+    MORECORE(-extra);
+    /* Call the `morecore' hook if necessary.  */
+    void (*hook) (void) = force_reg (__after_morecore_hook);
+    if (__builtin_expect (hook != NULL, 0))
+      (*hook) ();
+    new_brk = (char*)(MORECORE(0));
 
-      LIBC_PROBE (memory_sbrk_less, 2, new_brk, extra);
+    LIBC_PROBE (memory_sbrk_less, 2, new_brk, extra);
 
-      if (new_brk != (char*)MORECORE_FAILURE) {
-	released = (long)(current_brk - new_brk);
+    if (new_brk != (char*)MORECORE_FAILURE) {
+      released = (long)(current_brk - new_brk);
 
-	if (released != 0) {
-	  /* Success. Adjust top. */
-	  av->system_mem -= released;
-	  set_head(av->top, (top_size - released) | PREV_INUSE);
-	  check_malloc_state(av);
-	  return 1;
-	}
-      }
+      if (released != 0) {
+	/* Success. Adjust top. */
+	av->system_mem -= released;
+	set_head(av->top, (top_size - released) | PREV_INUSE);
+	check_malloc_state(av);
+	return 1;
+       }
     }
   }
   return 0;