Support new tunables system

Add new tunables:

malloc_tcache_max
malloc_tcache_count
malloc_tcache_unsorted_limit

(the last is new, it limits how deeply we scan the unsorted list
looking for chunks to move into the tcache)

3 files changed, 106 insertions, 9 deletions

diff --git a/elf/dl-tunables.list b/elf/dl-tunables.list
index d8cd912..3e49875 100644
--- a/elf/dl-tunables.list
+++ b/elf/dl-tunables.list
@@ -64,5 +64,17 @@ glibc {
       env_alias: MALLOC_ARENA_TEST
       minval: 1
     }
+    tcache_max {
+      type: SIZE_T
+      env_alias: MALLOC_TCACHE_MAX
+    }
+    tcache_count {
+      type: SIZE_T
+      env_alias: MALLOC_TCACHE_COUNT
+    }
+    tcache_unsorted_limit {
+      type: SIZE_T
+      env_alias: MALLOC_TCACHE_UNSORTED_LIMIT
+    }
   }
 }
diff --git a/malloc/arena.c b/malloc/arena.c
index b91d7d6..74616df 100644
--- a/malloc/arena.c
+++ b/malloc/arena.c
@@ -236,6 +236,9 @@ DL_TUNABLE_CALLBACK_FNDECL (set_perturb_byte, int32_t)
 DL_TUNABLE_CALLBACK_FNDECL (set_trim_threshold, size_t)
 DL_TUNABLE_CALLBACK_FNDECL (set_arena_max, size_t)
 DL_TUNABLE_CALLBACK_FNDECL (set_arena_test, size_t)
+DL_TUNABLE_CALLBACK_FNDECL (set_tcache_max, size_t)
+DL_TUNABLE_CALLBACK_FNDECL (set_tcache_count, size_t)
+DL_TUNABLE_CALLBACK_FNDECL (set_tcache_unsorted_limit, size_t)
 #else
 /* Initialization routine. */
 #include <string.h>
@@ -322,6 +325,11 @@ ptmalloc_init (void)
   TUNABLE_SET_VAL_WITH_CALLBACK (mmap_max, NULL, set_mmaps_max);
   TUNABLE_SET_VAL_WITH_CALLBACK (arena_max, NULL, set_arena_max);
   TUNABLE_SET_VAL_WITH_CALLBACK (arena_test, NULL, set_arena_test);
+#if USE_TCACHE
+  TUNABLE_SET_VAL_WITH_CALLBACK (tcache_max, NULL, set_tcache_max);
+  TUNABLE_SET_VAL_WITH_CALLBACK (tcache_count, NULL, set_tcache_count);
+  TUNABLE_SET_VAL_WITH_CALLBACK (tcache_unsorted_limit, NULL, set_tcache_unsorted_limit);
+#endif
   __libc_lock_unlock (main_arena.mutex);
 #else
   const char *s = NULL;
@@ -371,7 +379,23 @@ ptmalloc_init (void)
                   if (memcmp (envline, "ARENA_TEST", 10) == 0)
                     __libc_mallopt (M_ARENA_TEST, atoi (&envline[11]));
                 }
+#if USE_TCACHE
+              if (!__builtin_expect (__libc_enable_secure, 0))
+                {
+                  if (memcmp (envline, "TCACHE_MAX", 10) == 0)
+                    __libc_mallopt (M_TCACHE_MAX, atoi (&envline[11]));
+                }
+#endif
               break;
+#if USE_TCACHE
+            case 12:
+              if (!__builtin_expect (__libc_enable_secure, 0))
+                {
+                  if (memcmp (envline, "TCACHE_COUNT", 12) == 0)
+                    __libc_mallopt (M_TCACHE_COUNT, atoi (&envline[13]));
+                }
+	      break;
+#endif
             case 15:
               if (!__builtin_expect (__libc_enable_secure, 0))
                 {
@@ -381,6 +405,15 @@ ptmalloc_init (void)
                     __libc_mallopt (M_MMAP_THRESHOLD, atoi (&envline[16]));
                 }
               break;
+#if USE_TCACHE
+            case 21:
+              if (!__builtin_expect (__libc_enable_secure, 0))
+                {
+                  if (memcmp (envline, "TCACHE_UNSORTED_LIMIT", 21) == 0)
+                    __libc_mallopt (M_TCACHE_UNSORTED_LIMIT, atoi (&envline[22]));
+                }
+	      break;
+#endif
             default:
               break;
             }
diff --git a/malloc/malloc.c b/malloc/malloc.c
index 1621fb3..a653607 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -1736,6 +1736,9 @@ struct malloc_par
   size_t tcache_max;
   /* Maximum number of chunks in each bucket.  */
   size_t tcache_count;
+  /* Maximum number of chunks to remove from the unsorted list, which
+     don't match.  */
+  size_t tcache_unsorted_limit;
 #endif
 };
 
@@ -1778,7 +1781,8 @@ static struct malloc_par mp_ =
 #if USE_TCACHE
   ,
   .tcache_count = TCACHE_FILL_COUNT,
-  .tcache_max = TCACHE_IDX-1
+  .tcache_max = TCACHE_IDX-1,
+  .tcache_unsorted_limit = 0 /* No limit */
 #endif
 };
 
@@ -1786,6 +1790,7 @@ static struct malloc_par mp_ =
 #if USE_TCACHE
 #define M_TCACHE_COUNT  -9
 #define M_TCACHE_MAX  -10
+#define M_TCACHE_UNSORTED_LIMIT  -11
 #endif
 
 /* Maximum size of memory handled in fastbins.  */
@@ -3437,6 +3442,10 @@ _int_malloc (mstate av, size_t bytes)
   mchunkptr fwd;                    /* misc temp for linking */
   mchunkptr bck;                    /* misc temp for linking */
 
+#if USE_TCACHE
+  size_t tcache_unsorted_count;	    /* count of unsorted chunks processed */
+#endif
+
   const char *errstr = NULL;
 
   /*
@@ -3637,6 +3646,8 @@ _int_malloc (mstate av, size_t bytes)
     tcache_nb = nb;
   size_t tc_idx = size2tidx (nb-SIZE_SZ);
   int return_cached = 0;
+
+  tcache_unsorted_count = 0;
 #endif
 
   for (;; )
@@ -3788,6 +3799,21 @@ _int_malloc (mstate av, size_t bytes)
           fwd->bk = victim;
           bck->fd = victim;
 
+#if USE_TCACHE
+      /* If we've processed as many chunks as we're allowed while
+	 filling the cache, return one of the cached ones.  */
+      tcache_unsorted_count ++;
+      if (return_cached
+	  && mp_.tcache_unsorted_limit > 0
+	  && tcache_unsorted_count > mp_.tcache_unsorted_limit)
+	{
+	  TCacheEntry *e = tcache.entries[tc_idx];
+	  tcache.entries[tc_idx] = e->next;
+	  tcache.counts[tc_idx] --;
+	  return (void *) e;
+	}
+#endif
+
 #define MAX_ITERS       10000
           if (++iters >= MAX_ITERS)
             break;
@@ -5067,6 +5093,34 @@ do_set_arena_max (size_t value)
   return 1;
 }
 
+#ifdef USE_TCACHE
+static inline int
+__always_inline
+do_set_tcache_max (size_t value)
+{
+  LIBC_PROBE (memory_mallopt_tcache_max, 2, value, mp_.tcache_max);
+  mp_.tcache_max = value;
+  return 1;
+}
+
+static inline int
+__always_inline
+do_set_tcache_count (size_t value)
+{
+  LIBC_PROBE (memory_mallopt_tcache_count, 2, value, mp_.tcache_count);
+  mp_.tcache_count = value;
+  return 1;
+}
+
+static inline int
+__always_inline
+do_set_tcache_unsorted_limit (size_t value)
+{
+  LIBC_PROBE (memory_mallopt_tcache_unsorted_limit, 2, value, mp_.tcache_unsorted_limit);
+  mp_.tcache_unsorted_limit = value;
+  return 1;
+}
+#endif
 
 int
 __libc_mallopt (int param_number, int value)
@@ -5130,22 +5184,20 @@ __libc_mallopt (int param_number, int value)
 #if USE_TCACHE
     case M_TCACHE_COUNT:
       if (value >= 0)
-        {
-          LIBC_PROBE (memory_mallopt_tcache_count, 2, value, mp_.tcache_count);
-          mp_.tcache_count = value;
-        }
+	do_set_tcache_max (value);
       break;
     case M_TCACHE_MAX:
       if (value >= 0)
         {
           value = size2tidx (value);
 	  if (value < TCACHE_IDX)
-	    {
-              LIBC_PROBE (memory_mallopt_tcache_max, 2, value, mp_.tcache_max);
-	      mp_.tcache_max = value;
-            }
+	    do_set_tcache_max (value);
         }
       break;
+    case M_TCACHE_UNSORTED_LIMIT:
+      if (value >= 0)
+	do_set_tcache_unsorted_limit (value);
+      break;
 #endif
     }
   __libc_lock_unlock (av->mutex);