diff options
Diffstat (limited to 'malloc')
98 files changed, 947 insertions, 626 deletions
diff --git a/malloc/Makefile b/malloc/Makefile index ccc96ef..83f6c87 100644 --- a/malloc/Makefile +++ b/malloc/Makefile @@ -1,4 +1,4 @@ -# Copyright (C) 1991-2024 Free Software Foundation, Inc. +# Copyright (C) 1991-2025 Free Software Foundation, Inc. # This file is part of the GNU C Library. # The GNU C Library is free software; you can redistribute it and/or @@ -63,7 +63,7 @@ tests := \ tst-realloc \ tst-reallocarray \ tst-safe-linking \ - tst-tcfree1 tst-tcfree2 tst-tcfree3 \ + tst-tcfree1 tst-tcfree2 tst-tcfree3 tst-tcfree4 \ tst-trim1 \ tst-valloc \ # tests @@ -147,6 +147,22 @@ tests-malloc-hugetlb1 = \ tests-malloc-hugetlb2 = \ $(filter-out $(tests-exclude-hugetlb2), $(tests)) +tests-exclude-largetcache = \ + tst-compathooks-off \ + tst-compathooks-on \ + tst-interpose-nothread \ + tst-interpose-static-nothread \ + tst-interpose-static-thread \ + tst-interpose-thread \ + tst-malloc-backtrace \ + tst-malloc-usable \ + tst-malloc-usable-tunables \ + tst-mallocstate \ +# tests-exclude-largetcache + +tests-malloc-largetcache = \ + $(filter-out $(tests-exclude-largetcache), $(tests)) + # -lmcheck needs __malloc_initialize_hook, which was deprecated in 2.24. ifeq ($(have-GLIBC_2.23)$(build-shared),yesyes) # Tests that don't play well with mcheck. They are either bugs in mcheck or @@ -219,32 +235,61 @@ libmemusage-inhibit-o = $(filter-out .os,$(object-suffixes)) libc_malloc_debug-routines = malloc-debug $(sysdep_malloc_debug_routines) libc_malloc_debug-inhibit-o = $(filter-out .os,$(object-suffixes)) -$(objpfx)tst-malloc-backtrace: $(shared-thread-library) -$(objpfx)tst-malloc-thread-exit: $(shared-thread-library) -$(objpfx)tst-malloc-thread-fail: $(shared-thread-library) -$(objpfx)tst-mallocfork3: $(shared-thread-library) -$(objpfx)tst-mallocfork3-mcheck: $(shared-thread-library) -$(objpfx)tst-malloc-fork-deadlock: $(shared-thread-library) -$(objpfx)tst-malloc-stats-cancellation: $(shared-thread-library) -$(objpfx)tst-malloc-backtrace-mcheck: $(shared-thread-library) -$(objpfx)tst-malloc-thread-exit-mcheck: $(shared-thread-library) -$(objpfx)tst-malloc-thread-fail-mcheck: $(shared-thread-library) -$(objpfx)tst-malloc-fork-deadlock-mcheck: $(shared-thread-library) -$(objpfx)tst-malloc-stats-cancellation-mcheck: $(shared-thread-library) -$(objpfx)tst-mallocfork3-malloc-check: $(shared-thread-library) -$(objpfx)tst-malloc-backtrace-malloc-check: $(shared-thread-library) -$(objpfx)tst-malloc-thread-exit-malloc-check: $(shared-thread-library) -$(objpfx)tst-malloc-thread-fail-malloc-check: $(shared-thread-library) -$(objpfx)tst-malloc-fork-deadlock-malloc-check: $(shared-thread-library) -$(objpfx)tst-malloc-stats-cancellation-malloc-check: $(shared-thread-library) -$(objpfx)tst-malloc-thread-exit-malloc-hugetlb1: $(shared-thread-library) -$(objpfx)tst-malloc-thread-fail-malloc-hugetlb1: $(shared-thread-library) -$(objpfx)tst-malloc-fork-deadlock-malloc-hugetlb1: $(shared-thread-library) -$(objpfx)tst-malloc-stats-cancellation-malloc-hugetlb1: $(shared-thread-library) -$(objpfx)tst-malloc-thread-exit-malloc-hugetlb2: $(shared-thread-library) -$(objpfx)tst-malloc-thread-fail-malloc-hugetlb2: $(shared-thread-library) -$(objpfx)tst-malloc-fork-deadlock-malloc-hugetlb2: $(shared-thread-library) -$(objpfx)tst-malloc-stats-cancellation-malloc-hugetlb2: $(shared-thread-library) +tests-link-with-libpthread = \ + tst-aligned-alloc-random-thread \ + tst-aligned-alloc-random-thread-cross \ + tst-aligned-alloc-random-thread-cross-malloc-check \ + tst-aligned-alloc-random-thread-cross-malloc-hugetlb1 \ + tst-aligned-alloc-random-thread-cross-malloc-hugetlb2 \ + tst-aligned-alloc-random-thread-malloc-check \ + tst-aligned-alloc-random-thread-malloc-hugetlb1 \ + tst-aligned-alloc-random-thread-malloc-hugetlb2 \ + tst-interpose-thread \ + tst-interpose-thread-malloc-check \ + tst-interpose-thread-mcheck \ + tst-malloc-backtrace \ + tst-malloc-backtrace-malloc-check \ + tst-malloc-backtrace-mcheck \ + tst-malloc-fork-deadlock \ + tst-malloc-fork-deadlock-malloc-check \ + tst-malloc-fork-deadlock-malloc-hugetlb1 \ + tst-malloc-fork-deadlock-malloc-hugetlb2 \ + tst-malloc-fork-deadlock-mcheck \ + tst-malloc-stats-cancellation \ + tst-malloc-stats-cancellation-malloc-check \ + tst-malloc-stats-cancellation-malloc-hugetlb1 \ + tst-malloc-stats-cancellation-malloc-hugetlb2 \ + tst-malloc-stats-cancellation-mcheck \ + tst-malloc-tcache-leak \ + tst-malloc-thread-exit \ + tst-malloc-thread-exit-malloc-check \ + tst-malloc-thread-exit-malloc-hugetlb1 \ + tst-malloc-thread-exit-malloc-hugetlb2 \ + tst-malloc-thread-exit-mcheck \ + tst-malloc-thread-fail \ + tst-malloc-thread-fail-malloc-check \ + tst-malloc-thread-fail-malloc-hugetlb1 \ + tst-malloc-thread-fail-malloc-hugetlb2 \ + tst-malloc-thread-fail-mcheck \ + tst-malloc_info \ + tst-malloc_info-malloc-check \ + tst-malloc_info-malloc-hugetlb1 \ + tst-malloc_info-malloc-hugetlb2 \ + tst-malloc_info-mcheck \ + tst-mallocfork2 \ + tst-mallocfork2-malloc-check \ + tst-mallocfork3 \ + tst-mallocfork3-malloc-check \ + tst-mallocfork3-mcheck \ + tst-memalign-3 \ + tst-memalign-3-malloc-hugetlb1 \ + tst-memalign-3-malloc-hugetlb2 \ +# tests-link-with-libpthread + +$(addprefix $(objpfx), $(tests-link-with-libpthread)): $(shared-thread-library) +# Not all these tests are actually built, see tests-exclude-largetcache. +$(tests-link-with-libpthread:%=$(objpfx)%-malloc-largetcache): \ + $(shared-thread-library) # These should be removed by `make clean'. extra-objs = mcheck-init.o libmcheck.a @@ -372,12 +417,9 @@ $(objpfx)tst-interpose-nothread: $(objpfx)tst-interpose-aux-nothread.o $(objpfx)tst-interpose-nothread-mcheck: $(objpfx)tst-interpose-aux-nothread.o $(objpfx)tst-interpose-nothread-malloc-check: \ $(objpfx)tst-interpose-aux-nothread.o -$(objpfx)tst-interpose-thread: \ - $(objpfx)tst-interpose-aux-thread.o $(shared-thread-library) -$(objpfx)tst-interpose-thread-mcheck: \ - $(objpfx)tst-interpose-aux-thread.o $(shared-thread-library) -$(objpfx)tst-interpose-thread-malloc-check: \ - $(objpfx)tst-interpose-aux-thread.o $(shared-thread-library) +$(objpfx)tst-interpose-thread: $(objpfx)tst-interpose-aux-thread.o +$(objpfx)tst-interpose-thread-mcheck: $(objpfx)tst-interpose-aux-thread.o +$(objpfx)tst-interpose-thread-malloc-check: $(objpfx)tst-interpose-aux-thread.o $(objpfx)tst-interpose-static-nothread: $(objpfx)tst-interpose-aux-nothread.o $(objpfx)tst-interpose-static-thread: \ $(objpfx)tst-interpose-aux-thread.o $(static-thread-library) @@ -394,18 +436,6 @@ $(objpfx)tst-dynarray-fail-mem.out: $(objpfx)tst-dynarray-fail.out $(common-objpfx)malloc/mtrace $(objpfx)tst-dynarray-fail.mtrace > $@; \ $(evaluate-test) -$(objpfx)tst-malloc-tcache-leak: $(shared-thread-library) -$(objpfx)tst-malloc_info: $(shared-thread-library) -$(objpfx)tst-mallocfork2: $(shared-thread-library) -$(objpfx)tst-malloc_info-mcheck: $(shared-thread-library) -$(objpfx)tst-malloc_info-malloc-check: $(shared-thread-library) -$(objpfx)tst-mallocfork2-malloc-check: $(shared-thread-library) -$(objpfx)tst-malloc_info-malloc-hugetlb1: $(shared-thread-library) -$(objpfx)tst-malloc_info-malloc-hugetlb2: $(shared-thread-library) -$(objpfx)tst-memalign-3: $(shared-thread-library) -$(objpfx)tst-memalign-3-malloc-hugetlb1: $(shared-thread-library) -$(objpfx)tst-memalign-3-malloc-hugetlb2: $(shared-thread-library) - tst-compathooks-on-ENV = LD_PRELOAD=$(objpfx)libc_malloc_debug.so tst-compathooks-on-mcheck-ENV = LD_PRELOAD=$(objpfx)libc_malloc_debug.so tst-compathooks-on-malloc-check-ENV = \ @@ -419,9 +449,7 @@ $(objpfx)tst-mallocstate: $(objpfx)libc_malloc_debug.so $(objpfx)tst-mallocstate-malloc-check: $(objpfx)libc_malloc_debug.so $(objpfx)tst-aligned-alloc-random.out: $(objpfx)tst-aligned_alloc-lib.so -$(objpfx)tst-aligned-alloc-random-thread: $(shared-thread-library) $(objpfx)tst-aligned-alloc-random-thread.out: $(objpfx)tst-aligned_alloc-lib.so -$(objpfx)tst-aligned-alloc-random-thread-cross: $(shared-thread-library) $(objpfx)tst-aligned-alloc-random-thread-cross.out: $(objpfx)tst-aligned_alloc-lib.so $(objpfx)tst-malloc-random.out: $(objpfx)tst-aligned_alloc-lib.so diff --git a/malloc/alloc_buffer_alloc_array.c b/malloc/alloc_buffer_alloc_array.c index 31cb9ec..1650330 100644 --- a/malloc/alloc_buffer_alloc_array.c +++ b/malloc/alloc_buffer_alloc_array.c @@ -1,5 +1,5 @@ /* Array allocation from a fixed-size buffer. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/alloc_buffer_allocate.c b/malloc/alloc_buffer_allocate.c index a121d1d..f3f6fd7 100644 --- a/malloc/alloc_buffer_allocate.c +++ b/malloc/alloc_buffer_allocate.c @@ -1,5 +1,5 @@ /* Allocate a fixed-size allocation buffer using malloc. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/alloc_buffer_copy_bytes.c b/malloc/alloc_buffer_copy_bytes.c index 69348e6..77f9537 100644 --- a/malloc/alloc_buffer_copy_bytes.c +++ b/malloc/alloc_buffer_copy_bytes.c @@ -1,5 +1,5 @@ /* Copy an array of bytes into the buffer. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/alloc_buffer_copy_string.c b/malloc/alloc_buffer_copy_string.c index 035e2f1..16068c7 100644 --- a/malloc/alloc_buffer_copy_string.c +++ b/malloc/alloc_buffer_copy_string.c @@ -1,5 +1,5 @@ /* Copy a string into the allocation buffer. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/alloc_buffer_create_failure.c b/malloc/alloc_buffer_create_failure.c index b456c63..cbafedc 100644 --- a/malloc/alloc_buffer_create_failure.c +++ b/malloc/alloc_buffer_create_failure.c @@ -1,5 +1,5 @@ /* Terminate the process as the result of an invalid allocation buffer. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/arena.c b/malloc/arena.c index cfb1ff8..90c526f 100644 --- a/malloc/arena.c +++ b/malloc/arena.c @@ -1,5 +1,5 @@ /* Malloc implementation for multiple threads without lock contention. - Copyright (C) 2001-2024 Free Software Foundation, Inc. + Copyright (C) 2001-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -40,19 +40,20 @@ mmap threshold, so that requests with a size just below that threshold can be fulfilled without creating too many heaps. */ -/* When huge pages are used to create new arenas, the maximum and minimum - size are based on the runtime defined huge page size. */ +/* HEAP_MAX_SIZE should be larger than the huge page size, otherwise heaps will + use not huge pages. It is a constant so arena_for_chunk() is efficient. */ -static inline size_t +static __always_inline size_t heap_min_size (void) { - return mp_.hp_pagesize == 0 ? HEAP_MIN_SIZE : mp_.hp_pagesize; + return mp_.hp_pagesize == 0 || mp_.hp_pagesize > HEAP_MAX_SIZE + ? HEAP_MIN_SIZE : mp_.hp_pagesize; } -static inline size_t +static __always_inline size_t heap_max_size (void) { - return mp_.hp_pagesize == 0 ? HEAP_MAX_SIZE : mp_.hp_pagesize * 4; + return HEAP_MAX_SIZE; } /***************************************************************************/ @@ -112,9 +113,6 @@ static mstate free_list; acquired. */ __libc_lock_define_initialized (static, list_lock); -/* Already initialized? */ -static bool __malloc_initialized = false; - /**************************************************************************/ @@ -140,14 +138,14 @@ static bool __malloc_initialized = false; /* find the heap and corresponding arena for a given ptr */ -static inline heap_info * +static __always_inline heap_info * heap_for_ptr (void *ptr) { size_t max_size = heap_max_size (); return PTR_ALIGN_DOWN (ptr, max_size); } -static inline struct malloc_state * +static __always_inline struct malloc_state * arena_for_chunk (mchunkptr ptr) { return chunk_main_arena (ptr) ? &main_arena : heap_for_ptr (ptr)->ar_ptr; @@ -167,9 +165,6 @@ arena_for_chunk (mchunkptr ptr) void __malloc_fork_lock_parent (void) { - if (!__malloc_initialized) - return; - /* We do not acquire free_list_lock here because we completely reconstruct free_list in __malloc_fork_unlock_child. */ @@ -187,9 +182,6 @@ __malloc_fork_lock_parent (void) void __malloc_fork_unlock_parent (void) { - if (!__malloc_initialized) - return; - for (mstate ar_ptr = &main_arena;; ) { __libc_lock_unlock (ar_ptr->mutex); @@ -203,9 +195,6 @@ __malloc_fork_unlock_parent (void) void __malloc_fork_unlock_child (void) { - if (!__malloc_initialized) - return; - /* Push all arenas to the free list, except thread_arena, which is attached to the current thread. */ __libc_lock_init (free_list_lock); @@ -231,8 +220,8 @@ __malloc_fork_unlock_child (void) } #define TUNABLE_CALLBACK_FNDECL(__name, __type) \ -static inline int do_ ## __name (__type value); \ -static void \ +static __always_inline int do_ ## __name (__type value); \ +static void \ TUNABLE_CALLBACK (__name) (tunable_val_t *valp) \ { \ __type value = (__type) (valp)->numval; \ @@ -258,14 +247,9 @@ TUNABLE_CALLBACK_FNDECL (set_hugetlb, size_t) static void tcache_key_initialize (void); #endif -static void -ptmalloc_init (void) +void +__ptmalloc_init (void) { - if (__malloc_initialized) - return; - - __malloc_initialized = true; - #if USE_TCACHE tcache_key_initialize (); #endif @@ -313,7 +297,7 @@ ptmalloc_init (void) TUNABLE_GET (mxfast, size_t, TUNABLE_CALLBACK (set_mxfast)); TUNABLE_GET (hugetlb, size_t, TUNABLE_CALLBACK (set_hugetlb)); - if (mp_.hp_pagesize > 0) + if (mp_.hp_pagesize > 0 && mp_.hp_pagesize <= heap_max_size ()) { /* Force mmap for main arena instead of sbrk, so MAP_HUGETLB is always tried. Also tune the mmap threshold, so allocation smaller than the @@ -389,7 +373,7 @@ alloc_new_heap (size_t size, size_t top_pad, size_t pagesize, else if (size + top_pad <= max_size) size += top_pad; else if (size > max_size) - return 0; + return NULL; else size = max_size; size = ALIGN_UP (size, pagesize); @@ -411,7 +395,7 @@ alloc_new_heap (size_t size, size_t top_pad, size_t pagesize, } if (p2 == MAP_FAILED) { - p1 = (char *) MMAP (0, max_size << 1, PROT_NONE, mmap_flags); + p1 = (char *) MMAP (NULL, max_size << 1, PROT_NONE, mmap_flags); if (p1 != MAP_FAILED) { p2 = (char *) (((uintptr_t) p1 + (max_size - 1)) @@ -427,21 +411,21 @@ alloc_new_heap (size_t size, size_t top_pad, size_t pagesize, { /* Try to take the chance that an allocation of only max_size is already aligned. */ - p2 = (char *) MMAP (0, max_size, PROT_NONE, mmap_flags); + p2 = (char *) MMAP (NULL, max_size, PROT_NONE, mmap_flags); if (p2 == MAP_FAILED) - return 0; + return NULL; if ((unsigned long) p2 & (max_size - 1)) { __munmap (p2, max_size); - return 0; + return NULL; } } } if (__mprotect (p2, size, mtag_mmap_flags | PROT_READ | PROT_WRITE) != 0) { __munmap (p2, max_size); - return 0; + return NULL; } /* Only considere the actual usable range. */ @@ -460,7 +444,7 @@ alloc_new_heap (size_t size, size_t top_pad, size_t pagesize, static heap_info * new_heap (size_t size, size_t top_pad) { - if (__glibc_unlikely (mp_.hp_pagesize != 0)) + if (mp_.hp_pagesize != 0 && mp_.hp_pagesize <= heap_max_size ()) { heap_info *h = alloc_new_heap (size, top_pad, mp_.hp_pagesize, mp_.hp_flags); @@ -644,7 +628,7 @@ _int_new_arena (size_t size) to deal with the large request via mmap_chunk(). */ h = new_heap (sizeof (*h) + sizeof (*a) + MALLOC_ALIGNMENT, mp_.top_pad); if (!h) - return 0; + return NULL; } a = h->ar_ptr = (mstate) (h + 1); malloc_init_state (a); diff --git a/malloc/dynarray-skeleton.c b/malloc/dynarray-skeleton.c index 7b3c1d9..fa73f5a 100644 --- a/malloc/dynarray-skeleton.c +++ b/malloc/dynarray-skeleton.c @@ -1,5 +1,5 @@ /* Type-safe arrays which grow dynamically. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/dynarray.h b/malloc/dynarray.h index eada6c6..ae4b410 100644 --- a/malloc/dynarray.h +++ b/malloc/dynarray.h @@ -1,5 +1,5 @@ /* Type-safe arrays which grow dynamically. Shared definitions. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/dynarray_at_failure.c b/malloc/dynarray_at_failure.c index e8be8ce..b8a44b7 100644 --- a/malloc/dynarray_at_failure.c +++ b/malloc/dynarray_at_failure.c @@ -1,5 +1,5 @@ /* Report an dynamic array index out of bounds condition. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/dynarray_emplace_enlarge.c b/malloc/dynarray_emplace_enlarge.c index 55d4269..137e46f 100644 --- a/malloc/dynarray_emplace_enlarge.c +++ b/malloc/dynarray_emplace_enlarge.c @@ -1,5 +1,5 @@ /* Increase the size of a dynamic array in preparation of an emplace operation. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/dynarray_finalize.c b/malloc/dynarray_finalize.c index cd81677..8267aad 100644 --- a/malloc/dynarray_finalize.c +++ b/malloc/dynarray_finalize.c @@ -1,5 +1,5 @@ /* Copy the dynamically-allocated area to an explicitly-sized heap allocation. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/dynarray_resize.c b/malloc/dynarray_resize.c index 51197b4..87a8e9c 100644 --- a/malloc/dynarray_resize.c +++ b/malloc/dynarray_resize.c @@ -1,5 +1,5 @@ /* Increase the size of a dynamic array. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/dynarray_resize_clear.c b/malloc/dynarray_resize_clear.c index 85b5b46..c9bdf23 100644 --- a/malloc/dynarray_resize_clear.c +++ b/malloc/dynarray_resize_clear.c @@ -1,5 +1,5 @@ /* Increase the size of a dynamic array and clear the new part. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/hooks.c b/malloc/hooks.c index 79e6442..225824a 100644 --- a/malloc/hooks.c +++ b/malloc/hooks.c @@ -1,5 +1,5 @@ /* Compatibility code for malloc debugging and state management. - Copyright (C) 2001-2024 Free Software Foundation, Inc. + Copyright (C) 2001-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/malloc-check.c b/malloc/malloc-check.c index da1158b..f5ca5fb 100644 --- a/malloc/malloc-check.c +++ b/malloc/malloc-check.c @@ -1,5 +1,5 @@ /* glibc.malloc.check implementation. - Copyright (C) 2001-2024 Free Software Foundation, Inc. + Copyright (C) 2001-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -111,7 +111,7 @@ mem2chunk_check (void *mem, unsigned char **magic_p) INTERNAL_SIZE_T sz, c; unsigned char magic; - if (!aligned_OK (mem)) + if (misaligned_mem (mem)) return NULL; p = mem2chunk (mem); @@ -235,7 +235,7 @@ free_check (void *mem) { /* Mark the chunk as belonging to the library again. */ (void)tag_region (chunk2mem (p), memsize (p)); - _int_free (&main_arena, p, 1); + _int_free_chunk (&main_arena, p, chunksize (p), 1); __libc_lock_unlock (main_arena.mutex); } __set_errno (err); @@ -245,7 +245,7 @@ static void * realloc_check (void *oldmem, size_t bytes) { INTERNAL_SIZE_T chnb; - void *newmem = 0; + void *newmem = NULL; unsigned char *magic_p; size_t rb; @@ -254,7 +254,7 @@ realloc_check (void *oldmem, size_t bytes) __set_errno (ENOMEM); return NULL; } - if (oldmem == 0) + if (oldmem == NULL) return malloc_check (bytes); if (bytes == 0) @@ -389,7 +389,7 @@ initialize_malloc_check (void) { /* This is the copy of the malloc initializer that we pulled in along with malloc-check. This does not affect any of the libc malloc structures. */ - ptmalloc_init (); + __ptmalloc_init (); TUNABLE_GET (check, int32_t, TUNABLE_CALLBACK (set_mallopt_check)); return __is_malloc_debug_enabled (MALLOC_CHECK_HOOK); } diff --git a/malloc/malloc-debug.c b/malloc/malloc-debug.c index c9b4166..8bcb565 100644 --- a/malloc/malloc-debug.c +++ b/malloc/malloc-debug.c @@ -1,5 +1,5 @@ /* Malloc debug DSO. - Copyright (C) 2021-2024 Free Software Foundation, Inc. + Copyright (C) 2021-2025 Free Software Foundation, Inc. Copyright The GNU Toolchain Authors. This file is part of the GNU C Library. @@ -169,7 +169,7 @@ static void * __debug_malloc (size_t bytes) { void *(*hook) (size_t, const void *) = atomic_forced_read (__malloc_hook); - if (__builtin_expect (hook != NULL, 0)) + if (__glibc_unlikely (hook != NULL)) return (*hook)(bytes, RETURN_ADDRESS (0)); void *victim = NULL; @@ -193,7 +193,7 @@ static void __debug_free (void *mem) { void (*hook) (void *, const void *) = atomic_forced_read (__free_hook); - if (__builtin_expect (hook != NULL, 0)) + if (__glibc_unlikely (hook != NULL)) { (*hook)(mem, RETURN_ADDRESS (0)); return; @@ -218,7 +218,7 @@ __debug_realloc (void *oldmem, size_t bytes) { void *(*hook) (void *, size_t, const void *) = atomic_forced_read (__realloc_hook); - if (__builtin_expect (hook != NULL, 0)) + if (__glibc_unlikely (hook != NULL)) return (*hook)(oldmem, bytes, RETURN_ADDRESS (0)); size_t orig_bytes = bytes, oldsize = 0; @@ -272,7 +272,7 @@ _debug_mid_memalign (size_t alignment, size_t bytes, const void *address) { void *(*hook) (size_t, size_t, const void *) = atomic_forced_read (__memalign_hook); - if (__builtin_expect (hook != NULL, 0)) + if (__glibc_unlikely (hook != NULL)) return (*hook)(alignment, bytes, address); void *victim = NULL; @@ -371,7 +371,7 @@ __debug_calloc (size_t nmemb, size_t size) } void *(*hook) (size_t, const void *) = atomic_forced_read (__malloc_hook); - if (__builtin_expect (hook != NULL, 0)) + if (__glibc_unlikely (hook != NULL)) { void *mem = (*hook)(bytes, RETURN_ADDRESS (0)); diff --git a/malloc/malloc-internal.h b/malloc/malloc-internal.h index cba0343..0f1b3a1 100644 --- a/malloc/malloc-internal.h +++ b/malloc/malloc-internal.h @@ -1,5 +1,5 @@ /* Internal declarations for malloc, for use within libc. - Copyright (C) 2016-2024 Free Software Foundation, Inc. + Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -23,6 +23,7 @@ #include <malloc-sysdep.h> #include <malloc-size.h> #include <malloc-hugepages.h> +#include <calloc-clear-memory.h> /* Called in the parent process before a fork. */ void __malloc_fork_lock_parent (void) attribute_hidden; @@ -39,4 +40,7 @@ void __malloc_arena_thread_freeres (void) attribute_hidden; /* Activate a standard set of debugging hooks. */ void __malloc_check_init (void) attribute_hidden; +/* Initialize malloc. */ +void __ptmalloc_init (void) attribute_hidden; + #endif /* _MALLOC_INTERNAL_H */ diff --git a/malloc/malloc.c b/malloc/malloc.c index bcb6e5b..6da40ad 100644 --- a/malloc/malloc.c +++ b/malloc/malloc.c @@ -1,5 +1,5 @@ /* Malloc implementation for multiple threads without lock contention. - Copyright (C) 1996-2024 Free Software Foundation, Inc. + Copyright (C) 1996-2025 Free Software Foundation, Inc. Copyright The GNU Toolchain Authors. This file is part of the GNU C Library. @@ -291,16 +291,18 @@ #if USE_TCACHE /* We want 64 entries. This is an arbitrary limit, which tunables can reduce. */ -# define TCACHE_MAX_BINS 64 -# define MAX_TCACHE_SIZE tidx2usize (TCACHE_MAX_BINS-1) +# define TCACHE_SMALL_BINS 64 +# define TCACHE_LARGE_BINS 12 /* Up to 4M chunks */ +# define TCACHE_MAX_BINS (TCACHE_SMALL_BINS + TCACHE_LARGE_BINS) +# define MAX_TCACHE_SMALL_SIZE tidx2usize (TCACHE_MAX_BINS-1) /* Only used to pre-fill the tunables. */ # define tidx2usize(idx) (((size_t) idx) * MALLOC_ALIGNMENT + MINSIZE - SIZE_SZ) /* When "x" is from chunksize(). */ -# define csize2tidx(x) (((x) - MINSIZE + MALLOC_ALIGNMENT - 1) / MALLOC_ALIGNMENT) +# define csize2tidx(x) (((x) - MINSIZE) / MALLOC_ALIGNMENT) /* When "x" is a user-provided size. */ -# define usize2tidx(x) csize2tidx (request2size (x)) +# define usize2tidx(x) csize2tidx (checked_request2size (x)) /* With rounding and alignment, the bins are... idx 0 bytes 0..24 (64-bit) or 0..12 (32-bit) @@ -313,7 +315,7 @@ # define TCACHE_FILL_COUNT 7 /* Maximum chunks in tcache bins for tunables. This value must fit the range - of tcache->counts[] entries, else they may overflow. */ + of tcache->num_slots[] entries, else they may overflow. */ # define MAX_TCACHE_COUNT UINT16_MAX #endif @@ -369,7 +371,7 @@ #include "morecore.c" #define MORECORE (*__glibc_morecore) -#define MORECORE_FAILURE 0 +#define MORECORE_FAILURE NULL /* Memory tagging. */ @@ -588,9 +590,12 @@ tag_at (void *ptr) differs across systems, but is in all cases less than the maximum representable value of a size_t. */ -void* __libc_malloc(size_t); +void *__libc_malloc (size_t); libc_hidden_proto (__libc_malloc) +static void *__libc_calloc2 (size_t); +static void *__libc_malloc2 (size_t); + /* free(void* p) Releases the chunk of memory pointed to by p, that had been previously @@ -1086,7 +1091,7 @@ typedef struct malloc_chunk* mchunkptr; /* Internal routines. */ static void* _int_malloc(mstate, size_t); -static void _int_free(mstate, mchunkptr, int); +static void _int_free_chunk (mstate, mchunkptr, INTERNAL_SIZE_T, int); static void _int_free_merge_chunk (mstate, mchunkptr, INTERNAL_SIZE_T); static INTERNAL_SIZE_T _int_free_create_chunk (mstate, mchunkptr, INTERNAL_SIZE_T, @@ -1096,9 +1101,12 @@ static void* _int_realloc(mstate, mchunkptr, INTERNAL_SIZE_T, INTERNAL_SIZE_T); static void* _int_memalign(mstate, size_t, size_t); #if IS_IN (libc) -static void* _mid_memalign(size_t, size_t, void *); +static void* _mid_memalign(size_t, size_t); #endif +#if USE_TCACHE +static void malloc_printerr_tail(const char *str); +#endif static void malloc_printerr(const char *str) __attribute__ ((noreturn)); static void munmap_chunk(mchunkptr p); @@ -1271,7 +1279,6 @@ nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ sysmalloc: Returns untagged memory. _int_malloc: Returns untagged memory. - _int_free: Takes untagged memory. _int_memalign: Returns untagged memory. _int_memalign: Returns untagged memory. _mid_memalign: Returns tagged memory. @@ -1302,11 +1309,9 @@ nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ /* Check if m has acceptable alignment */ -#define aligned_OK(m) (((unsigned long)(m) & MALLOC_ALIGN_MASK) == 0) +#define misaligned_mem(m) ((uintptr_t)(m) & MALLOC_ALIGN_MASK) -#define misaligned_chunk(p) \ - ((uintptr_t)(MALLOC_ALIGNMENT == CHUNK_HDR_SZ ? (p) : chunk2mem (p)) \ - & MALLOC_ALIGN_MASK) +#define misaligned_chunk(p) (misaligned_mem( chunk2mem (p))) /* pad request bytes into a usable size -- internal version */ /* Note: This must be a macro that evaluates to a compile time constant @@ -1320,9 +1325,12 @@ nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ value is less than PTRDIFF_T. Returns the requested size or MINSIZE in case the value is less than MINSIZE, or 0 if any of the previous checks fail. */ -static inline size_t +static __always_inline size_t checked_request2size (size_t req) __nonnull (1) { + _Static_assert (PTRDIFF_MAX <= SIZE_MAX / 2, + "PTRDIFF_MAX is not more than half of SIZE_MAX"); + if (__glibc_unlikely (req > PTRDIFF_MAX)) return 0; @@ -1613,7 +1621,7 @@ unlink_chunk (mstate av, mchunkptr p) mchunkptr fd = p->fd; mchunkptr bk = p->bk; - if (__builtin_expect (fd->bk != p || bk->fd != p, 0)) + if (__glibc_unlikely (fd->bk != p || bk->fd != p)) malloc_printerr ("corrupted double-linked list"); fd->bk = bk; @@ -1780,7 +1788,7 @@ static uint8_t global_max_fast; global_max_fast = (((size_t) (s) <= MALLOC_ALIGN_MASK - SIZE_SZ) \ ? MIN_CHUNK_SIZE / 2 : ((s + SIZE_SZ) & ~MALLOC_ALIGN_MASK)) -static inline INTERNAL_SIZE_T +static __always_inline INTERNAL_SIZE_T get_max_fast (void) { /* Tell the GCC optimizers that global_max_fast is never larger @@ -1886,8 +1894,8 @@ struct malloc_par char *sbrk_base; #if USE_TCACHE - /* Maximum number of buckets to use. */ - size_t tcache_bins; + /* Maximum number of small buckets to use. */ + size_t tcache_small_bins; size_t tcache_max_bytes; /* Maximum number of chunks in each bucket. */ size_t tcache_count; @@ -1923,8 +1931,8 @@ static struct malloc_par mp_ = #if USE_TCACHE , .tcache_count = TCACHE_FILL_COUNT, - .tcache_bins = TCACHE_MAX_BINS, - .tcache_max_bytes = tidx2usize (TCACHE_MAX_BINS-1), + .tcache_small_bins = TCACHE_SMALL_BINS, + .tcache_max_bytes = MAX_TCACHE_SMALL_SIZE, .tcache_unsorted_limit = 0 /* No limit. */ #endif }; @@ -1932,7 +1940,7 @@ static struct malloc_par mp_ = /* Initialize a malloc_state struct. - This is called from ptmalloc_init () or from _int_new_arena () + This is called from __ptmalloc_init () or from _int_new_arena () when creating a new arena. */ @@ -2093,7 +2101,7 @@ do_check_chunk (mstate av, mchunkptr p) /* chunk is page-aligned */ assert (((prev_size (p) + sz) & (GLRO (dl_pagesize) - 1)) == 0); /* mem is aligned */ - assert (aligned_OK (chunk2mem (p))); + assert (!misaligned_chunk (p)); } } @@ -2117,7 +2125,7 @@ do_check_free_chunk (mstate av, mchunkptr p) if ((unsigned long) (sz) >= MINSIZE) { assert ((sz & MALLOC_ALIGN_MASK) == 0); - assert (aligned_OK (chunk2mem (p))); + assert (!misaligned_chunk (p)); /* ... matching footer field */ assert (prev_size (next_chunk (p)) == sz); /* ... and is fully consolidated */ @@ -2196,7 +2204,7 @@ do_check_remalloced_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T s) assert ((sz & MALLOC_ALIGN_MASK) == 0); assert ((unsigned long) (sz) >= MINSIZE); /* ... and alignment */ - assert (aligned_OK (chunk2mem (p))); + assert (!misaligned_chunk (p)); /* chunk is less than MINSIZE more than request */ assert ((long) (sz) - (long) (s) >= 0); assert ((long) (sz) - (long) (s + MINSIZE) < 0); @@ -2418,7 +2426,7 @@ sysmalloc_mmap (INTERNAL_SIZE_T nb, size_t pagesize, int extra_flags, mstate av) if ((unsigned long) (size) <= (unsigned long) (nb)) return MAP_FAILED; - char *mm = (char *) MMAP (0, size, + char *mm = (char *) MMAP (NULL, size, mtag_mmap_flags | PROT_READ | PROT_WRITE, extra_flags); if (mm == MAP_FAILED) @@ -2505,7 +2513,7 @@ sysmalloc_mmap_fallback (long int *s, INTERNAL_SIZE_T nb, if ((unsigned long) (size) <= (unsigned long) (nb)) return MORECORE_FAILURE; - char *mbrk = (char *) (MMAP (0, size, + char *mbrk = (char *) (MMAP (NULL, size, mtag_mmap_flags | PROT_READ | PROT_WRITE, extra_flags)); if (mbrk == MAP_FAILED) @@ -2581,7 +2589,7 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) /* There are no usable arenas and mmap also failed. */ if (av == NULL) - return 0; + return NULL; /* Record incoming configuration of top */ @@ -2644,7 +2652,7 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) CHUNK_HDR_SZ | PREV_INUSE); set_foot (chunk_at_offset (old_top, old_size), CHUNK_HDR_SZ); set_head (old_top, old_size | PREV_INUSE | NON_MAIN_ARENA); - _int_free (av, old_top, 1); + _int_free_chunk (av, old_top, chunksize (old_top), 1); } else { @@ -2741,7 +2749,7 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) if (brk != (char *) (MORECORE_FAILURE)) { - if (mp_.sbrk_base == 0) + if (mp_.sbrk_base == NULL) mp_.sbrk_base = brk; av->system_mem += size; @@ -2910,7 +2918,7 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) /* If possible, release the rest. */ if (old_size >= MINSIZE) { - _int_free (av, old_top, 1); + _int_free_chunk (av, old_top, chunksize (old_top), 1); } } } @@ -2940,7 +2948,7 @@ sysmalloc (INTERNAL_SIZE_T nb, mstate av) /* catch all failure paths */ __set_errno (ENOMEM); - return 0; + return NULL; } @@ -3078,13 +3086,13 @@ mremap_chunk (mchunkptr p, size_t new_size) MREMAP_MAYMOVE); if (cp == MAP_FAILED) - return 0; + return NULL; madvise_thp (cp, new_size); p = (mchunkptr) (cp + offset); - assert (aligned_OK (chunk2mem (p))); + assert (!misaligned_chunk (p)); assert (prev_size (p) == offset); set_head (p, (new_size - offset) | IS_MMAPPED); @@ -3112,12 +3120,13 @@ typedef struct tcache_entry /* There is one of these for each thread, which contains the per-thread cache (hence "tcache_perthread_struct"). Keeping - overall size low is mildly important. Note that COUNTS and ENTRIES - are redundant (we could have just counted the linked list each - time), this is for performance reasons. */ + overall size low is mildly important. The 'entries' field is linked list of + free blocks, while 'num_slots' contains the number of free blocks that can + be added. Each bin may allow a different maximum number of free blocks, + and can be disabled by initializing 'num_slots' to zero. */ typedef struct tcache_perthread_struct { - uint16_t counts[TCACHE_MAX_BINS]; + uint16_t num_slots[TCACHE_MAX_BINS]; tcache_entry *entries[TCACHE_MAX_BINS]; } tcache_perthread_struct; @@ -3140,8 +3149,8 @@ static void tcache_key_initialize (void) { /* We need to use the _nostatus version here, see BZ 29624. */ - if (__getrandom_nocancel_nostatus (&tcache_key, sizeof(tcache_key), - GRND_NONBLOCK) + if (__getrandom_nocancel_nostatus_direct (&tcache_key, sizeof(tcache_key), + GRND_NONBLOCK) != sizeof (tcache_key)) { tcache_key = random_bits (); @@ -3151,59 +3160,186 @@ tcache_key_initialize (void) } } +static __always_inline size_t +large_csize2tidx(size_t nb) +{ + size_t idx = TCACHE_SMALL_BINS + + __builtin_clz (MAX_TCACHE_SMALL_SIZE) + - __builtin_clz (nb); + return idx; +} + /* Caller must ensure that we know tc_idx is valid and there's room for more chunks. */ static __always_inline void -tcache_put (mchunkptr chunk, size_t tc_idx) +tcache_put_n (mchunkptr chunk, size_t tc_idx, tcache_entry **ep, bool mangled) { tcache_entry *e = (tcache_entry *) chunk2mem (chunk); - /* Mark this chunk as "in the tcache" so the test in _int_free will + /* Mark this chunk as "in the tcache" so the test in __libc_free will detect a double free. */ e->key = tcache_key; - e->next = PROTECT_PTR (&e->next, tcache->entries[tc_idx]); - tcache->entries[tc_idx] = e; - ++(tcache->counts[tc_idx]); + if (!mangled) + { + e->next = PROTECT_PTR (&e->next, *ep); + *ep = e; + } + else + { + e->next = PROTECT_PTR (&e->next, REVEAL_PTR (*ep)); + *ep = PROTECT_PTR (ep, e); + } + --(tcache->num_slots[tc_idx]); } /* Caller must ensure that we know tc_idx is valid and there's available chunks to remove. Removes chunk from the middle of the list. */ static __always_inline void * -tcache_get_n (size_t tc_idx, tcache_entry **ep) +tcache_get_n (size_t tc_idx, tcache_entry **ep, bool mangled) { tcache_entry *e; - if (ep == &(tcache->entries[tc_idx])) + if (!mangled) e = *ep; else e = REVEAL_PTR (*ep); - if (__glibc_unlikely (!aligned_OK (e))) + if (__glibc_unlikely (misaligned_mem (e))) malloc_printerr ("malloc(): unaligned tcache chunk detected"); - if (ep == &(tcache->entries[tc_idx])) - *ep = REVEAL_PTR (e->next); + void *ne = e == NULL ? NULL : REVEAL_PTR (e->next); + if (!mangled) + *ep = ne; else - *ep = PROTECT_PTR (ep, REVEAL_PTR (e->next)); + *ep = PROTECT_PTR (ep, ne); - --(tcache->counts[tc_idx]); + ++(tcache->num_slots[tc_idx]); e->key = 0; return (void *) e; } +static __always_inline void +tcache_put (mchunkptr chunk, size_t tc_idx) +{ + tcache_put_n (chunk, tc_idx, &tcache->entries[tc_idx], false); +} + /* Like the above, but removes from the head of the list. */ static __always_inline void * tcache_get (size_t tc_idx) { - return tcache_get_n (tc_idx, & tcache->entries[tc_idx]); + return tcache_get_n (tc_idx, & tcache->entries[tc_idx], false); +} + +static __always_inline tcache_entry ** +tcache_location_large (size_t nb, size_t tc_idx, bool *mangled) +{ + tcache_entry **tep = &(tcache->entries[tc_idx]); + tcache_entry *te = *tep; + while (te != NULL + && __glibc_unlikely (chunksize (mem2chunk (te)) < nb)) + { + tep = & (te->next); + te = REVEAL_PTR (te->next); + *mangled = true; + } + + return tep; +} + +static __always_inline void +tcache_put_large (mchunkptr chunk, size_t tc_idx) +{ + tcache_entry **entry; + bool mangled = false; + entry = tcache_location_large (chunksize (chunk), tc_idx, &mangled); + + return tcache_put_n (chunk, tc_idx, entry, mangled); +} + +static __always_inline void * +tcache_get_large (size_t tc_idx, size_t nb) +{ + tcache_entry **entry; + bool mangled = false; + entry = tcache_location_large (nb, tc_idx, &mangled); + + if ((mangled && REVEAL_PTR (*entry) == NULL) + || (!mangled && *entry == NULL)) + return NULL; + + return tcache_get_n (tc_idx, entry, mangled); } -/* Iterates through the tcache linked list. */ -static __always_inline tcache_entry * -tcache_next (tcache_entry *e) +static void tcache_init (void); + +static __always_inline void * +tcache_get_align (size_t nb, size_t alignment) { - return (tcache_entry *) REVEAL_PTR (e->next); + if (nb < mp_.tcache_max_bytes) + { + if (__glibc_unlikely (tcache == NULL)) + { + tcache_init (); + return NULL; + } + + size_t tc_idx = csize2tidx (nb); + if (__glibc_unlikely (tc_idx >= TCACHE_SMALL_BINS)) + tc_idx = large_csize2tidx (nb); + + /* The tcache itself isn't encoded, but the chain is. */ + tcache_entry **tep = & tcache->entries[tc_idx]; + tcache_entry *te = *tep; + bool mangled = false; + size_t csize; + + while (te != NULL + && ((csize = chunksize (mem2chunk (te))) < nb + || (csize == nb + && !PTR_IS_ALIGNED (te, alignment)))) + { + tep = & (te->next); + te = REVEAL_PTR (te->next); + mangled = true; + } + + if (te != NULL + && csize == nb + && PTR_IS_ALIGNED (te, alignment)) + return tag_new_usable (tcache_get_n (tc_idx, tep, mangled)); + } + return NULL; +} + +/* Verify if the suspicious tcache_entry is double free. + It's not expected to execute very often, mark it as noinline. */ +static __attribute__ ((noinline)) void +tcache_double_free_verify (tcache_entry *e) +{ + tcache_entry *tmp; + for (size_t tc_idx = 0; tc_idx < TCACHE_MAX_BINS; ++tc_idx) + { + size_t cnt = 0; + LIBC_PROBE (memory_tcache_double_free, 2, e, tc_idx); + for (tmp = tcache->entries[tc_idx]; + tmp; + tmp = REVEAL_PTR (tmp->next), ++cnt) + { + if (cnt >= mp_.tcache_count) + malloc_printerr ("free(): too many chunks detected in tcache"); + if (__glibc_unlikely (misaligned_mem (tmp))) + malloc_printerr ("free(): unaligned chunk detected in tcache 2"); + if (tmp == e) + malloc_printerr ("free(): double free detected in tcache 2"); + } + } + /* No double free detected - it might be in a tcache of another thread, + or user data that happens to match the key. Since we are not sure, + clear the key and retry freeing it. */ + e->key = 0; + __libc_free (e); } static void @@ -3227,7 +3363,7 @@ tcache_thread_shutdown (void) while (tcache_tmp->entries[i]) { tcache_entry *e = tcache_tmp->entries[i]; - if (__glibc_unlikely (!aligned_OK (e))) + if (__glibc_unlikely (misaligned_mem (e))) malloc_printerr ("tcache_thread_shutdown(): " "unaligned tcache chunk detected"); tcache_tmp->entries[i] = REVEAL_PTR (e->next); @@ -3238,47 +3374,45 @@ tcache_thread_shutdown (void) __libc_free (tcache_tmp); } +/* Initialize tcache. In the rare case there isn't any memory available, + later calls will retry initialization. */ static void -tcache_init(void) +tcache_init (void) { - mstate ar_ptr; - void *victim = 0; - const size_t bytes = sizeof (tcache_perthread_struct); - if (tcache_shutting_down) return; - arena_get (ar_ptr, bytes); - victim = _int_malloc (ar_ptr, bytes); - if (!victim && ar_ptr != NULL) - { - ar_ptr = arena_get_retry (ar_ptr, bytes); - victim = _int_malloc (ar_ptr, bytes); - } - + /* Check minimum mmap chunk is larger than max tcache size. This means + mmap chunks with their different layout are never added to tcache. */ + if (MAX_TCACHE_SMALL_SIZE >= GLRO (dl_pagesize) / 2) + malloc_printerr ("max tcache size too large"); - if (ar_ptr != NULL) - __libc_lock_unlock (ar_ptr->mutex); + size_t bytes = sizeof (tcache_perthread_struct); + tcache = (tcache_perthread_struct *) __libc_malloc2 (bytes); - /* In a low memory situation, we may not be able to allocate memory - - in which case, we just keep trying later. However, we - typically do this very early, so either there is sufficient - memory, or there isn't enough memory to do non-trivial - allocations anyway. */ - if (victim) + if (tcache != NULL) { - tcache = (tcache_perthread_struct *) victim; - memset (tcache, 0, sizeof (tcache_perthread_struct)); + memset (tcache, 0, bytes); + for (int i = 0; i < TCACHE_MAX_BINS; i++) + tcache->num_slots[i] = mp_.tcache_count; } +} +static void * __attribute_noinline__ +tcache_calloc_init (size_t bytes) +{ + tcache_init (); + return __libc_calloc2 (bytes); } -# define MAYBE_INIT_TCACHE() \ - if (__glibc_unlikely (tcache == NULL)) \ - tcache_init(); +static void * __attribute_noinline__ +tcache_malloc_init (size_t bytes) +{ + tcache_init (); + return __libc_malloc2 (bytes); +} #else /* !USE_TCACHE */ -# define MAYBE_INIT_TCACHE() static void tcache_thread_shutdown (void) @@ -3289,40 +3423,13 @@ tcache_thread_shutdown (void) #endif /* !USE_TCACHE */ #if IS_IN (libc) -void * -__libc_malloc (size_t bytes) + +static void * __attribute_noinline__ +__libc_malloc2 (size_t bytes) { mstate ar_ptr; void *victim; - _Static_assert (PTRDIFF_MAX <= SIZE_MAX / 2, - "PTRDIFF_MAX is not more than half of SIZE_MAX"); - - if (!__malloc_initialized) - ptmalloc_init (); -#if USE_TCACHE - /* int_free also calls request2size, be careful to not pad twice. */ - size_t tbytes = checked_request2size (bytes); - if (tbytes == 0) - { - __set_errno (ENOMEM); - return NULL; - } - size_t tc_idx = csize2tidx (tbytes); - - MAYBE_INIT_TCACHE (); - - DIAG_PUSH_NEEDS_COMMENT; - if (tc_idx < mp_.tcache_bins - && tcache != NULL - && tcache->counts[tc_idx] > 0) - { - victim = tcache_get (tc_idx); - return tag_new_usable (victim); - } - DIAG_POP_NEEDS_COMMENT; -#endif - if (SINGLE_THREAD_P) { victim = tag_new_usable (_int_malloc (&main_arena, bytes)); @@ -3352,15 +3459,49 @@ __libc_malloc (size_t bytes) ar_ptr == arena_for_chunk (mem2chunk (victim))); return victim; } + +void * +__libc_malloc (size_t bytes) +{ +#if USE_TCACHE + size_t nb = checked_request2size (bytes); + if (nb == 0) + { + __set_errno (ENOMEM); + return NULL; + } + + if (nb < mp_.tcache_max_bytes) + { + size_t tc_idx = csize2tidx (nb); + if(__glibc_unlikely (tcache == NULL)) + return tcache_malloc_init (bytes); + + if (__glibc_likely (tc_idx < TCACHE_SMALL_BINS)) + { + if (tcache->entries[tc_idx] != NULL) + return tag_new_usable (tcache_get (tc_idx)); + } + else + { + tc_idx = large_csize2tidx (nb); + void *victim = tcache_get_large (tc_idx, nb); + if (victim != NULL) + return tag_new_usable (victim); + } + } +#endif + + return __libc_malloc2 (bytes); +} libc_hidden_def (__libc_malloc) void __libc_free (void *mem) { - mstate ar_ptr; mchunkptr p; /* chunk corresponding to mem */ - if (mem == 0) /* free(0) has no effect */ + if (mem == NULL) /* free(0) has no effect */ return; /* Quickly check that the freed pointer matches the tag for the memory. @@ -3368,37 +3509,51 @@ __libc_free (void *mem) if (__glibc_unlikely (mtag_enabled)) *(volatile char *)mem; - int err = errno; - p = mem2chunk (mem); - if (chunk_is_mmapped (p)) /* release mmapped memory. */ - { - /* See if the dynamic brk/mmap threshold needs adjusting. - Dumped fake mmapped chunks do not affect the threshold. */ - if (!mp_.no_dyn_threshold - && chunksize_nomask (p) > mp_.mmap_threshold - && chunksize_nomask (p) <= DEFAULT_MMAP_THRESHOLD_MAX) - { - mp_.mmap_threshold = chunksize (p); - mp_.trim_threshold = 2 * mp_.mmap_threshold; - LIBC_PROBE (memory_mallopt_free_dyn_thresholds, 2, - mp_.mmap_threshold, mp_.trim_threshold); - } - munmap_chunk (p); - } - else + /* Mark the chunk as belonging to the library again. */ + tag_region (chunk2mem (p), memsize (p)); + + INTERNAL_SIZE_T size = chunksize (p); + + if (__glibc_unlikely (misaligned_chunk (p))) + return malloc_printerr_tail ("free(): invalid pointer"); + + check_inuse_chunk (arena_for_chunk (p), p); + +#if USE_TCACHE + if (__glibc_likely (size < mp_.tcache_max_bytes && tcache != NULL)) { - MAYBE_INIT_TCACHE (); + /* Check to see if it's already in the tcache. */ + tcache_entry *e = (tcache_entry *) chunk2mem (p); - /* Mark the chunk as belonging to the library again. */ - (void)tag_region (chunk2mem (p), memsize (p)); + /* Check for double free - verify if the key matches. */ + if (__glibc_unlikely (e->key == tcache_key)) + return tcache_double_free_verify (e); - ar_ptr = arena_for_chunk (p); - _int_free (ar_ptr, p, 0); + size_t tc_idx = csize2tidx (size); + if (__glibc_likely (tc_idx < TCACHE_SMALL_BINS)) + { + if (__glibc_likely (tcache->num_slots[tc_idx] != 0)) + return tcache_put (p, tc_idx); + } + else + { + tc_idx = large_csize2tidx (size); + if (size >= MINSIZE + && !chunk_is_mmapped (p) + && __glibc_likely (tcache->num_slots[tc_idx] != 0)) + return tcache_put_large (p, tc_idx); + } } +#endif + + /* Check size >= MINSIZE and p + size does not overflow. */ + if (__glibc_unlikely (__builtin_add_overflow_p ((uintptr_t) p, size - MINSIZE, + (uintptr_t) 0))) + return malloc_printerr_tail ("free(): invalid size"); - __set_errno (err); + _int_free_chunk (arena_for_chunk (p), p, size, 0); } libc_hidden_def (__libc_free) @@ -3410,18 +3565,15 @@ __libc_realloc (void *oldmem, size_t bytes) void *newp; /* chunk to return */ - if (!__malloc_initialized) - ptmalloc_init (); - #if REALLOC_ZERO_BYTES_FREES if (bytes == 0 && oldmem != NULL) { - __libc_free (oldmem); return 0; + __libc_free (oldmem); return NULL; } #endif /* realloc of null is supposed to be same as malloc */ - if (oldmem == 0) + if (oldmem == NULL) return __libc_malloc (bytes); /* Perform a quick check to ensure that the pointer's tag matches the @@ -3452,17 +3604,14 @@ __libc_realloc (void *oldmem, size_t bytes) if (chunk_is_mmapped (oldp)) ar_ptr = NULL; else - { - MAYBE_INIT_TCACHE (); - ar_ptr = arena_for_chunk (oldp); - } + ar_ptr = arena_for_chunk (oldp); /* Little security check which won't hurt performance: the allocator never wraps around at the end of the address space. Therefore we can exclude some size values which might appear here by accident or by "design" from some intruder. */ - if ((__builtin_expect ((uintptr_t) oldp > (uintptr_t) -oldsize, 0) - || __builtin_expect (misaligned_chunk (oldp), 0))) + if (__glibc_unlikely ((uintptr_t) oldp > (uintptr_t) -oldsize + || misaligned_chunk (oldp))) malloc_printerr ("realloc(): invalid pointer"); nb = checked_request2size (bytes); @@ -3495,8 +3644,8 @@ __libc_realloc (void *oldmem, size_t bytes) /* Must alloc, copy, free. */ newmem = __libc_malloc (bytes); - if (newmem == 0) - return 0; /* propagate failure */ + if (newmem == NULL) + return NULL; /* propagate failure */ memcpy (newmem, oldmem, oldsize - CHUNK_HDR_SZ); munmap_chunk (oldp); @@ -3530,7 +3679,7 @@ __libc_realloc (void *oldmem, size_t bytes) size_t sz = memsize (oldp); memcpy (newp, oldmem, sz); (void) tag_region (chunk2mem (oldp), sz); - _int_free (ar_ptr, oldp, 0); + _int_free_chunk (ar_ptr, oldp, chunksize (oldp), 0); } } @@ -3541,11 +3690,7 @@ libc_hidden_def (__libc_realloc) void * __libc_memalign (size_t alignment, size_t bytes) { - if (!__malloc_initialized) - ptmalloc_init (); - - void *address = RETURN_ADDRESS (0); - return _mid_memalign (alignment, bytes, address); + return _mid_memalign (alignment, bytes); } libc_hidden_def (__libc_memalign) @@ -3554,9 +3699,6 @@ void * weak_function aligned_alloc (size_t alignment, size_t bytes) { - if (!__malloc_initialized) - ptmalloc_init (); - /* Similar to memalign, but starting with ISO C17 the standard requires an error for alignments that are not supported by the implementation. Valid alignments for the current implementation @@ -3564,15 +3706,14 @@ aligned_alloc (size_t alignment, size_t bytes) if (!powerof2 (alignment) || alignment == 0) { __set_errno (EINVAL); - return 0; + return NULL; } - void *address = RETURN_ADDRESS (0); - return _mid_memalign (alignment, bytes, address); + return _mid_memalign (alignment, bytes); } static void * -_mid_memalign (size_t alignment, size_t bytes, void *address) +_mid_memalign (size_t alignment, size_t bytes) { mstate ar_ptr; void *p; @@ -3590,7 +3731,7 @@ _mid_memalign (size_t alignment, size_t bytes, void *address) if (alignment > SIZE_MAX / 2 + 1) { __set_errno (EINVAL); - return 0; + return NULL; } @@ -3604,35 +3745,15 @@ _mid_memalign (size_t alignment, size_t bytes, void *address) } #if USE_TCACHE - { - size_t tbytes; - tbytes = checked_request2size (bytes); - if (tbytes == 0) - { - __set_errno (ENOMEM); - return NULL; - } - size_t tc_idx = csize2tidx (tbytes); - - if (tc_idx < mp_.tcache_bins - && tcache != NULL - && tcache->counts[tc_idx] > 0) - { - /* The tcache itself isn't encoded, but the chain is. */ - tcache_entry **tep = & tcache->entries[tc_idx]; - tcache_entry *te = *tep; - while (te != NULL && !PTR_IS_ALIGNED (te, alignment)) - { - tep = & (te->next); - te = tcache_next (te); - } - if (te != NULL) - { - void *victim = tcache_get_n (tc_idx, tep); - return tag_new_usable (victim); - } - } - } + size_t nb = checked_request2size (bytes); + if (nb == 0) + { + __set_errno (ENOMEM); + return NULL; + } + void *victim = tcache_get_align (nb, alignment); + if (victim != NULL) + return tag_new_usable (victim); #endif if (SINGLE_THREAD_P) @@ -3664,21 +3785,12 @@ _mid_memalign (size_t alignment, size_t bytes, void *address) void * __libc_valloc (size_t bytes) { - if (!__malloc_initialized) - ptmalloc_init (); - - void *address = RETURN_ADDRESS (0); - size_t pagesize = GLRO (dl_pagesize); - return _mid_memalign (pagesize, bytes, address); + return _mid_memalign (GLRO (dl_pagesize), bytes); } void * __libc_pvalloc (size_t bytes) { - if (!__malloc_initialized) - ptmalloc_init (); - - void *address = RETURN_ADDRESS (0); size_t pagesize = GLRO (dl_pagesize); size_t rounded_bytes; /* ALIGN_UP with overflow check. */ @@ -3687,37 +3799,20 @@ __libc_pvalloc (size_t bytes) &rounded_bytes))) { __set_errno (ENOMEM); - return 0; + return NULL; } - rounded_bytes = rounded_bytes & -(pagesize - 1); - return _mid_memalign (pagesize, rounded_bytes, address); + return _mid_memalign (pagesize, rounded_bytes & -pagesize); } -void * -__libc_calloc (size_t n, size_t elem_size) +static void * __attribute_noinline__ +__libc_calloc2 (size_t sz) { mstate av; - mchunkptr oldtop; - INTERNAL_SIZE_T sz, oldtopsize; + mchunkptr oldtop, p; + INTERNAL_SIZE_T oldtopsize, csz; void *mem; unsigned long clearsize; - unsigned long nclears; - INTERNAL_SIZE_T *d; - ptrdiff_t bytes; - - if (__glibc_unlikely (__builtin_mul_overflow (n, elem_size, &bytes))) - { - __set_errno (ENOMEM); - return NULL; - } - - sz = bytes; - - if (!__malloc_initialized) - ptmalloc_init (); - - MAYBE_INIT_TCACHE (); if (SINGLE_THREAD_P) av = &main_arena; @@ -3748,7 +3843,7 @@ __libc_calloc (size_t n, size_t elem_size) else { /* No usable arenas. */ - oldtop = 0; + oldtop = NULL; oldtopsize = 0; } mem = _int_malloc (av, sz); @@ -3758,7 +3853,7 @@ __libc_calloc (size_t n, size_t elem_size) if (!SINGLE_THREAD_P) { - if (mem == 0 && av != NULL) + if (mem == NULL && av != NULL) { LIBC_PROBE (memory_calloc_retry, 1, sz); av = arena_get_retry (av, sz); @@ -3770,10 +3865,10 @@ __libc_calloc (size_t n, size_t elem_size) } /* Allocation failed even after a retry. */ - if (mem == 0) - return 0; + if (mem == NULL) + return NULL; - mchunkptr p = mem2chunk (mem); + p = mem2chunk (mem); /* If we are using memory tagging, then we need to set the tags regardless of MORECORE_CLEARS, so we zero the whole block while @@ -3781,12 +3876,12 @@ __libc_calloc (size_t n, size_t elem_size) if (__glibc_unlikely (mtag_enabled)) return tag_new_zero_region (mem, memsize (p)); - INTERNAL_SIZE_T csz = chunksize (p); + csz = chunksize (p); /* Two optional cases in which clearing not necessary */ if (chunk_is_mmapped (p)) { - if (__builtin_expect (perturb_byte, 0)) + if (__glibc_unlikely (perturb_byte)) return memset (mem, 0, sz); return mem; @@ -3800,40 +3895,61 @@ __libc_calloc (size_t n, size_t elem_size) } #endif - /* Unroll clear of <= 36 bytes (72 if 8byte sizes). We know that - contents have an odd number of INTERNAL_SIZE_T-sized words; - minimally 3. */ - d = (INTERNAL_SIZE_T *) mem; clearsize = csz - SIZE_SZ; - nclears = clearsize / sizeof (INTERNAL_SIZE_T); - assert (nclears >= 3); + return clear_memory ((INTERNAL_SIZE_T *) mem, clearsize); +} - if (nclears > 9) - return memset (d, 0, clearsize); +void * +__libc_calloc (size_t n, size_t elem_size) +{ + size_t bytes; - else + if (__glibc_unlikely (__builtin_mul_overflow (n, elem_size, &bytes))) { - *(d + 0) = 0; - *(d + 1) = 0; - *(d + 2) = 0; - if (nclears > 4) - { - *(d + 3) = 0; - *(d + 4) = 0; - if (nclears > 6) - { - *(d + 5) = 0; - *(d + 6) = 0; - if (nclears > 8) - { - *(d + 7) = 0; - *(d + 8) = 0; - } - } - } + __set_errno (ENOMEM); + return NULL; } - return mem; +#if USE_TCACHE + size_t nb = checked_request2size (bytes); + if (nb == 0) + { + __set_errno (ENOMEM); + return NULL; + } + if (nb < mp_.tcache_max_bytes) + { + if (__glibc_unlikely (tcache == NULL)) + return tcache_calloc_init (bytes); + + size_t tc_idx = csize2tidx (nb); + + if (__glibc_unlikely (tc_idx < TCACHE_SMALL_BINS)) + { + if (tcache->entries[tc_idx] != NULL) + { + void *mem = tcache_get (tc_idx); + if (__glibc_unlikely (mtag_enabled)) + return tag_new_zero_region (mem, memsize (mem2chunk (mem))); + + return clear_memory ((INTERNAL_SIZE_T *) mem, tidx2usize (tc_idx)); + } + } + else + { + tc_idx = large_csize2tidx (nb); + void *mem = tcache_get_large (tc_idx, nb); + if (mem != NULL) + { + if (__glibc_unlikely (mtag_enabled)) + return tag_new_zero_region (mem, memsize (mem2chunk (mem))); + + return memset (mem, 0, memsize (mem2chunk (mem))); + } + } + } +#endif + return __libc_calloc2 (bytes); } #endif /* IS_IN (libc) */ @@ -3930,23 +4046,25 @@ _int_malloc (mstate av, size_t bytes) if (__glibc_likely (victim != NULL)) { size_t victim_idx = fastbin_index (chunksize (victim)); - if (__builtin_expect (victim_idx != idx, 0)) + if (__glibc_unlikely (victim_idx != idx)) malloc_printerr ("malloc(): memory corruption (fast)"); check_remalloced_chunk (av, victim, nb); #if USE_TCACHE /* While we're here, if we see other chunks of the same size, stash them in the tcache. */ size_t tc_idx = csize2tidx (nb); - if (tcache != NULL && tc_idx < mp_.tcache_bins) + if (tcache != NULL && tc_idx < mp_.tcache_small_bins) { mchunkptr tc_victim; /* While bin not empty and tcache not full, copy chunks. */ - while (tcache->counts[tc_idx] < mp_.tcache_count - && (tc_victim = *fb) != NULL) + while (tcache->num_slots[tc_idx] != 0 && (tc_victim = *fb) != NULL) { if (__glibc_unlikely (misaligned_chunk (tc_victim))) malloc_printerr ("malloc(): unaligned fastbin chunk detected 3"); + size_t victim_tc_idx = csize2tidx (chunksize (tc_victim)); + if (__glibc_unlikely (tc_idx != victim_tc_idx)) + malloc_printerr ("malloc(): chunk size mismatch in fastbin"); if (SINGLE_THREAD_P) *fb = REVEAL_PTR (tc_victim->fd); else @@ -3995,15 +4113,15 @@ _int_malloc (mstate av, size_t bytes) /* While we're here, if we see other chunks of the same size, stash them in the tcache. */ size_t tc_idx = csize2tidx (nb); - if (tcache != NULL && tc_idx < mp_.tcache_bins) + if (tcache != NULL && tc_idx < mp_.tcache_small_bins) { mchunkptr tc_victim; /* While bin not empty and tcache not full, copy chunks over. */ - while (tcache->counts[tc_idx] < mp_.tcache_count + while (tcache->num_slots[tc_idx] != 0 && (tc_victim = last (bin)) != bin) { - if (tc_victim != 0) + if (tc_victim != NULL) { bck = tc_victim->bk; set_inuse_bit_at_offset (tc_victim, nb); @@ -4057,7 +4175,7 @@ _int_malloc (mstate av, size_t bytes) #if USE_TCACHE INTERNAL_SIZE_T tcache_nb = 0; size_t tc_idx = csize2tidx (nb); - if (tcache != NULL && tc_idx < mp_.tcache_bins) + if (tcache != NULL && tc_idx < mp_.tcache_small_bins) tcache_nb = nb; int return_cached = 0; @@ -4138,7 +4256,7 @@ _int_malloc (mstate av, size_t bytes) /* Fill cache first, return to user only if cache fills. We may return one of these chunks later. */ if (tcache_nb > 0 - && tcache->counts[tc_idx] < mp_.tcache_count) + && tcache->num_slots[tc_idx] != 0) { tcache_put (victim, tc_idx); return_cached = 1; @@ -4156,9 +4274,9 @@ _int_malloc (mstate av, size_t bytes) #endif } - /* place chunk in bin */ - - if (in_smallbin_range (size)) + /* Place chunk in bin. Only malloc_consolidate() and splitting can put + small chunks into the unsorted bin. */ + if (__glibc_unlikely (in_smallbin_range (size))) { victim_index = smallbin_index (size); bck = bin_at (av, victim_index); @@ -4183,6 +4301,9 @@ _int_malloc (mstate av, size_t bytes) fwd = bck; bck = bck->bk; + if (__glibc_unlikely (fwd->fd->bk_nextsize->fd_nextsize != fwd->fd)) + malloc_printerr ("malloc(): largebin double linked list corrupted (nextsize)"); + victim->fd_nextsize = fwd->fd; victim->bk_nextsize = fwd->fd->bk_nextsize; fwd->fd->bk_nextsize = victim->bk_nextsize->fd_nextsize = victim; @@ -4490,69 +4611,14 @@ _int_malloc (mstate av, size_t bytes) ------------------------------ free ------------------------------ */ +/* Free chunk P of SIZE bytes to the arena. HAVE_LOCK indicates where + the arena for P has already been locked. Caller must ensure chunk + and size are valid. */ static void -_int_free (mstate av, mchunkptr p, int have_lock) +_int_free_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size, int have_lock) { - INTERNAL_SIZE_T size; /* its size */ mfastbinptr *fb; /* associated fastbin */ - size = chunksize (p); - - /* Little security check which won't hurt performance: the - allocator never wraps around at the end of the address space. - Therefore we can exclude some size values which might appear - here by accident or by "design" from some intruder. */ - if (__builtin_expect ((uintptr_t) p > (uintptr_t) -size, 0) - || __builtin_expect (misaligned_chunk (p), 0)) - malloc_printerr ("free(): invalid pointer"); - /* We know that each chunk is at least MINSIZE bytes in size or a - multiple of MALLOC_ALIGNMENT. */ - if (__glibc_unlikely (size < MINSIZE || !aligned_OK (size))) - malloc_printerr ("free(): invalid size"); - - check_inuse_chunk(av, p); - -#if USE_TCACHE - { - size_t tc_idx = csize2tidx (size); - if (tcache != NULL && tc_idx < mp_.tcache_bins) - { - /* Check to see if it's already in the tcache. */ - tcache_entry *e = (tcache_entry *) chunk2mem (p); - - /* This test succeeds on double free. However, we don't 100% - trust it (it also matches random payload data at a 1 in - 2^<size_t> chance), so verify it's not an unlikely - coincidence before aborting. */ - if (__glibc_unlikely (e->key == tcache_key)) - { - tcache_entry *tmp; - size_t cnt = 0; - LIBC_PROBE (memory_tcache_double_free, 2, e, tc_idx); - for (tmp = tcache->entries[tc_idx]; - tmp; - tmp = REVEAL_PTR (tmp->next), ++cnt) - { - if (cnt >= mp_.tcache_count) - malloc_printerr ("free(): too many chunks detected in tcache"); - if (__glibc_unlikely (!aligned_OK (tmp))) - malloc_printerr ("free(): unaligned chunk detected in tcache 2"); - if (tmp == e) - malloc_printerr ("free(): double free detected in tcache 2"); - /* If we get here, it was a coincidence. We've wasted a - few cycles, but don't abort. */ - } - } - - if (tcache->counts[tc_idx] < mp_.tcache_count) - { - tcache_put (p, tc_idx); - return; - } - } - } -#endif - /* If eligible, place chunk on a fastbin so it can be found and used quickly in malloc. @@ -4569,10 +4635,9 @@ _int_free (mstate av, mchunkptr p, int have_lock) #endif ) { - if (__builtin_expect (chunksize_nomask (chunk_at_offset (p, size)) - <= CHUNK_HDR_SZ, 0) - || __builtin_expect (chunksize (chunk_at_offset (p, size)) - >= av->system_mem, 0)) + if (__glibc_unlikely ( + chunksize_nomask (chunk_at_offset(p, size)) <= CHUNK_HDR_SZ + || chunksize (chunk_at_offset(p, size)) >= av->system_mem)) { bool fail = true; /* We might not have a lock at this point and concurrent modifications @@ -4603,7 +4668,7 @@ _int_free (mstate av, mchunkptr p, int have_lock) { /* Check that the top of the bin is not the record we are going to add (i.e., double free). */ - if (__builtin_expect (old == p, 0)) + if (__glibc_unlikely (old == p)) malloc_printerr ("double free or corruption (fasttop)"); p->fd = PROTECT_PTR (&p->fd, old); *fb = p; @@ -4613,7 +4678,7 @@ _int_free (mstate av, mchunkptr p, int have_lock) { /* Check that the top of the bin is not the record we are going to add (i.e., double free). */ - if (__builtin_expect (old == p, 0)) + if (__glibc_unlikely (old == p)) malloc_printerr ("double free or corruption (fasttop)"); old2 = old; p->fd = PROTECT_PTR (&p->fd, old); @@ -4626,7 +4691,7 @@ _int_free (mstate av, mchunkptr p, int have_lock) only if we have the lock, otherwise it might have already been allocated again. */ if (have_lock && old != NULL - && __builtin_expect (fastbin_index (chunksize (old)) != idx, 0)) + && __glibc_unlikely (fastbin_index (chunksize (old)) != idx)) malloc_printerr ("invalid fastbin entry (free)"); } @@ -4636,6 +4701,9 @@ _int_free (mstate av, mchunkptr p, int have_lock) else if (!chunk_is_mmapped(p)) { + /* Preserve errno in case block merging results in munmap. */ + int err = errno; + /* If we're single-threaded, don't lock the arena. */ if (SINGLE_THREAD_P) have_lock = true; @@ -4647,13 +4715,33 @@ _int_free (mstate av, mchunkptr p, int have_lock) if (!have_lock) __libc_lock_unlock (av->mutex); + + __set_errno (err); } /* If the chunk was allocated via mmap, release via munmap(). */ else { + + /* Preserve errno in case munmap sets it. */ + int err = errno; + + /* See if the dynamic brk/mmap threshold needs adjusting. + Dumped fake mmapped chunks do not affect the threshold. */ + if (!mp_.no_dyn_threshold + && chunksize_nomask (p) > mp_.mmap_threshold + && chunksize_nomask (p) <= DEFAULT_MMAP_THRESHOLD_MAX) + { + mp_.mmap_threshold = chunksize (p); + mp_.trim_threshold = 2 * mp_.mmap_threshold; + LIBC_PROBE (memory_mallopt_free_dyn_thresholds, 2, + mp_.mmap_threshold, mp_.trim_threshold); + } + munmap_chunk (p); + + __set_errno (err); } } @@ -4670,17 +4758,17 @@ _int_free_merge_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size) if (__glibc_unlikely (p == av->top)) malloc_printerr ("double free or corruption (top)"); /* Or whether the next chunk is beyond the boundaries of the arena. */ - if (__builtin_expect (contiguous (av) + if (__glibc_unlikely (contiguous (av) && (char *) nextchunk - >= ((char *) av->top + chunksize(av->top)), 0)) + >= ((char *) av->top + chunksize(av->top)))) malloc_printerr ("double free or corruption (out)"); /* Or whether the block is actually not marked used. */ if (__glibc_unlikely (!prev_inuse(nextchunk))) malloc_printerr ("double free or corruption (!prev)"); INTERNAL_SIZE_T nextsize = chunksize(nextchunk); - if (__builtin_expect (chunksize_nomask (nextchunk) <= CHUNK_HDR_SZ, 0) - || __builtin_expect (nextsize >= av->system_mem, 0)) + if (__glibc_unlikely (chunksize_nomask (nextchunk) <= CHUNK_HDR_SZ + || nextsize >= av->system_mem)) malloc_printerr ("free(): invalid next size (normal)"); free_perturb (chunk2mem(p), size - CHUNK_HDR_SZ); @@ -4723,23 +4811,39 @@ _int_free_create_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size, } else clear_inuse_bit_at_offset(nextchunk, 0); - /* - Place the chunk in unsorted chunk list. Chunks are - not placed into regular bins until after they have - been given one chance to be used in malloc. - */ + mchunkptr bck, fwd; + + if (!in_smallbin_range (size)) + { + /* Place large chunks in unsorted chunk list. Large chunks are + not placed into regular bins until after they have + been given one chance to be used in malloc. + + This branch is first in the if-statement to help branch + prediction on consecutive adjacent frees. */ + bck = unsorted_chunks (av); + fwd = bck->fd; + if (__glibc_unlikely (fwd->bk != bck)) + malloc_printerr ("free(): corrupted unsorted chunks"); + p->fd_nextsize = NULL; + p->bk_nextsize = NULL; + } + else + { + /* Place small chunks directly in their smallbin, so they + don't pollute the unsorted bin. */ + int chunk_index = smallbin_index (size); + bck = bin_at (av, chunk_index); + fwd = bck->fd; + + if (__glibc_unlikely (fwd->bk != bck)) + malloc_printerr ("free(): chunks in smallbin corrupted"); + + mark_bin (av, chunk_index); + } - mchunkptr bck = unsorted_chunks (av); - mchunkptr fwd = bck->fd; - if (__glibc_unlikely (fwd->bk != bck)) - malloc_printerr ("free(): corrupted unsorted chunks"); - p->fd = fwd; p->bk = bck; - if (!in_smallbin_range(size)) - { - p->fd_nextsize = NULL; - p->bk_nextsize = NULL; - } + p->fd = fwd; bck->fd = p; fwd->bk = p; @@ -4748,7 +4852,6 @@ _int_free_create_chunk (mstate av, mchunkptr p, INTERNAL_SIZE_T size, check_free_chunk(av, p); } - else { /* If the chunk borders the current high end of memory, @@ -4839,7 +4942,7 @@ static void malloc_consolidate(mstate av) fb = &fastbin (av, 0); do { p = atomic_exchange_acquire (fb, NULL); - if (p != 0) { + if (p != NULL) { do { { if (__glibc_unlikely (misaligned_chunk (p))) @@ -4898,7 +5001,7 @@ static void malloc_consolidate(mstate av) av->top = p; } - } while ( (p = nextp) != 0); + } while ( (p = nextp) != NULL); } } while (fb++ != maxfb); @@ -4922,9 +5025,9 @@ _int_realloc (mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize, unsigned long remainder_size; /* its size */ /* oldmem size */ - if (__builtin_expect (chunksize_nomask (oldp) <= CHUNK_HDR_SZ, 0) - || __builtin_expect (oldsize >= av->system_mem, 0) - || __builtin_expect (oldsize != chunksize (oldp), 0)) + if (__glibc_unlikely (chunksize_nomask (oldp) <= CHUNK_HDR_SZ + || oldsize >= av->system_mem + || oldsize != chunksize (oldp))) malloc_printerr ("realloc(): invalid old size"); check_inuse_chunk (av, oldp); @@ -4934,8 +5037,8 @@ _int_realloc (mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize, next = chunk_at_offset (oldp, oldsize); INTERNAL_SIZE_T nextsize = chunksize (next); - if (__builtin_expect (chunksize_nomask (next) <= CHUNK_HDR_SZ, 0) - || __builtin_expect (nextsize >= av->system_mem, 0)) + if (__glibc_unlikely (chunksize_nomask (next) <= CHUNK_HDR_SZ + || nextsize >= av->system_mem)) malloc_printerr ("realloc(): invalid next size"); if ((unsigned long) (oldsize) >= (unsigned long) (nb)) @@ -4973,8 +5076,8 @@ _int_realloc (mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize, else { newmem = _int_malloc (av, nb - MALLOC_ALIGN_MASK); - if (newmem == 0) - return 0; /* propagate failure */ + if (newmem == NULL) + return NULL; /* propagate failure */ newp = mem2chunk (newmem); newsize = chunksize (newp); @@ -4994,7 +5097,7 @@ _int_realloc (mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize, (void) tag_region (oldmem, sz); newmem = tag_new_usable (newmem); memcpy (newmem, oldmem, sz); - _int_free (av, oldp, 1); + _int_free_chunk (av, oldp, chunksize (oldp), 1); check_inuse_chunk (av, newp); return newmem; } @@ -5022,7 +5125,7 @@ _int_realloc (mstate av, mchunkptr oldp, INTERNAL_SIZE_T oldsize, (av != &main_arena ? NON_MAIN_ARENA : 0)); /* Mark remainder as inuse so free() won't complain */ set_inuse_bit_at_offset (remainder, remainder_size); - _int_free (av, remainder, 1); + _int_free_chunk (av, remainder, chunksize (remainder), 1); } check_inuse_chunk (av, newp); @@ -5068,8 +5171,8 @@ _int_memalign (mstate av, size_t alignment, size_t bytes) /* Call malloc with worst case padding to hit alignment. */ m = (char *) (_int_malloc (av, nb + alignment + MINSIZE)); - if (m == 0) - return 0; /* propagate failure */ + if (m == NULL) + return NULL; /* propagate failure */ p = mem2chunk (m); @@ -5208,9 +5311,6 @@ __malloc_trim (size_t s) { int result = 0; - if (!__malloc_initialized) - ptmalloc_init (); - mstate ar_ptr = &main_arena; do { @@ -5281,7 +5381,7 @@ int_mallinfo (mstate av, struct mallinfo2 *m) for (i = 0; i < NFASTBINS; ++i) { for (p = fastbin (av, i); - p != 0; + p != NULL; p = REVEAL_PTR (p->fd)) { if (__glibc_unlikely (misaligned_chunk (p))) @@ -5327,9 +5427,6 @@ __libc_mallinfo2 (void) struct mallinfo2 m; mstate ar_ptr; - if (!__malloc_initialized) - ptmalloc_init (); - memset (&m, 0, sizeof (m)); ar_ptr = &main_arena; do @@ -5378,8 +5475,6 @@ __malloc_stats (void) mstate ar_ptr; unsigned int in_use_b = mp_.mmapped_mem, system_b = in_use_b; - if (!__malloc_initialized) - ptmalloc_init (); _IO_flockfile (stderr); int old_flags2 = stderr->_flags2; stderr->_flags2 |= _IO_FLAGS2_NOTCANCEL; @@ -5492,13 +5587,27 @@ do_set_arena_max (size_t value) static __always_inline int do_set_tcache_max (size_t value) { - if (value <= MAX_TCACHE_SIZE) + size_t nb = request2size (value); + size_t tc_idx = csize2tidx (nb); + + /* To check that value is not too big and request2size does not return an + overflown value. */ + if (value > nb) + return 0; + + if (nb > MAX_TCACHE_SMALL_SIZE) + tc_idx = large_csize2tidx (nb); + + LIBC_PROBE (memory_tunable_tcache_max_bytes, 2, value, mp_.tcache_max_bytes); + + if (tc_idx < TCACHE_MAX_BINS) { - LIBC_PROBE (memory_tunable_tcache_max_bytes, 2, value, mp_.tcache_max_bytes); - mp_.tcache_max_bytes = value; - mp_.tcache_bins = csize2tidx (request2size(value)) + 1; + if (tc_idx < TCACHE_SMALL_BINS) + mp_.tcache_small_bins = tc_idx + 1; + mp_.tcache_max_bytes = nb; return 1; } + return 0; } @@ -5560,8 +5669,6 @@ __libc_mallopt (int param_number, int value) mstate av = &main_arena; int res = 1; - if (!__malloc_initialized) - ptmalloc_init (); __libc_lock_lock (av->mutex); LIBC_PROBE (memory_mallopt, 2, param_number, value); @@ -5776,6 +5883,20 @@ malloc_printerr (const char *str) __builtin_unreachable (); } +#if USE_TCACHE + +static volatile int dummy_var; + +static __attribute_noinline__ void +malloc_printerr_tail (const char *str) +{ + /* Ensure this cannot be a no-return function. */ + if (dummy_var) + return; + malloc_printerr (str); +} +#endif + #if IS_IN (libc) /* We need a wrapper function for one of the additions of POSIX. */ int @@ -5783,9 +5904,6 @@ __posix_memalign (void **memptr, size_t alignment, size_t size) { void *mem; - if (!__malloc_initialized) - ptmalloc_init (); - /* Test whether the SIZE argument is valid. It must be a power of two multiple of sizeof (void *). */ if (alignment % sizeof (void *) != 0 @@ -5794,8 +5912,7 @@ __posix_memalign (void **memptr, size_t alignment, size_t size) return EINVAL; - void *address = RETURN_ADDRESS (0); - mem = _mid_memalign (alignment, size, address); + mem = _mid_memalign (alignment, size); if (mem != NULL) { @@ -5826,11 +5943,6 @@ __malloc_info (int options, FILE *fp) size_t total_aspace = 0; size_t total_aspace_mprotect = 0; - - - if (!__malloc_initialized) - ptmalloc_init (); - fputs ("<malloc version=\"1\">\n", fp); /* Iterate over all arenas currently in use. */ diff --git a/malloc/malloc.h b/malloc/malloc.h index 16a5e54..6de9ee8 100644 --- a/malloc/malloc.h +++ b/malloc/malloc.h @@ -1,5 +1,5 @@ /* Prototypes and definition for malloc implementation. - Copyright (C) 1996-2024 Free Software Foundation, Inc. + Copyright (C) 1996-2025 Free Software Foundation, Inc. Copyright The GNU Toolchain Authors. This file is part of the GNU C Library. diff --git a/malloc/mcheck-impl.c b/malloc/mcheck-impl.c index eff4873..a874211 100644 --- a/malloc/mcheck-impl.c +++ b/malloc/mcheck-impl.c @@ -1,5 +1,5 @@ /* mcheck debugging hooks for malloc. - Copyright (C) 1990-2024 Free Software Foundation, Inc. + Copyright (C) 1990-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/mcheck-init.c b/malloc/mcheck-init.c index 2285561..3a674fd 100644 --- a/malloc/mcheck-init.c +++ b/malloc/mcheck-init.c @@ -1,4 +1,4 @@ -/* Copyright (C) 1991-2024 Free Software Foundation, Inc. +/* Copyright (C) 1991-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/mcheck.c b/malloc/mcheck.c index 41b29ca..63d687c 100644 --- a/malloc/mcheck.c +++ b/malloc/mcheck.c @@ -1,5 +1,5 @@ /* The mcheck() interface. - Copyright (C) 1990-2024 Free Software Foundation, Inc. + Copyright (C) 1990-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/mcheck.h b/malloc/mcheck.h index 632d5ea..7ba3c3a 100644 --- a/malloc/mcheck.h +++ b/malloc/mcheck.h @@ -1,4 +1,4 @@ -/* Copyright (C) 1996-2024 Free Software Foundation, Inc. +/* Copyright (C) 1996-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/memusage.c b/malloc/memusage.c index f80225b..347abba 100644 --- a/malloc/memusage.c +++ b/malloc/memusage.c @@ -1,5 +1,5 @@ /* Profile heap and stack memory usage of running program. - Copyright (C) 1998-2024 Free Software Foundation, Inc. + Copyright (C) 1998-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/memusage.sh b/malloc/memusage.sh index d2d9d17..8ae435d 100755 --- a/malloc/memusage.sh +++ b/malloc/memusage.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (C) 1999-2024 Free Software Foundation, Inc. +# Copyright (C) 1999-2025 Free Software Foundation, Inc. # This file is part of the GNU C Library. # The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/memusagestat.c b/malloc/memusagestat.c index 60fba6e..86497a5 100644 --- a/malloc/memusagestat.c +++ b/malloc/memusagestat.c @@ -1,5 +1,5 @@ /* Generate graphic from memory profiling data. - Copyright (C) 1998-2024 Free Software Foundation, Inc. + Copyright (C) 1998-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. This program is free software; you can redistribute it and/or modify diff --git a/malloc/morecore.c b/malloc/morecore.c index c65307c..9a9c1cf 100644 --- a/malloc/morecore.c +++ b/malloc/morecore.c @@ -1,4 +1,4 @@ -/* Copyright (C) 1991-2024 Free Software Foundation, Inc. +/* Copyright (C) 1991-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/mtrace-impl.c b/malloc/mtrace-impl.c index ee29853..822f065 100644 --- a/malloc/mtrace-impl.c +++ b/malloc/mtrace-impl.c @@ -1,5 +1,5 @@ /* mtrace implementation for `malloc'. - Copyright (C) 1991-2024 Free Software Foundation, Inc. + Copyright (C) 1991-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/mtrace.c b/malloc/mtrace.c index eccdb9d..fbb5c3c 100644 --- a/malloc/mtrace.c +++ b/malloc/mtrace.c @@ -1,5 +1,5 @@ /* mtrace API for `malloc'. - Copyright (C) 1991-2024 Free Software Foundation, Inc. + Copyright (C) 1991-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/mtrace.pl b/malloc/mtrace.pl index 32b4da9..71d693a 100644 --- a/malloc/mtrace.pl +++ b/malloc/mtrace.pl @@ -7,7 +7,7 @@ q { exec perl -e '$_ = shift; $_ = "./$_" unless m,^/,; do $_' "$0" "$@" } ; -# Copyright (C) 1997-2024 Free Software Foundation, Inc. +# Copyright (C) 1997-2025 Free Software Foundation, Inc. # This file is part of the GNU C Library. # Based on the mtrace.awk script. @@ -55,7 +55,7 @@ arglist: while (@ARGV) { $ARGV[0] eq "--vers" || $ARGV[0] eq "--versi" || $ARGV[0] eq "--versio" || $ARGV[0] eq "--version") { print "mtrace $PKGVERSION$VERSION\n"; - print "Copyright (C) 2024 Free Software Foundation, Inc.\n"; + print "Copyright (C) 2025 Free Software Foundation, Inc.\n"; print "This is free software; see the source for copying conditions. There is NO\n"; print "warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n"; print "Written by Ulrich Drepper <drepper\@gnu.org>\n"; diff --git a/malloc/obstack.c b/malloc/obstack.c index 579c693..c3eeb9b 100644 --- a/malloc/obstack.c +++ b/malloc/obstack.c @@ -1,5 +1,5 @@ /* obstack.c - subroutines used implicitly by object stack macros - Copyright (C) 1988-2024 Free Software Foundation, Inc. + Copyright (C) 1988-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -107,7 +107,7 @@ int obstack_exit_failure = EXIT_FAILURE; /* A looong time ago (before 1994, anyway; we're not sure) this global variable was used by non-GNU-C macros to avoid multiple evaluation. The GNU C library still exports it because somebody might use it. */ -struct obstack *_obstack_compat = 0; +struct obstack *_obstack_compat = NULL; compat_symbol (libc, _obstack_compat, _obstack, GLIBC_2_0); # endif # endif @@ -180,7 +180,7 @@ _obstack_begin (struct obstack *h, alignment - 1); h->chunk_limit = chunk->limit = (char *) chunk + h->chunk_size; - chunk->prev = 0; + chunk->prev = NULL; /* The initial chunk now contains no empty object. */ h->maybe_empty_object = 0; h->alloc_failed = 0; @@ -228,7 +228,7 @@ _obstack_begin_1 (struct obstack *h, int size, int alignment, alignment - 1); h->chunk_limit = chunk->limit = (char *) chunk + h->chunk_size; - chunk->prev = 0; + chunk->prev = NULL; /* The initial chunk now contains no empty object. */ h->maybe_empty_object = 0; h->alloc_failed = 0; @@ -328,12 +328,12 @@ _obstack_allocated_p (struct obstack *h, void *obj) /* We use >= rather than > since the object cannot be exactly at the beginning of the chunk but might be an empty object exactly at the end of an adjacent chunk. */ - while (lp != 0 && ((void *) lp >= obj || (void *) (lp)->limit < obj)) + while (lp != NULL && ((void *) lp >= obj || (void *) (lp)->limit < obj)) { plp = lp->prev; lp = plp; } - return lp != 0; + return lp != NULL; } /* Free objects in obstack H, including OBJ and everything allocate @@ -351,7 +351,7 @@ __obstack_free (struct obstack *h, void *obj) /* We use >= because there cannot be an object at the beginning of a chunk. But there can be an empty object at that address at the end of another chunk. */ - while (lp != 0 && ((void *) lp >= obj || (void *) (lp)->limit < obj)) + while (lp != NULL && ((void *) lp >= obj || (void *) (lp)->limit < obj)) { plp = lp->prev; CALL_FREEFUN (h, lp); @@ -366,7 +366,7 @@ __obstack_free (struct obstack *h, void *obj) h->chunk_limit = lp->limit; h->chunk = lp; } - else if (obj != 0) + else if (obj != NULL) /* obj is not in any of the chunks! */ abort (); } @@ -383,7 +383,7 @@ _obstack_memory_used (struct obstack *h) struct _obstack_chunk *lp; int nbytes = 0; - for (lp = h->chunk; lp != 0; lp = lp->prev) + for (lp = h->chunk; lp != NULL; lp = lp->prev) { nbytes += lp->limit - (char *) lp; } diff --git a/malloc/obstack.h b/malloc/obstack.h index 95e4907..7e3dee2 100644 --- a/malloc/obstack.h +++ b/malloc/obstack.h @@ -1,5 +1,5 @@ /* obstack.h - object stack macros - Copyright (C) 1988-2024 Free Software Foundation, Inc. + Copyright (C) 1988-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/reallocarray.c b/malloc/reallocarray.c index 2a5f2a4..885eae5 100644 --- a/malloc/reallocarray.c +++ b/malloc/reallocarray.c @@ -1,5 +1,5 @@ /* Change the size of an allocated block. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -27,7 +27,7 @@ __libc_reallocarray (void *optr, size_t nmemb, size_t elem_size) if (__builtin_mul_overflow (nmemb, elem_size, &bytes)) { __set_errno (ENOMEM); - return 0; + return NULL; } return realloc (optr, bytes); } diff --git a/malloc/scratch_buffer_grow.c b/malloc/scratch_buffer_grow.c index c082757..2cb5325 100644 --- a/malloc/scratch_buffer_grow.c +++ b/malloc/scratch_buffer_grow.c @@ -1,5 +1,5 @@ /* Variable-sized buffer with on-stack default allocation. - Copyright (C) 2015-2024 Free Software Foundation, Inc. + Copyright (C) 2015-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/scratch_buffer_grow_preserve.c b/malloc/scratch_buffer_grow_preserve.c index 6891db8..304635d 100644 --- a/malloc/scratch_buffer_grow_preserve.c +++ b/malloc/scratch_buffer_grow_preserve.c @@ -1,5 +1,5 @@ /* Variable-sized buffer with on-stack default allocation. - Copyright (C) 2015-2024 Free Software Foundation, Inc. + Copyright (C) 2015-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/scratch_buffer_set_array_size.c b/malloc/scratch_buffer_set_array_size.c index 0582bef..b70398f 100644 --- a/malloc/scratch_buffer_set_array_size.c +++ b/malloc/scratch_buffer_set_array_size.c @@ -1,5 +1,5 @@ /* Variable-sized buffer with on-stack default allocation. - Copyright (C) 2015-2024 Free Software Foundation, Inc. + Copyright (C) 2015-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/set-freeres.c b/malloc/set-freeres.c index 4976d55..2ac1df3 100644 --- a/malloc/set-freeres.c +++ b/malloc/set-freeres.c @@ -1,4 +1,4 @@ -/* Copyright (C) 1997-2024 Free Software Foundation, Inc. +/* Copyright (C) 1997-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -28,7 +28,6 @@ # pragma weak __nss_module_freeres # pragma weak __nss_action_freeres # pragma weak __nss_database_freeres -# pragma weak __dl_libc_freemem # pragma weak __hdestroy # pragma weak __gconv_cache_freemem # pragma weak __gconv_conf_freemem @@ -136,7 +135,6 @@ __libc_freeres (void) _IO_cleanup (); /* We run the resource freeing after IO cleanup. */ - call_function_static_weak (__dl_libc_freemem); call_function_static_weak (__hdestroy); call_function_static_weak (__gconv_cache_freemem); call_function_static_weak (__gconv_conf_freemem); diff --git a/malloc/thread-freeres.c b/malloc/thread-freeres.c index 55ba4e7..c7ec9f9 100644 --- a/malloc/thread-freeres.c +++ b/malloc/thread-freeres.c @@ -1,5 +1,5 @@ /* Free resources stored in thread-local variables on thread exit. - Copyright (C) 2003-2024 Free Software Foundation, Inc. + Copyright (C) 2003-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-aligned-alloc-random-thread-cross.c b/malloc/tst-aligned-alloc-random-thread-cross.c index 360ecc5..1b70c4b 100644 --- a/malloc/tst-aligned-alloc-random-thread-cross.c +++ b/malloc/tst-aligned-alloc-random-thread-cross.c @@ -1,5 +1,5 @@ /* multi-threaded memory allocation and cross-thread deallocation test. - Copyright (C) 2024 Free Software Foundation, Inc. + Copyright (C) 2024-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-aligned-alloc-random-thread.c b/malloc/tst-aligned-alloc-random-thread.c index e95f792..36653e3 100644 --- a/malloc/tst-aligned-alloc-random-thread.c +++ b/malloc/tst-aligned-alloc-random-thread.c @@ -1,5 +1,5 @@ /* multi-threaded memory allocation/deallocation test. - Copyright (C) 2024 Free Software Foundation, Inc. + Copyright (C) 2024-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-aligned-alloc-random.c b/malloc/tst-aligned-alloc-random.c index f2825ce..b979b2f 100644 --- a/malloc/tst-aligned-alloc-random.c +++ b/malloc/tst-aligned-alloc-random.c @@ -1,5 +1,5 @@ /* Test for randomized malloc that calls aligned_alloc - Copyright (C) 2024 Free Software Foundation, Inc. + Copyright (C) 2024-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-aligned-alloc.c b/malloc/tst-aligned-alloc.c index 91167d1..c169805 100644 --- a/malloc/tst-aligned-alloc.c +++ b/malloc/tst-aligned-alloc.c @@ -1,5 +1,5 @@ /* Test for C17 alignment requirements. - Copyright (C) 2023-2024 Free Software Foundation, Inc. + Copyright (C) 2023-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -25,6 +25,8 @@ #include <libc-diag.h> #include <support/check.h> +#include "tst-malloc-aux.h" + static int do_test (void) { diff --git a/malloc/tst-aligned_alloc-lib.c b/malloc/tst-aligned_alloc-lib.c index 9ef1f83..052a296 100644 --- a/malloc/tst-aligned_alloc-lib.c +++ b/malloc/tst-aligned_alloc-lib.c @@ -1,5 +1,5 @@ /* Module used for improved aligned_alloc testing. - Copyright (C) 2024 Free Software Foundation, Inc. + Copyright (C) 2024-2025 Free Software Foundation, Inc. Copyright The GNU Toolchain Authors. This file is part of the GNU C Library. diff --git a/malloc/tst-alloc_buffer.c b/malloc/tst-alloc_buffer.c index 97a60b0..7c2a15a 100644 --- a/malloc/tst-alloc_buffer.c +++ b/malloc/tst-alloc_buffer.c @@ -1,5 +1,5 @@ /* Tests for struct alloc_buffer. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-calloc.c b/malloc/tst-calloc.c index 01f17f9..e0c1a06 100644 --- a/malloc/tst-calloc.c +++ b/malloc/tst-calloc.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2000-2024 Free Software Foundation, Inc. +/* Copyright (C) 2000-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -23,6 +23,7 @@ #include <stdio.h> #include <libc-diag.h> +#include "tst-malloc-aux.h" /* Number of samples per size. */ #define N 50000 @@ -94,16 +95,19 @@ random_test (void) static void null_test (void) { + /* Obscure allocation size from the compiler. */ + volatile size_t max_size = UINT_MAX; + volatile size_t zero_size = 0; /* If the size is 0 the result is implementation defined. Just make sure the program doesn't crash. The result of calloc is deliberately ignored, so do not warn about that. */ DIAG_PUSH_NEEDS_COMMENT; DIAG_IGNORE_NEEDS_COMMENT (10, "-Wunused-result"); calloc (0, 0); - calloc (0, UINT_MAX); - calloc (UINT_MAX, 0); - calloc (0, ~((size_t) 0)); - calloc (~((size_t) 0), 0); + calloc (0, max_size); + calloc (max_size, 0); + calloc (0, ~zero_size); + calloc (~zero_size, 0); DIAG_POP_NEEDS_COMMENT; } diff --git a/malloc/tst-compathooks-off.c b/malloc/tst-compathooks-off.c index d0106f3..82c47eb 100644 --- a/malloc/tst-compathooks-off.c +++ b/malloc/tst-compathooks-off.c @@ -1,5 +1,5 @@ /* Minimal tests to verify libc_malloc_debug.so functionality. - Copyright (C) 2021-2024 Free Software Foundation, Inc. + Copyright (C) 2021-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -25,6 +25,8 @@ #include <support/check.h> #include <support/support.h> +#include "tst-malloc-aux.h" + extern void (*volatile __free_hook) (void *, const void *); extern void *(*volatile __malloc_hook)(size_t, const void *); extern void *(*volatile __realloc_hook)(void *, size_t, const void *); diff --git a/malloc/tst-dynarray-at-fail.c b/malloc/tst-dynarray-at-fail.c index e5477f9..8684aaf 100644 --- a/malloc/tst-dynarray-at-fail.c +++ b/malloc/tst-dynarray-at-fail.c @@ -1,5 +1,5 @@ /* Test reporting of out-of-bounds access for dynamic arrays. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-dynarray-fail.c b/malloc/tst-dynarray-fail.c index f76f7a9..21524be 100644 --- a/malloc/tst-dynarray-fail.c +++ b/malloc/tst-dynarray-fail.c @@ -1,5 +1,5 @@ /* Test allocation failures with dynamic arrays. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-dynarray-shared.h b/malloc/tst-dynarray-shared.h index 50b6a3d..7318aef 100644 --- a/malloc/tst-dynarray-shared.h +++ b/malloc/tst-dynarray-shared.h @@ -1,5 +1,5 @@ /* Shared definitions for dynarray tests. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-dynarray.c b/malloc/tst-dynarray.c index 757add2..4f212e0 100644 --- a/malloc/tst-dynarray.c +++ b/malloc/tst-dynarray.c @@ -1,5 +1,5 @@ /* Test for dynamic arrays. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -482,7 +482,7 @@ test_long_overflow (void) iteration would invoke undefined behavior. That loop iteration can never be executed because an allocation of this size must fail. */ - DIAG_IGNORE_NEEDS_COMMENT (12, "-Waggressive-loop-optimizations"); + DIAG_IGNORE_NEEDS_COMMENT_GCC (12, "-Waggressive-loop-optimizations"); TEST_VERIFY (!dynarray_long_resize (&dyn, (SIZE_MAX / sizeof (long)) + 1)); DIAG_POP_NEEDS_COMMENT; @@ -499,7 +499,7 @@ test_long_overflow (void) iteration would invoke undefined behavior. That loop iteration can never be executed because an allocation of this size must fail. */ - DIAG_IGNORE_NEEDS_COMMENT (12, "-Waggressive-loop-optimizations"); + DIAG_IGNORE_NEEDS_COMMENT_GCC (12, "-Waggressive-loop-optimizations"); TEST_VERIFY (!dynarray_long_noscratch_resize (&dyn, (SIZE_MAX / sizeof (long)) + 1)); DIAG_POP_NEEDS_COMMENT; diff --git a/malloc/tst-free-errno.c b/malloc/tst-free-errno.c index 78ae450..0e7a0f2 100644 --- a/malloc/tst-free-errno.c +++ b/malloc/tst-free-errno.c @@ -1,5 +1,5 @@ /* Test that free preserves errno. - Copyright (C) 2020-2024 Free Software Foundation, Inc. + Copyright (C) 2020-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-interpose-aux-nothread.c b/malloc/tst-interpose-aux-nothread.c index ec44ac1..ca8aee3 100644 --- a/malloc/tst-interpose-aux-nothread.c +++ b/malloc/tst-interpose-aux-nothread.c @@ -1,5 +1,5 @@ /* Interposed malloc, version without threading support. - Copyright (C) 2016-2024 Free Software Foundation, Inc. + Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-interpose-aux-thread.c b/malloc/tst-interpose-aux-thread.c index a4aab10b..15dbf2a 100644 --- a/malloc/tst-interpose-aux-thread.c +++ b/malloc/tst-interpose-aux-thread.c @@ -1,5 +1,5 @@ /* Interposed malloc, version with threading support. - Copyright (C) 2016-2024 Free Software Foundation, Inc. + Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-interpose-aux.c b/malloc/tst-interpose-aux.c index 4f8020b..cf4b8ab 100644 --- a/malloc/tst-interpose-aux.c +++ b/malloc/tst-interpose-aux.c @@ -1,5 +1,5 @@ /* Minimal malloc implementation for interposition tests. - Copyright (C) 2016-2024 Free Software Foundation, Inc. + Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-interpose-aux.h b/malloc/tst-interpose-aux.h index a8cd71e..0e18fc2 100644 --- a/malloc/tst-interpose-aux.h +++ b/malloc/tst-interpose-aux.h @@ -1,5 +1,5 @@ /* Statistics interface for the minimal malloc implementation. - Copyright (C) 2016-2024 Free Software Foundation, Inc. + Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-interpose-nothread.c b/malloc/tst-interpose-nothread.c index 1d07317..a95b9b2 100644 --- a/malloc/tst-interpose-nothread.c +++ b/malloc/tst-interpose-nothread.c @@ -1,5 +1,5 @@ /* Malloc interposition test, dynamically-linked version without threads. - Copyright (C) 2016-2024 Free Software Foundation, Inc. + Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-interpose-skeleton.c b/malloc/tst-interpose-skeleton.c index 88194d0..0a868df 100644 --- a/malloc/tst-interpose-skeleton.c +++ b/malloc/tst-interpose-skeleton.c @@ -1,5 +1,5 @@ /* Test driver for malloc interposition tests. - Copyright (C) 2016-2024 Free Software Foundation, Inc. + Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-interpose-static-nothread.c b/malloc/tst-interpose-static-nothread.c index 1c2657e..a1f1835 100644 --- a/malloc/tst-interpose-static-nothread.c +++ b/malloc/tst-interpose-static-nothread.c @@ -1,5 +1,5 @@ /* Malloc interposition test, static version without threads. - Copyright (C) 2016-2024 Free Software Foundation, Inc. + Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-interpose-static-thread.c b/malloc/tst-interpose-static-thread.c index 5ade16c..73e4086 100644 --- a/malloc/tst-interpose-static-thread.c +++ b/malloc/tst-interpose-static-thread.c @@ -1,5 +1,5 @@ /* Malloc interposition test, static version with threads. - Copyright (C) 2016-2024 Free Software Foundation, Inc. + Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-interpose-thread.c b/malloc/tst-interpose-thread.c index 7d07221..5f6ed0f 100644 --- a/malloc/tst-interpose-thread.c +++ b/malloc/tst-interpose-thread.c @@ -1,5 +1,5 @@ /* Malloc interposition test, dynamically-linked version with threads. - Copyright (C) 2016-2024 Free Software Foundation, Inc. + Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-mallinfo2.c b/malloc/tst-mallinfo2.c index 2c02f5f..28ff6e2 100644 --- a/malloc/tst-mallinfo2.c +++ b/malloc/tst-mallinfo2.c @@ -1,5 +1,5 @@ /* Smoke test for mallinfo2 - Copyright (C) 2020-2024 Free Software Foundation, Inc. + Copyright (C) 2020-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -23,6 +23,8 @@ #include <stdlib.h> #include <support/check.h> +#include "tst-malloc-aux.h" + /* This is not specifically needed for the test, but (1) does something to the data so gcc doesn't optimize it away, and (2) may help when developing future tests. */ diff --git a/malloc/tst-malloc-alternate-path.c b/malloc/tst-malloc-alternate-path.c index 43ae916..413dae3 100644 --- a/malloc/tst-malloc-alternate-path.c +++ b/malloc/tst-malloc-alternate-path.c @@ -1,5 +1,5 @@ /* Test that malloc uses mmap when sbrk or brk fails. - Copyright (C) 2024 Free Software Foundation, Inc. + Copyright (C) 2024-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-malloc-aux.h b/malloc/tst-malloc-aux.h new file mode 100644 index 0000000..cd752af --- /dev/null +++ b/malloc/tst-malloc-aux.h @@ -0,0 +1,56 @@ +/* Wrappers for malloc-like functions to allow testing the implementation + without optimization. + Copyright (C) 2024-2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License as + published by the Free Software Foundation; either version 2.1 of the + License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; see the file COPYING.LIB. If + not, see <https://www.gnu.org/licenses/>. */ + +#ifndef TST_MALLOC_AUX_H +#define TST_MALLOC_AUX_H + +#include <stddef.h> +#include <stdlib.h> +#include <malloc.h> + +static __typeof (aligned_alloc) * volatile aligned_alloc_indirect + = aligned_alloc; +static __typeof (calloc) * volatile calloc_indirect = calloc; +static __typeof (malloc) * volatile malloc_indirect = malloc; +static __typeof (memalign) * volatile memalign_indirect = memalign; +static __typeof (posix_memalign) * volatile posix_memalign_indirect + = posix_memalign; +static __typeof (pvalloc) * volatile pvalloc_indirect = pvalloc; +static __typeof (realloc) * volatile realloc_indirect = realloc; +static __typeof (valloc) * volatile valloc_indirect = valloc; + +#undef aligned_alloc +#undef calloc +#undef malloc +#undef memalign +#undef posix_memalign +#undef pvalloc +#undef realloc +#undef valloc + +#define aligned_alloc aligned_alloc_indirect +#define calloc calloc_indirect +#define malloc malloc_indirect +#define memalign memalign_indirect +#define posix_memalign posix_memalign_indirect +#define pvalloc pvalloc_indirect +#define realloc realloc_indirect +#define valloc valloc_indirect + +#endif /* TST_MALLOC_AUX_H */ diff --git a/malloc/tst-malloc-backtrace.c b/malloc/tst-malloc-backtrace.c index c7b1d65..4254b45 100644 --- a/malloc/tst-malloc-backtrace.c +++ b/malloc/tst-malloc-backtrace.c @@ -1,5 +1,5 @@ /* Verify that backtrace does not deadlock on itself on memory corruption. - Copyright (C) 2015-2024 Free Software Foundation, Inc. + Copyright (C) 2015-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -22,6 +22,8 @@ #include <support/support.h> #include <libc-diag.h> +#include "tst-malloc-aux.h" + #define SIZE 4096 /* Wrap free with a function to prevent gcc from optimizing it out. */ diff --git a/malloc/tst-malloc-check.c b/malloc/tst-malloc-check.c index fde8863..330e535 100644 --- a/malloc/tst-malloc-check.c +++ b/malloc/tst-malloc-check.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2005-2024 Free Software Foundation, Inc. +/* Copyright (C) 2005-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -20,6 +20,8 @@ #include <stdlib.h> #include <libc-diag.h> +#include "tst-malloc-aux.h" + static int errors = 0; static void diff --git a/malloc/tst-malloc-fork-deadlock.c b/malloc/tst-malloc-fork-deadlock.c index cc17f3e..dfa209b 100644 --- a/malloc/tst-malloc-fork-deadlock.c +++ b/malloc/tst-malloc-fork-deadlock.c @@ -1,5 +1,5 @@ /* Test concurrent fork, getline, and fflush (NULL). - Copyright (C) 2016-2024 Free Software Foundation, Inc. + Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-malloc-random.c b/malloc/tst-malloc-random.c index 762b70c..3359551 100644 --- a/malloc/tst-malloc-random.c +++ b/malloc/tst-malloc-random.c @@ -1,6 +1,6 @@ /* Test malloc with random calls to aligned_alloc and calloc. - Copyright (C) 2024 Free Software Foundation, Inc. + Copyright (C) 2024-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-malloc-tcache-leak.c b/malloc/tst-malloc-tcache-leak.c index 3462693..7bf7e47 100644 --- a/malloc/tst-malloc-tcache-leak.c +++ b/malloc/tst-malloc-tcache-leak.c @@ -1,5 +1,5 @@ /* Bug 22111: Test that threads do not leak their per thread cache. - Copyright (C) 2015-2024 Free Software Foundation, Inc. + Copyright (C) 2015-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-malloc-thread-exit.c b/malloc/tst-malloc-thread-exit.c index aec3897..ebee969 100644 --- a/malloc/tst-malloc-thread-exit.c +++ b/malloc/tst-malloc-thread-exit.c @@ -1,5 +1,5 @@ /* Test malloc with concurrent thread termination. - Copyright (C) 2015-2024 Free Software Foundation, Inc. + Copyright (C) 2015-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -42,7 +42,7 @@ static int inner_thread_count = 4; static size_t malloc_size = 32; static void -__attribute__ ((noinline, noclone)) +__attribute_optimization_barrier__ unoptimized_free (void *ptr) { free (ptr); diff --git a/malloc/tst-malloc-thread-fail.c b/malloc/tst-malloc-thread-fail.c index f698871..f00c394 100644 --- a/malloc/tst-malloc-thread-fail.c +++ b/malloc/tst-malloc-thread-fail.c @@ -1,5 +1,5 @@ /* Test allocation function behavior on allocation failure. - Copyright (C) 2015-2024 Free Software Foundation, Inc. + Copyright (C) 2015-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -33,7 +33,7 @@ /* Wrapper for calloc with an optimization barrier. */ static void * -__attribute__ ((noinline, noclone)) +__attribute_optimization_barrier__ allocate_zeroed (size_t a, size_t b) { return calloc (a, b); diff --git a/malloc/tst-malloc-too-large.c b/malloc/tst-malloc-too-large.c index 8e9e0d5..a548a37 100644 --- a/malloc/tst-malloc-too-large.c +++ b/malloc/tst-malloc-too-large.c @@ -1,5 +1,5 @@ /* Test and verify that too-large memory allocations fail with ENOMEM. - Copyright (C) 2018-2024 Free Software Foundation, Inc. + Copyright (C) 2018-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -43,6 +43,7 @@ #include <unistd.h> #include <sys/param.h> +#include "tst-malloc-aux.h" /* This function prepares for each 'too-large memory allocation' test by performing a small successful malloc/free and resetting errno prior to diff --git a/malloc/tst-malloc-usable.c b/malloc/tst-malloc-usable.c index 35cb10b..7688884 100644 --- a/malloc/tst-malloc-usable.c +++ b/malloc/tst-malloc-usable.c @@ -1,7 +1,7 @@ /* Ensure that malloc_usable_size returns the request size with MALLOC_CHECK_ exported to a positive value. - Copyright (C) 2012-2024 Free Software Foundation, Inc. + Copyright (C) 2012-2025 Free Software Foundation, Inc. Copyright The GNU Toolchain Authors. This file is part of the GNU C Library. diff --git a/malloc/tst-malloc.c b/malloc/tst-malloc.c index f7a6e46..77aefeb 100644 --- a/malloc/tst-malloc.c +++ b/malloc/tst-malloc.c @@ -1,4 +1,4 @@ -/* Copyright (C) 1999-2024 Free Software Foundation, Inc. +/* Copyright (C) 1999-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -22,6 +22,8 @@ #include <libc-diag.h> #include <time.h> +#include "tst-malloc-aux.h" + static int errors = 0; static void diff --git a/malloc/tst-malloc_info.c b/malloc/tst-malloc_info.c index 50cf4ea..2258091 100644 --- a/malloc/tst-malloc_info.c +++ b/malloc/tst-malloc_info.c @@ -1,5 +1,5 @@ /* Smoke test for malloc_info. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-mallocalign1.c b/malloc/tst-mallocalign1.c index 0f92b58..b8839e3 100644 --- a/malloc/tst-mallocalign1.c +++ b/malloc/tst-mallocalign1.c @@ -1,5 +1,5 @@ /* Verify that MALLOC_ALIGNMENT is honored by malloc. - Copyright (C) 2012-2024 Free Software Foundation, Inc. + Copyright (C) 2012-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-mallocfork2.c b/malloc/tst-mallocfork2.c index 6837bbd..40c8bcd 100644 --- a/malloc/tst-mallocfork2.c +++ b/malloc/tst-mallocfork2.c @@ -1,5 +1,5 @@ /* Test case for async-signal-safe fork (with respect to malloc). - Copyright (C) 2016-2024 Free Software Foundation, Inc. + Copyright (C) 2016-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-mallocfork3.c b/malloc/tst-mallocfork3.c index 8240702..b9cf95e 100644 --- a/malloc/tst-mallocfork3.c +++ b/malloc/tst-mallocfork3.c @@ -1,5 +1,5 @@ /* Test case for async-signal-safe _Fork (with respect to malloc). - Copyright (C) 2021-2024 Free Software Foundation, Inc. + Copyright (C) 2021-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-mallocstate.c b/malloc/tst-mallocstate.c index f72e1a1..ccfa055 100644 --- a/malloc/tst-mallocstate.c +++ b/malloc/tst-mallocstate.c @@ -1,5 +1,5 @@ /* Emulate Emacs heap dumping to test malloc_set_state. - Copyright (C) 2001-2024 Free Software Foundation, Inc. + Copyright (C) 2001-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -366,7 +366,7 @@ full_heap_check (void) } /* Used as an optimization barrier to force a heap allocation. */ -__attribute__ ((noinline, noclone)) +__attribute_optimization_barrier__ static void my_free (void *ptr) { diff --git a/malloc/tst-mallopt.c b/malloc/tst-mallopt.c index 9b8a10d..06821f8 100644 --- a/malloc/tst-mallopt.c +++ b/malloc/tst-mallopt.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2014-2024 Free Software Foundation, Inc. +/* Copyright (C) 2014-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-memalign-2.c b/malloc/tst-memalign-2.c index 2d097f4..4494e64 100644 --- a/malloc/tst-memalign-2.c +++ b/malloc/tst-memalign-2.c @@ -1,5 +1,5 @@ /* Test for memalign chunk reuse. - Copyright (C) 2022-2024 Free Software Foundation, Inc. + Copyright (C) 2022-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-memalign-3.c b/malloc/tst-memalign-3.c index 31ba0e2..4b462f0 100644 --- a/malloc/tst-memalign-3.c +++ b/malloc/tst-memalign-3.c @@ -1,5 +1,5 @@ /* Test for memalign chunk reuse. - Copyright (C) 2022-2024 Free Software Foundation, Inc. + Copyright (C) 2022-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-memalign.c b/malloc/tst-memalign.c index 563f641..f73ccf6 100644 --- a/malloc/tst-memalign.c +++ b/malloc/tst-memalign.c @@ -1,5 +1,5 @@ /* Test for memalign. - Copyright (C) 2013-2024 Free Software Foundation, Inc. + Copyright (C) 2013-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -23,6 +23,8 @@ #include <unistd.h> #include <libc-diag.h> +#include "tst-malloc-aux.h" + static int errors = 0; static void diff --git a/malloc/tst-mtrace.c b/malloc/tst-mtrace.c index e59bffc..07165c3 100644 --- a/malloc/tst-mtrace.c +++ b/malloc/tst-mtrace.c @@ -1,5 +1,5 @@ /* Test program for mtrace. - Copyright (C) 2000-2024 Free Software Foundation, Inc. + Copyright (C) 2000-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-mtrace.sh b/malloc/tst-mtrace.sh index 3913375..0d62f2d 100755 --- a/malloc/tst-mtrace.sh +++ b/malloc/tst-mtrace.sh @@ -1,6 +1,6 @@ #!/bin/sh # Testing the mtrace function. -# Copyright (C) 2000-2024 Free Software Foundation, Inc. +# Copyright (C) 2000-2025 Free Software Foundation, Inc. # This file is part of the GNU C Library. # The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-mxfast.c b/malloc/tst-mxfast.c index b48f882..a27b543 100644 --- a/malloc/tst-mxfast.c +++ b/malloc/tst-mxfast.c @@ -1,5 +1,5 @@ /* Test that glibc.malloc.mxfast tunable works. - Copyright (C) 2019-2024 Free Software Foundation, Inc. + Copyright (C) 2019-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-posix_memalign.c b/malloc/tst-posix_memalign.c index 1d54f27..659edb0 100644 --- a/malloc/tst-posix_memalign.c +++ b/malloc/tst-posix_memalign.c @@ -1,5 +1,5 @@ /* Test for posix_memalign. - Copyright (C) 2013-2024 Free Software Foundation, Inc. + Copyright (C) 2013-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-pvalloc-fortify.c b/malloc/tst-pvalloc-fortify.c index 6bd29bd..df74702 100644 --- a/malloc/tst-pvalloc-fortify.c +++ b/malloc/tst-pvalloc-fortify.c @@ -1,5 +1,5 @@ /* Test fortify-source allocation size handling in pvalloc (bug 25401). - Copyright (C) 2020-2024 Free Software Foundation, Inc. + Copyright (C) 2020-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-pvalloc.c b/malloc/tst-pvalloc.c index c4fd806..8514646 100644 --- a/malloc/tst-pvalloc.c +++ b/malloc/tst-pvalloc.c @@ -1,5 +1,5 @@ /* Test for pvalloc. - Copyright (C) 2013-2024 Free Software Foundation, Inc. + Copyright (C) 2013-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-realloc.c b/malloc/tst-realloc.c index f50499e..997d901 100644 --- a/malloc/tst-realloc.c +++ b/malloc/tst-realloc.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2013-2024 Free Software Foundation, Inc. +/* Copyright (C) 2013-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -23,6 +23,8 @@ #include <libc-diag.h> #include <support/check.h> +#include "tst-malloc-aux.h" + static int do_test (void) { diff --git a/malloc/tst-reallocarray.c b/malloc/tst-reallocarray.c index 1cafbc5..b4df4aa 100644 --- a/malloc/tst-reallocarray.c +++ b/malloc/tst-reallocarray.c @@ -1,5 +1,5 @@ /* Test for reallocarray. - Copyright (C) 2017-2024 Free Software Foundation, Inc. + Copyright (C) 2017-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-safe-linking.c b/malloc/tst-safe-linking.c index 01dd070..b44b8f5 100644 --- a/malloc/tst-safe-linking.c +++ b/malloc/tst-safe-linking.c @@ -1,5 +1,5 @@ /* Test reporting of Safe-Linking caught errors. - Copyright (C) 2020-2024 Free Software Foundation, Inc. + Copyright (C) 2020-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -26,6 +26,8 @@ #include <support/capture_subprocess.h> #include <support/check.h> +#include "tst-malloc-aux.h" + /* Run CALLBACK and check that the data on standard error equals EXPECTED. */ static void @@ -111,22 +113,37 @@ test_fastbin (void *closure) int i; int mask = ((int *)closure)[0]; size_t size = TCACHE_ALLOC_SIZE; + void * ps[TCACHE_FILL_COUNT]; + void * pps[TCACHE_FILL_COUNT]; printf ("++ fastbin ++\n"); + /* Populate the fastbin list. */ + void * volatile a = calloc (1, size); + void * volatile b = calloc (1, size); + void * volatile c = calloc (1, size); + printf ("a=%p, b=%p, c=%p\n", a, b, c); + + /* Chunks for later tcache filling from fastbins. */ + for (i = 0; i < TCACHE_FILL_COUNT; ++i) + { + void * volatile p = calloc (1, size); + pps[i] = p; + } + /* Take the tcache out of the game. */ for (i = 0; i < TCACHE_FILL_COUNT; ++i) { void * volatile p = calloc (1, size); - printf ("p=%p\n", p); - free (p); + ps[i] = p; } - /* Populate the fastbin list. */ - void * volatile a = calloc (1, size); - void * volatile b = calloc (1, size); - void * volatile c = calloc (1, size); - printf ("a=%p, b=%p, c=%p\n", a, b, c); + for (i = 0; i < TCACHE_FILL_COUNT; ++i) + { + free (ps[i]); + } + + /* Free abc will return to fastbin in FIFO order. */ free (a); free (b); free (c); @@ -136,11 +153,43 @@ test_fastbin (void *closure) memset (c, mask & 0xFF, size); printf ("After: c=%p, c[0]=%p\n", c, ((void **)c)[0]); + /* Filling fastbins, will be copied to tcache later. */ + for (i = 0; i < TCACHE_FILL_COUNT; ++i) + { + free (pps[i]); + } + + /* Drain out tcache to make sure later alloc from fastbins. */ + for (i = 0; i < TCACHE_FILL_COUNT; ++i) + { + void * volatile p = calloc (1, size); + ps[i] = p; + } + + /* This line will also filling tcache with remain pps and c. */ + pps[TCACHE_FILL_COUNT - 1] = calloc (1, size); + + /* Tcache is FILO, now the first one is c, take it out. */ c = calloc (1, size); printf ("Allocated: c=%p\n", c); + + /* Drain out remain pps from tcache. */ + for (i = 0; i < TCACHE_FILL_COUNT - 1; ++i) + { + void * volatile p = calloc (1, size); + pps[i] = p; + } + /* This line will trigger the Safe-Linking check. */ b = calloc (1, size); printf ("b=%p\n", b); + + /* Free previous pointers. */ + for (i = 0; i < TCACHE_FILL_COUNT; ++i) + { + free (ps[i]); + free (pps[i]); + } } /* Try corrupting the fastbin list and trigger a consolidate. */ @@ -150,21 +199,29 @@ test_fastbin_consolidate (void *closure) int i; int mask = ((int*)closure)[0]; size_t size = TCACHE_ALLOC_SIZE; + void * ps[TCACHE_FILL_COUNT]; printf ("++ fastbin consolidate ++\n"); + /* Populate the fastbin list. */ + void * volatile a = calloc (1, size); + void * volatile b = calloc (1, size); + void * volatile c = calloc (1, size); + printf ("a=%p, b=%p, c=%p\n", a, b, c); + /* Take the tcache out of the game. */ for (i = 0; i < TCACHE_FILL_COUNT; ++i) { void * volatile p = calloc (1, size); - free (p); + ps[i] = p; } - /* Populate the fastbin list. */ - void * volatile a = calloc (1, size); - void * volatile b = calloc (1, size); - void * volatile c = calloc (1, size); - printf ("a=%p, b=%p, c=%p\n", a, b, c); + for (i = 0; i < TCACHE_FILL_COUNT; ++i) + { + free (ps[i]); + } + + /* Free abc will return to fastbin. */ free (a); free (b); free (c); diff --git a/malloc/tst-scratch_buffer.c b/malloc/tst-scratch_buffer.c index 2e27a18..624c064 100644 --- a/malloc/tst-scratch_buffer.c +++ b/malloc/tst-scratch_buffer.c @@ -1,5 +1,5 @@ /* - Copyright (C) 2015-2024 Free Software Foundation, Inc. + Copyright (C) 2015-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-tcfree1.c b/malloc/tst-tcfree1.c index 29c0a9e..1775aa2 100644 --- a/malloc/tst-tcfree1.c +++ b/malloc/tst-tcfree1.c @@ -1,5 +1,5 @@ /* Test that malloc tcache catches double free. - Copyright (C) 2018-2024 Free Software Foundation, Inc. + Copyright (C) 2018-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-tcfree2.c b/malloc/tst-tcfree2.c index bbc0871..a1ce9b7 100644 --- a/malloc/tst-tcfree2.c +++ b/malloc/tst-tcfree2.c @@ -1,5 +1,5 @@ /* Test that malloc tcache catches double free. - Copyright (C) 2018-2024 Free Software Foundation, Inc. + Copyright (C) 2018-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-tcfree3.c b/malloc/tst-tcfree3.c index e1ae185..6d71572 100644 --- a/malloc/tst-tcfree3.c +++ b/malloc/tst-tcfree3.c @@ -1,5 +1,5 @@ /* Test that malloc tcache catches double free. - Copyright (C) 2018-2024 Free Software Foundation, Inc. + Copyright (C) 2018-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or diff --git a/malloc/tst-tcfree4.c b/malloc/tst-tcfree4.c new file mode 100644 index 0000000..03850dd --- /dev/null +++ b/malloc/tst-tcfree4.c @@ -0,0 +1,59 @@ +/* Test that malloc tcache catches double free. + Copyright (C) 2025 Free Software Foundation, Inc. + This file is part of the GNU C Library. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, see + <https://www.gnu.org/licenses/>. */ + +#include <errno.h> +#include <error.h> +#include <limits.h> +#include <malloc.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <sys/signal.h> + +/* Test for a double free where the size information gets overwritten by a + * terminating null byte. */ +static int +do_test (void) +{ + /* The payload is exactly 0x19 Bytes long: + * 0x18 bytes 'B' and one terminating null byte + */ + const char *payload = "BBBBBBBBBBBBBBBBBBBBBBBB"; + + char *volatile first_chunk + = malloc (strlen (payload)); // <-- off by one error + char *volatile second_chunk = malloc (0x118); + + // free the second chunk the first time now it is in the tcache with tc_idx = + free (second_chunk); + + // change the the size of the second_chunk using the terminating null byte if + // the PAYLOAD + strcpy (first_chunk, payload); + + // now the second_chunk has a new size + // calling free a second time will not trigger the double free detection + free (second_chunk); + + printf ("FAIL: tcache double free not detected\n"); + return 1; +} + +#define TEST_FUNCTION do_test +#define EXPECTED_SIGNAL SIGABRT +#include <support/test-driver.c> diff --git a/malloc/tst-valloc.c b/malloc/tst-valloc.c index 9bab8c6..344ca7d 100644 --- a/malloc/tst-valloc.c +++ b/malloc/tst-valloc.c @@ -1,5 +1,5 @@ /* Test for valloc. - Copyright (C) 2013-2024 Free Software Foundation, Inc. + Copyright (C) 2013-2025 Free Software Foundation, Inc. This file is part of the GNU C Library. The GNU C Library is free software; you can redistribute it and/or @@ -23,6 +23,8 @@ #include <unistd.h> #include <libc-diag.h> +#include "tst-malloc-aux.h" + static int errors = 0; static void |