diff options
133 files changed, 2284 insertions, 2708 deletions
diff --git a/libsanitizer/MERGE b/libsanitizer/MERGE index c4731d0..81d00f2 100644 --- a/libsanitizer/MERGE +++ b/libsanitizer/MERGE @@ -1,4 +1,4 @@ -f58e0513dd95944b81ce7a6e7b49ba656de7d75f +7704fedfff6ef5676adb6415f3be0ac927d1a746 The first line of this file holds the git revision number of the last merge done from the master library sources. diff --git a/libsanitizer/asan/Makefile.am b/libsanitizer/asan/Makefile.am index 2e63855..74658ca 100644 --- a/libsanitizer/asan/Makefile.am +++ b/libsanitizer/asan/Makefile.am @@ -38,7 +38,6 @@ asan_files = \ asan_posix.cpp \ asan_premap_shadow.cpp \ asan_report.cpp \ - asan_rtems.cpp \ asan_rtl.cpp \ asan_shadow_setup.cpp \ asan_stack.cpp \ diff --git a/libsanitizer/asan/Makefile.in b/libsanitizer/asan/Makefile.in index 25c7fd7..53efe52 100644 --- a/libsanitizer/asan/Makefile.in +++ b/libsanitizer/asan/Makefile.in @@ -156,9 +156,9 @@ am__objects_1 = asan_activation.lo asan_allocator.lo asan_debugging.lo \ asan_interceptors_memintrinsics.lo asan_linux.lo asan_mac.lo \ asan_malloc_linux.lo asan_malloc_mac.lo asan_malloc_win.lo \ asan_memory_profile.lo asan_new_delete.lo asan_poisoning.lo \ - asan_posix.lo asan_premap_shadow.lo asan_report.lo \ - asan_rtems.lo asan_rtl.lo asan_shadow_setup.lo asan_stack.lo \ - asan_stats.lo asan_suppressions.lo asan_thread.lo asan_win.lo \ + asan_posix.lo asan_premap_shadow.lo asan_report.lo asan_rtl.lo \ + asan_shadow_setup.lo asan_stack.lo asan_stats.lo \ + asan_suppressions.lo asan_thread.lo asan_win.lo \ asan_win_dll_thunk.lo asan_win_dynamic_runtime_thunk.lo \ asan_interceptors_vfork.lo am_libasan_la_OBJECTS = $(am__objects_1) @@ -446,7 +446,6 @@ asan_files = \ asan_posix.cpp \ asan_premap_shadow.cpp \ asan_report.cpp \ - asan_rtems.cpp \ asan_rtl.cpp \ asan_shadow_setup.cpp \ asan_stack.cpp \ @@ -604,7 +603,6 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_posix.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_premap_shadow.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_report.Plo@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_rtems.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_rtl.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_shadow_setup.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/asan_stack.Plo@am__quote@ diff --git a/libsanitizer/asan/asan_allocator.cpp b/libsanitizer/asan/asan_allocator.cpp index 7c8bb50..414fba3 100644 --- a/libsanitizer/asan/asan_allocator.cpp +++ b/libsanitizer/asan/asan_allocator.cpp @@ -852,12 +852,12 @@ struct Allocator { quarantine.PrintStats(); } - void ForceLock() { + void ForceLock() ACQUIRE(fallback_mutex) { allocator.ForceLock(); fallback_mutex.Lock(); } - void ForceUnlock() { + void ForceUnlock() RELEASE(fallback_mutex) { fallback_mutex.Unlock(); allocator.ForceUnlock(); } @@ -1081,11 +1081,9 @@ uptr asan_mz_size(const void *ptr) { return instance.AllocationSize(reinterpret_cast<uptr>(ptr)); } -void asan_mz_force_lock() { - instance.ForceLock(); -} +void asan_mz_force_lock() NO_THREAD_SAFETY_ANALYSIS { instance.ForceLock(); } -void asan_mz_force_unlock() { +void asan_mz_force_unlock() NO_THREAD_SAFETY_ANALYSIS { instance.ForceUnlock(); } diff --git a/libsanitizer/asan/asan_errors.cpp b/libsanitizer/asan/asan_errors.cpp index e68e697..45166c0 100644 --- a/libsanitizer/asan/asan_errors.cpp +++ b/libsanitizer/asan/asan_errors.cpp @@ -533,7 +533,6 @@ static void PrintLegend(InternalScopedString *str) { PrintShadowByte(str, " ASan internal: ", kAsanInternalHeapMagic); PrintShadowByte(str, " Left alloca redzone: ", kAsanAllocaLeftMagic); PrintShadowByte(str, " Right alloca redzone: ", kAsanAllocaRightMagic); - PrintShadowByte(str, " Shadow gap: ", kAsanShadowGap); } static void PrintShadowBytes(InternalScopedString *str, const char *before, diff --git a/libsanitizer/asan/asan_fake_stack.cpp b/libsanitizer/asan/asan_fake_stack.cpp index 1f873fe..bf5c342 100644 --- a/libsanitizer/asan/asan_fake_stack.cpp +++ b/libsanitizer/asan/asan_fake_stack.cpp @@ -187,7 +187,7 @@ void SetTLSFakeStack(FakeStack *fs) { } static FakeStack *GetFakeStack() { AsanThread *t = GetCurrentThread(); if (!t) return nullptr; - return t->fake_stack(); + return t->get_or_create_fake_stack(); } static FakeStack *GetFakeStackFast() { @@ -198,7 +198,13 @@ static FakeStack *GetFakeStackFast() { return GetFakeStack(); } -ALWAYS_INLINE uptr OnMalloc(uptr class_id, uptr size) { +static FakeStack *GetFakeStackFastAlways() { + if (FakeStack *fs = GetTLSFakeStack()) + return fs; + return GetFakeStack(); +} + +static ALWAYS_INLINE uptr OnMalloc(uptr class_id, uptr size) { FakeStack *fs = GetFakeStackFast(); if (!fs) return 0; uptr local_stack; @@ -210,7 +216,21 @@ ALWAYS_INLINE uptr OnMalloc(uptr class_id, uptr size) { return ptr; } -ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size) { +static ALWAYS_INLINE uptr OnMallocAlways(uptr class_id, uptr size) { + FakeStack *fs = GetFakeStackFastAlways(); + if (!fs) + return 0; + uptr local_stack; + uptr real_stack = reinterpret_cast<uptr>(&local_stack); + FakeFrame *ff = fs->Allocate(fs->stack_size_log(), class_id, real_stack); + if (!ff) + return 0; // Out of fake stack. + uptr ptr = reinterpret_cast<uptr>(ff); + SetShadow(ptr, size, class_id, 0); + return ptr; +} + +static ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size) { FakeStack::Deallocate(ptr, class_id); SetShadow(ptr, size, class_id, kMagic8); } @@ -219,14 +239,18 @@ ALWAYS_INLINE void OnFree(uptr ptr, uptr class_id, uptr size) { // ---------------------- Interface ---------------- {{{1 using namespace __asan; -#define DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(class_id) \ - extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr \ - __asan_stack_malloc_##class_id(uptr size) { \ - return OnMalloc(class_id, size); \ - } \ - extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __asan_stack_free_##class_id( \ - uptr ptr, uptr size) { \ - OnFree(ptr, class_id, size); \ +#define DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(class_id) \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr \ + __asan_stack_malloc_##class_id(uptr size) { \ + return OnMalloc(class_id, size); \ + } \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE uptr \ + __asan_stack_malloc_always_##class_id(uptr size) { \ + return OnMallocAlways(class_id, size); \ + } \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __asan_stack_free_##class_id( \ + uptr ptr, uptr size) { \ + OnFree(ptr, class_id, size); \ } DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(0) @@ -240,7 +264,11 @@ DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(7) DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(8) DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(9) DEFINE_STACK_MALLOC_FREE_WITH_CLASS_ID(10) + extern "C" { +// TODO: remove this method and fix tests that use it by setting +// -asan-use-after-return=never, after modal UAR flag lands +// (https://github.com/google/sanitizers/issues/1394) SANITIZER_INTERFACE_ATTRIBUTE void *__asan_get_current_fake_stack() { return GetFakeStackFast(); } diff --git a/libsanitizer/asan/asan_flags.cpp b/libsanitizer/asan/asan_flags.cpp index cb6a89f..c64e464 100644 --- a/libsanitizer/asan/asan_flags.cpp +++ b/libsanitizer/asan/asan_flags.cpp @@ -155,10 +155,6 @@ void InitializeFlags() { CHECK_LE(f->max_redzone, 2048); CHECK(IsPowerOfTwo(f->redzone)); CHECK(IsPowerOfTwo(f->max_redzone)); - if (SANITIZER_RTEMS) { - CHECK(!f->unmap_shadow_on_exit); - CHECK(!f->protect_shadow_gap); - } // quarantine_size is deprecated but we still honor it. // quarantine_size can not be used together with quarantine_size_mb. diff --git a/libsanitizer/asan/asan_flags.inc b/libsanitizer/asan/asan_flags.inc index 43c70db..514b225 100644 --- a/libsanitizer/asan/asan_flags.inc +++ b/libsanitizer/asan/asan_flags.inc @@ -87,8 +87,7 @@ ASAN_FLAG(bool, check_malloc_usable_size, true, "295.*.") ASAN_FLAG(bool, unmap_shadow_on_exit, false, "If set, explicitly unmaps the (huge) shadow at exit.") -ASAN_FLAG(bool, protect_shadow_gap, !SANITIZER_RTEMS, - "If set, mprotect the shadow gap") +ASAN_FLAG(bool, protect_shadow_gap, true, "If set, mprotect the shadow gap") ASAN_FLAG(bool, print_stats, false, "Print various statistics after printing an error message or if " "atexit=1.") diff --git a/libsanitizer/asan/asan_globals.cpp b/libsanitizer/asan/asan_globals.cpp index e045c31..9d7dbc6 100644 --- a/libsanitizer/asan/asan_globals.cpp +++ b/libsanitizer/asan/asan_globals.cpp @@ -154,6 +154,23 @@ static void CheckODRViolationViaIndicator(const Global *g) { } } +// Check ODR violation for given global G by checking if it's already poisoned. +// We use this method in case compiler doesn't use private aliases for global +// variables. +static void CheckODRViolationViaPoisoning(const Global *g) { + if (__asan_region_is_poisoned(g->beg, g->size_with_redzone)) { + // This check may not be enough: if the first global is much larger + // the entire redzone of the second global may be within the first global. + for (ListOfGlobals *l = list_of_all_globals; l; l = l->next) { + if (g->beg == l->g->beg && + (flags()->detect_odr_violation >= 2 || g->size != l->g->size) && + !IsODRViolationSuppressed(g->name)) + ReportODRViolation(g, FindRegistrationSite(g), + l->g, FindRegistrationSite(l->g)); + } + } +} + // Clang provides two different ways for global variables protection: // it can poison the global itself or its private alias. In former // case we may poison same symbol multiple times, that can help us to @@ -199,6 +216,8 @@ static void RegisterGlobal(const Global *g) { // where two globals with the same name are defined in different modules. if (UseODRIndicator(g)) CheckODRViolationViaIndicator(g); + else + CheckODRViolationViaPoisoning(g); } if (CanPoisonMemory()) PoisonRedZones(*g); diff --git a/libsanitizer/asan/asan_interceptors.cpp b/libsanitizer/asan/asan_interceptors.cpp index 9db7db8..d0a6dd4 100644 --- a/libsanitizer/asan/asan_interceptors.cpp +++ b/libsanitizer/asan/asan_interceptors.cpp @@ -23,25 +23,25 @@ #include "lsan/lsan_common.h" #include "sanitizer_common/sanitizer_libc.h" -// There is no general interception at all on Fuchsia and RTEMS. +// There is no general interception at all on Fuchsia. // Only the functions in asan_interceptors_memintrinsics.cpp are // really defined to replace libc functions. -#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS +#if !SANITIZER_FUCHSIA -#if SANITIZER_POSIX -#include "sanitizer_common/sanitizer_posix.h" -#endif +# if SANITIZER_POSIX +# include "sanitizer_common/sanitizer_posix.h" +# endif -#if ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION || \ - ASAN_INTERCEPT__SJLJ_UNWIND_RAISEEXCEPTION -#include <unwind.h> -#endif +# if ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION || \ + ASAN_INTERCEPT__SJLJ_UNWIND_RAISEEXCEPTION +# include <unwind.h> +# endif -#if defined(__i386) && SANITIZER_LINUX -#define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.1" -#elif defined(__mips__) && SANITIZER_LINUX -#define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.2" -#endif +# if defined(__i386) && SANITIZER_LINUX +# define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.1" +# elif defined(__mips__) && SANITIZER_LINUX +# define ASAN_PTHREAD_CREATE_VERSION "GLIBC_2.2" +# endif namespace __asan { diff --git a/libsanitizer/asan/asan_interceptors.h b/libsanitizer/asan/asan_interceptors.h index 5bd5be6..a9249de 100644 --- a/libsanitizer/asan/asan_interceptors.h +++ b/libsanitizer/asan/asan_interceptors.h @@ -34,10 +34,10 @@ void InitializePlatformInterceptors(); } // namespace __asan -// There is no general interception at all on Fuchsia and RTEMS. +// There is no general interception at all on Fuchsia. // Only the functions in asan_interceptors_memintrinsics.h are // really defined to replace libc functions. -#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS +#if !SANITIZER_FUCHSIA // Use macro to describe if specific function should be // intercepted on a given platform. @@ -81,12 +81,7 @@ void InitializePlatformInterceptors(); #if ASAN_HAS_EXCEPTIONS && !SANITIZER_WINDOWS && !SANITIZER_SOLARIS && \ !SANITIZER_NETBSD # define ASAN_INTERCEPT___CXA_THROW 1 -# if ! defined(ASAN_HAS_CXA_RETHROW_PRIMARY_EXCEPTION) \ - || ASAN_HAS_CXA_RETHROW_PRIMARY_EXCEPTION -# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 1 -# else -# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 0 -# endif +# define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 1 # if defined(_GLIBCXX_SJLJ_EXCEPTIONS) || (SANITIZER_IOS && defined(__arm__)) # define ASAN_INTERCEPT__UNWIND_SJLJ_RAISEEXCEPTION 1 # else diff --git a/libsanitizer/asan/asan_interceptors_memintrinsics.cpp b/libsanitizer/asan/asan_interceptors_memintrinsics.cpp index ccdd515..9c316bb 100644 --- a/libsanitizer/asan/asan_interceptors_memintrinsics.cpp +++ b/libsanitizer/asan/asan_interceptors_memintrinsics.cpp @@ -30,9 +30,9 @@ void *__asan_memmove(void *to, const void *from, uptr size) { ASAN_MEMMOVE_IMPL(nullptr, to, from, size); } -#if SANITIZER_FUCHSIA || SANITIZER_RTEMS +#if SANITIZER_FUCHSIA -// Fuchsia and RTEMS don't use sanitizer_common_interceptors.inc, but +// Fuchsia doesn't use sanitizer_common_interceptors.inc, but // the only things there it wants are these three. Just define them // as aliases here rather than repeating the contents. @@ -40,4 +40,4 @@ extern "C" decltype(__asan_memcpy) memcpy[[gnu::alias("__asan_memcpy")]]; extern "C" decltype(__asan_memmove) memmove[[gnu::alias("__asan_memmove")]]; extern "C" decltype(__asan_memset) memset[[gnu::alias("__asan_memset")]]; -#endif // SANITIZER_FUCHSIA || SANITIZER_RTEMS +#endif // SANITIZER_FUCHSIA diff --git a/libsanitizer/asan/asan_interface.inc b/libsanitizer/asan/asan_interface.inc index 9480104..ea28fc8 100644 --- a/libsanitizer/asan/asan_interface.inc +++ b/libsanitizer/asan/asan_interface.inc @@ -134,6 +134,17 @@ INTERFACE_FUNCTION(__asan_stack_malloc_7) INTERFACE_FUNCTION(__asan_stack_malloc_8) INTERFACE_FUNCTION(__asan_stack_malloc_9) INTERFACE_FUNCTION(__asan_stack_malloc_10) +INTERFACE_FUNCTION(__asan_stack_malloc_always_0) +INTERFACE_FUNCTION(__asan_stack_malloc_always_1) +INTERFACE_FUNCTION(__asan_stack_malloc_always_2) +INTERFACE_FUNCTION(__asan_stack_malloc_always_3) +INTERFACE_FUNCTION(__asan_stack_malloc_always_4) +INTERFACE_FUNCTION(__asan_stack_malloc_always_5) +INTERFACE_FUNCTION(__asan_stack_malloc_always_6) +INTERFACE_FUNCTION(__asan_stack_malloc_always_7) +INTERFACE_FUNCTION(__asan_stack_malloc_always_8) +INTERFACE_FUNCTION(__asan_stack_malloc_always_9) +INTERFACE_FUNCTION(__asan_stack_malloc_always_10) INTERFACE_FUNCTION(__asan_store1) INTERFACE_FUNCTION(__asan_store2) INTERFACE_FUNCTION(__asan_store4) diff --git a/libsanitizer/asan/asan_internal.h b/libsanitizer/asan/asan_internal.h index cfb5492..ad33203 100644 --- a/libsanitizer/asan/asan_internal.h +++ b/libsanitizer/asan/asan_internal.h @@ -35,11 +35,11 @@ // If set, values like allocator chunk size, as well as defaults for some flags // will be changed towards less memory overhead. #ifndef ASAN_LOW_MEMORY -# if SANITIZER_IOS || SANITIZER_ANDROID || SANITIZER_RTEMS -# define ASAN_LOW_MEMORY 1 -# else -# define ASAN_LOW_MEMORY 0 -# endif +# if SANITIZER_IOS || SANITIZER_ANDROID +# define ASAN_LOW_MEMORY 1 +# else +# define ASAN_LOW_MEMORY 0 +# endif #endif #ifndef ASAN_DYNAMIC @@ -77,7 +77,7 @@ void InitializeShadowMemory(); // asan_malloc_linux.cpp / asan_malloc_mac.cpp void ReplaceSystemMalloc(); -// asan_linux.cpp / asan_mac.cpp / asan_rtems.cpp / asan_win.cpp +// asan_linux.cpp / asan_mac.cpp / asan_win.cpp uptr FindDynamicShadowStart(); void *AsanDoesNotSupportStaticLinkage(); void AsanCheckDynamicRTPrereqs(); @@ -159,9 +159,6 @@ const int kAsanArrayCookieMagic = 0xac; const int kAsanIntraObjectRedzone = 0xbb; const int kAsanAllocaLeftMagic = 0xca; const int kAsanAllocaRightMagic = 0xcb; -// Used to populate the shadow gap for systems without memory -// protection there (i.e. Myriad). -const int kAsanShadowGap = 0xcc; static const uptr kCurrentStackFrameMagic = 0x41B58AB3; static const uptr kRetiredStackFrameMagic = 0x45E0360E; diff --git a/libsanitizer/asan/asan_malloc_linux.cpp b/libsanitizer/asan/asan_malloc_linux.cpp index 9c3f0a5..c6bec85 100644 --- a/libsanitizer/asan/asan_malloc_linux.cpp +++ b/libsanitizer/asan/asan_malloc_linux.cpp @@ -15,23 +15,22 @@ #include "sanitizer_common/sanitizer_platform.h" #if SANITIZER_FREEBSD || SANITIZER_FUCHSIA || SANITIZER_LINUX || \ - SANITIZER_NETBSD || SANITIZER_RTEMS || SANITIZER_SOLARIS + SANITIZER_NETBSD || SANITIZER_SOLARIS -#include "sanitizer_common/sanitizer_allocator_checks.h" -#include "sanitizer_common/sanitizer_errno.h" -#include "sanitizer_common/sanitizer_tls_get_addr.h" -#include "asan_allocator.h" -#include "asan_interceptors.h" -#include "asan_internal.h" -#include "asan_malloc_local.h" -#include "asan_stack.h" +# include "asan_allocator.h" +# include "asan_interceptors.h" +# include "asan_internal.h" +# include "asan_stack.h" +# include "sanitizer_common/sanitizer_allocator_checks.h" +# include "sanitizer_common/sanitizer_errno.h" +# include "sanitizer_common/sanitizer_tls_get_addr.h" // ---------------------- Replacement functions ---------------- {{{1 using namespace __asan; static uptr allocated_for_dlsym; static uptr last_dlsym_alloc_size_in_words; -static const uptr kDlsymAllocPoolSize = SANITIZER_RTEMS ? 4096 : 1024; +static const uptr kDlsymAllocPoolSize = 1024; static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; static inline bool IsInDlsymAllocPool(const void *ptr) { @@ -82,27 +81,12 @@ static int PosixMemalignFromLocalPool(void **memptr, uptr alignment, return 0; } -#if SANITIZER_RTEMS -void* MemalignFromLocalPool(uptr alignment, uptr size) { - void *ptr = nullptr; - alignment = Max(alignment, kWordSize); - PosixMemalignFromLocalPool(&ptr, alignment, size); - return ptr; -} - -bool IsFromLocalPool(const void *ptr) { - return IsInDlsymAllocPool(ptr); -} -#endif - static inline bool MaybeInDlsym() { // Fuchsia doesn't use dlsym-based interceptors. return !SANITIZER_FUCHSIA && asan_init_is_running; } -static inline bool UseLocalPool() { - return EarlyMalloc() || MaybeInDlsym(); -} +static inline bool UseLocalPool() { return MaybeInDlsym(); } static void *ReallocFromLocalPool(void *ptr, uptr size) { const uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; diff --git a/libsanitizer/asan/asan_malloc_local.h b/libsanitizer/asan/asan_malloc_local.h deleted file mode 100644 index e2c9be0..0000000 --- a/libsanitizer/asan/asan_malloc_local.h +++ /dev/null @@ -1,52 +0,0 @@ -//===-- asan_malloc_local.h -------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is a part of AddressSanitizer, an address sanity checker. -// -// Provide interfaces to check for and handle local pool memory allocation. -//===----------------------------------------------------------------------===// - -#ifndef ASAN_MALLOC_LOCAL_H -#define ASAN_MALLOC_LOCAL_H - -#include "sanitizer_common/sanitizer_platform.h" -#include "asan_internal.h" - -static inline bool EarlyMalloc() { - return SANITIZER_RTEMS && - (!__asan::asan_inited || __asan::asan_init_is_running); -} - -#if SANITIZER_RTEMS - -bool IsFromLocalPool(const void *ptr); -void *MemalignFromLocalPool(uptr alignment, uptr size); - -// On RTEMS, we use the local pool to handle memory allocation when the ASan -// run-time is not up. This macro is expanded in the context of the operator new -// implementation. -#define MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow) \ - do { \ - if (UNLIKELY(EarlyMalloc())) { \ - void *res = MemalignFromLocalPool(SHADOW_GRANULARITY, size); \ - if (!nothrow) \ - CHECK(res); \ - return res; \ - } \ - } while (0) - -#define IS_FROM_LOCAL_POOL(ptr) UNLIKELY(IsFromLocalPool(ptr)) - -#else // SANITIZER_RTEMS - -#define MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow) -#define IS_FROM_LOCAL_POOL(ptr) 0 - -#endif // SANITIZER_RTEMS - -#endif // ASAN_MALLOC_LOCAL_H diff --git a/libsanitizer/asan/asan_mapping.h b/libsanitizer/asan/asan_mapping.h index 2511e70..e5a7f20 100644 --- a/libsanitizer/asan/asan_mapping.h +++ b/libsanitizer/asan/asan_mapping.h @@ -150,17 +150,11 @@ // || `[0x36000000, 0x39ffffff]` || ShadowGap || // || `[0x30000000, 0x35ffffff]` || LowShadow || // || `[0x00000000, 0x2fffffff]` || LowMem || -// -// Shadow mapping on Myriad2 (for shadow scale 5): -// || `[0x9ff80000, 0x9fffffff]` || ShadowGap || -// || `[0x9f000000, 0x9ff7ffff]` || LowShadow || -// || `[0x80000000, 0x9effffff]` || LowMem || -// || `[0x00000000, 0x7fffffff]` || Ignored || #if defined(ASAN_SHADOW_SCALE) static const u64 kDefaultShadowScale = ASAN_SHADOW_SCALE; #else -static const u64 kDefaultShadowScale = SANITIZER_MYRIAD2 ? 5 : 3; +static const u64 kDefaultShadowScale = 3; #endif static const u64 kDefaultShadowSentinel = ~(uptr)0; static const u64 kDefaultShadowOffset32 = 1ULL << 29; // 0x20000000 @@ -171,7 +165,7 @@ static const u64 kAArch64_ShadowOffset64 = 1ULL << 36; static const u64 kRiscv64_ShadowOffset64 = 0xd55550000; static const u64 kMIPS32_ShadowOffset32 = 0x0aaa0000; static const u64 kMIPS64_ShadowOffset64 = 1ULL << 37; -static const u64 kPPC64_ShadowOffset64 = 1ULL << 41; +static const u64 kPPC64_ShadowOffset64 = 1ULL << 44; static const u64 kSystemZ_ShadowOffset64 = 1ULL << 52; static const u64 kSPARC64_ShadowOffset64 = 1ULL << 43; // 0x80000000000 static const u64 kFreeBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000 @@ -180,15 +174,6 @@ static const u64 kNetBSD_ShadowOffset32 = 1ULL << 30; // 0x40000000 static const u64 kNetBSD_ShadowOffset64 = 1ULL << 46; // 0x400000000000 static const u64 kWindowsShadowOffset32 = 3ULL << 28; // 0x30000000 -static const u64 kMyriadMemoryOffset32 = 0x80000000ULL; -static const u64 kMyriadMemorySize32 = 0x20000000ULL; -static const u64 kMyriadMemoryEnd32 = - kMyriadMemoryOffset32 + kMyriadMemorySize32 - 1; -static const u64 kMyriadShadowOffset32 = - (kMyriadMemoryOffset32 + kMyriadMemorySize32 - - (kMyriadMemorySize32 >> kDefaultShadowScale)); -static const u64 kMyriadCacheBitMask32 = 0x40000000ULL; - #define SHADOW_SCALE kDefaultShadowScale #if SANITIZER_FUCHSIA @@ -206,8 +191,6 @@ static const u64 kMyriadCacheBitMask32 = 0x40000000ULL; # define SHADOW_OFFSET kWindowsShadowOffset32 # elif SANITIZER_IOS # define SHADOW_OFFSET __asan_shadow_memory_dynamic_address -# elif SANITIZER_MYRIAD2 -# define SHADOW_OFFSET kMyriadShadowOffset32 # else # define SHADOW_OFFSET kDefaultShadowOffset32 # endif @@ -278,10 +261,8 @@ extern uptr kHighMemEnd, kMidMemBeg, kMidMemEnd; // Initialized in __asan_init. } // namespace __asan -#if SANITIZER_MYRIAD2 -#include "asan_mapping_myriad.h" -#elif defined(__sparc__) && SANITIZER_WORDSIZE == 64 -#include "asan_mapping_sparc64.h" +#if defined(__sparc__) && SANITIZER_WORDSIZE == 64 +# include "asan_mapping_sparc64.h" #else #define MEM_TO_SHADOW(mem) (((mem) >> SHADOW_SCALE) + (SHADOW_OFFSET)) @@ -363,7 +344,7 @@ static inline bool AddrIsInShadowGap(uptr a) { } // namespace __asan -#endif // SANITIZER_MYRIAD2 +#endif namespace __asan { @@ -393,8 +374,6 @@ static inline bool AddrIsAlignedByGranularity(uptr a) { static inline bool AddressIsPoisoned(uptr a) { PROFILE_ASAN_MAPPING(); - if (SANITIZER_MYRIAD2 && !AddrIsInMem(a) && !AddrIsInShadow(a)) - return false; const uptr kAccessSize = 1; u8 *shadow_address = (u8*)MEM_TO_SHADOW(a); s8 shadow_value = *shadow_address; diff --git a/libsanitizer/asan/asan_mapping_myriad.h b/libsanitizer/asan/asan_mapping_myriad.h deleted file mode 100644 index 6969e3a..0000000 --- a/libsanitizer/asan/asan_mapping_myriad.h +++ /dev/null @@ -1,85 +0,0 @@ -//===-- asan_mapping_myriad.h -----------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is a part of AddressSanitizer, an address sanity checker. -// -// Myriad-specific definitions for ASan memory mapping. -//===----------------------------------------------------------------------===// -#ifndef ASAN_MAPPING_MYRIAD_H -#define ASAN_MAPPING_MYRIAD_H - -#define RAW_ADDR(mem) ((mem) & ~kMyriadCacheBitMask32) -#define MEM_TO_SHADOW(mem) \ - (((RAW_ADDR(mem) - kLowMemBeg) >> SHADOW_SCALE) + (SHADOW_OFFSET)) - -#define kLowMemBeg kMyriadMemoryOffset32 -#define kLowMemEnd (SHADOW_OFFSET - 1) - -#define kLowShadowBeg SHADOW_OFFSET -#define kLowShadowEnd MEM_TO_SHADOW(kLowMemEnd) - -#define kHighMemBeg 0 - -#define kHighShadowBeg 0 -#define kHighShadowEnd 0 - -#define kMidShadowBeg 0 -#define kMidShadowEnd 0 - -#define kShadowGapBeg (kLowShadowEnd + 1) -#define kShadowGapEnd kMyriadMemoryEnd32 - -#define kShadowGap2Beg 0 -#define kShadowGap2End 0 - -#define kShadowGap3Beg 0 -#define kShadowGap3End 0 - -namespace __asan { - -static inline bool AddrIsInLowMem(uptr a) { - PROFILE_ASAN_MAPPING(); - a = RAW_ADDR(a); - return a >= kLowMemBeg && a <= kLowMemEnd; -} - -static inline bool AddrIsInLowShadow(uptr a) { - PROFILE_ASAN_MAPPING(); - a = RAW_ADDR(a); - return a >= kLowShadowBeg && a <= kLowShadowEnd; -} - -static inline bool AddrIsInMidMem(uptr a) { - PROFILE_ASAN_MAPPING(); - return false; -} - -static inline bool AddrIsInMidShadow(uptr a) { - PROFILE_ASAN_MAPPING(); - return false; -} - -static inline bool AddrIsInHighMem(uptr a) { - PROFILE_ASAN_MAPPING(); - return false; -} - -static inline bool AddrIsInHighShadow(uptr a) { - PROFILE_ASAN_MAPPING(); - return false; -} - -static inline bool AddrIsInShadowGap(uptr a) { - PROFILE_ASAN_MAPPING(); - a = RAW_ADDR(a); - return a >= kShadowGapBeg && a <= kShadowGapEnd; -} - -} // namespace __asan - -#endif // ASAN_MAPPING_MYRIAD_H diff --git a/libsanitizer/asan/asan_new_delete.cpp b/libsanitizer/asan/asan_new_delete.cpp index 92a8648..da44607 100644 --- a/libsanitizer/asan/asan_new_delete.cpp +++ b/libsanitizer/asan/asan_new_delete.cpp @@ -11,16 +11,14 @@ // Interceptors for operators new and delete. //===----------------------------------------------------------------------===// +#include <stddef.h> + #include "asan_allocator.h" #include "asan_internal.h" -#include "asan_malloc_local.h" #include "asan_report.h" #include "asan_stack.h" - #include "interception/interception.h" -#include <stddef.h> - // C++ operators can't have dllexport attributes on Windows. We export them // anyway by passing extra -export flags to the linker, which is exactly that // dllexport would normally do. We need to export them in order to make the @@ -72,14 +70,12 @@ enum class align_val_t: size_t {}; // For local pool allocation, align to SHADOW_GRANULARITY to match asan // allocator behavior. #define OPERATOR_NEW_BODY(type, nothrow) \ - MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow); \ GET_STACK_TRACE_MALLOC; \ void *res = asan_memalign(0, size, &stack, type); \ if (!nothrow && UNLIKELY(!res)) \ ReportOutOfMemory(size, &stack); \ return res; #define OPERATOR_NEW_BODY_ALIGN(type, nothrow) \ - MAYBE_ALLOCATE_FROM_LOCAL_POOL(nothrow); \ GET_STACK_TRACE_MALLOC; \ void *res = asan_memalign((uptr)align, size, &stack, type); \ if (!nothrow && UNLIKELY(!res)) \ @@ -135,23 +131,19 @@ INTERCEPTOR(void *, _ZnamRKSt9nothrow_t, size_t size, std::nothrow_t const&) { #endif // !SANITIZER_MAC #define OPERATOR_DELETE_BODY(type) \ - if (IS_FROM_LOCAL_POOL(ptr)) return;\ - GET_STACK_TRACE_FREE;\ + GET_STACK_TRACE_FREE; \ asan_delete(ptr, 0, 0, &stack, type); #define OPERATOR_DELETE_BODY_SIZE(type) \ - if (IS_FROM_LOCAL_POOL(ptr)) return;\ - GET_STACK_TRACE_FREE;\ + GET_STACK_TRACE_FREE; \ asan_delete(ptr, size, 0, &stack, type); #define OPERATOR_DELETE_BODY_ALIGN(type) \ - if (IS_FROM_LOCAL_POOL(ptr)) return;\ - GET_STACK_TRACE_FREE;\ + GET_STACK_TRACE_FREE; \ asan_delete(ptr, 0, static_cast<uptr>(align), &stack, type); #define OPERATOR_DELETE_BODY_SIZE_ALIGN(type) \ - if (IS_FROM_LOCAL_POOL(ptr)) return;\ - GET_STACK_TRACE_FREE;\ + GET_STACK_TRACE_FREE; \ asan_delete(ptr, size, static_cast<uptr>(align), &stack, type); #if !SANITIZER_MAC diff --git a/libsanitizer/asan/asan_poisoning.cpp b/libsanitizer/asan/asan_poisoning.cpp index fa149ec..5f215fe 100644 --- a/libsanitizer/asan/asan_poisoning.cpp +++ b/libsanitizer/asan/asan_poisoning.cpp @@ -173,17 +173,13 @@ int __asan_address_is_poisoned(void const volatile *addr) { } uptr __asan_region_is_poisoned(uptr beg, uptr size) { - if (!size) return 0; + if (!size) + return 0; uptr end = beg + size; - if (SANITIZER_MYRIAD2) { - // On Myriad, address not in DRAM range need to be treated as - // unpoisoned. - if (!AddrIsInMem(beg) && !AddrIsInShadow(beg)) return 0; - if (!AddrIsInMem(end) && !AddrIsInShadow(end)) return 0; - } else { - if (!AddrIsInMem(beg)) return beg; - if (!AddrIsInMem(end)) return end; - } + if (!AddrIsInMem(beg)) + return beg; + if (!AddrIsInMem(end)) + return end; CHECK_LT(beg, end); uptr aligned_b = RoundUpTo(beg, SHADOW_GRANULARITY); uptr aligned_e = RoundDownTo(end, SHADOW_GRANULARITY); @@ -192,8 +188,7 @@ uptr __asan_region_is_poisoned(uptr beg, uptr size) { // First check the first and the last application bytes, // then check the SHADOW_GRANULARITY-aligned region by calling // mem_is_zero on the corresponding shadow. - if (!__asan::AddressIsPoisoned(beg) && - !__asan::AddressIsPoisoned(end - 1) && + if (!__asan::AddressIsPoisoned(beg) && !__asan::AddressIsPoisoned(end - 1) && (shadow_end <= shadow_beg || __sanitizer::mem_is_zero((const char *)shadow_beg, shadow_end - shadow_beg))) diff --git a/libsanitizer/asan/asan_poisoning.h b/libsanitizer/asan/asan_poisoning.h index 62dd9bd..3d536f2 100644 --- a/libsanitizer/asan/asan_poisoning.h +++ b/libsanitizer/asan/asan_poisoning.h @@ -51,9 +51,6 @@ ALWAYS_INLINE void FastPoisonShadow(uptr aligned_beg, uptr aligned_size, // probably provide higher-level interface for these operations. // For now, just memset on Windows. if (value || SANITIZER_WINDOWS == 1 || - // RTEMS doesn't have have pages, let alone a fast way to zero - // them, so default to memset. - SANITIZER_RTEMS == 1 || shadow_end - shadow_beg < common_flags()->clear_shadow_mmap_threshold) { REAL(memset)((void*)shadow_beg, value, shadow_end - shadow_beg); } else { diff --git a/libsanitizer/asan/asan_rtems.cpp b/libsanitizer/asan/asan_rtems.cpp deleted file mode 100644 index ea0b4ad..0000000 --- a/libsanitizer/asan/asan_rtems.cpp +++ /dev/null @@ -1,266 +0,0 @@ -//===-- asan_rtems.cpp ----------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is a part of AddressSanitizer, an address sanity checker. -// -// RTEMS-specific details. -//===----------------------------------------------------------------------===// - -#include "sanitizer_common/sanitizer_rtems.h" -#if SANITIZER_RTEMS - -#include "asan_internal.h" -#include "asan_interceptors.h" -#include "asan_mapping.h" -#include "asan_poisoning.h" -#include "asan_report.h" -#include "asan_stack.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_libc.h" - -#include <pthread.h> -#include <stdlib.h> - -namespace __asan { - -static void ResetShadowMemory() { - uptr shadow_start = SHADOW_OFFSET; - uptr shadow_end = MEM_TO_SHADOW(kMyriadMemoryEnd32); - uptr gap_start = MEM_TO_SHADOW(shadow_start); - uptr gap_end = MEM_TO_SHADOW(shadow_end); - - REAL(memset)((void *)shadow_start, 0, shadow_end - shadow_start); - REAL(memset)((void *)gap_start, kAsanShadowGap, gap_end - gap_start); -} - -void InitializeShadowMemory() { - kHighMemEnd = 0; - kMidMemBeg = 0; - kMidMemEnd = 0; - - ResetShadowMemory(); -} - -void AsanApplyToGlobals(globals_op_fptr op, const void *needle) { - UNIMPLEMENTED(); -} - -void FlushUnneededASanShadowMemory(uptr p, uptr size) { - // Since asan's mapping is compacting, the shadow chunk may be - // not page-aligned, so we only flush the page-aligned portion. - ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size)); -} - -void AsanCheckDynamicRTPrereqs() {} -void AsanCheckIncompatibleRT() {} -void InitializeAsanInterceptors() {} -void InitializePlatformInterceptors() {} -void InitializePlatformExceptionHandlers() {} - -// RTEMS only support static linking; it sufficies to return with no -// error. -void *AsanDoesNotSupportStaticLinkage() { return nullptr; } - -void AsanOnDeadlySignal(int signo, void *siginfo, void *context) { - UNIMPLEMENTED(); -} - -bool PlatformUnpoisonStacks() { return false; } - -void EarlyInit() { - // Provide early initialization of shadow memory so that - // instrumented code running before full initialzation will not - // report spurious errors. - ResetShadowMemory(); -} - -// We can use a plain thread_local variable for TSD. -static thread_local void *per_thread; - -void *AsanTSDGet() { return per_thread; } - -void AsanTSDSet(void *tsd) { per_thread = tsd; } - -// There's no initialization needed, and the passed-in destructor -// will never be called. Instead, our own thread destruction hook -// (below) will call AsanThread::TSDDtor directly. -void AsanTSDInit(void (*destructor)(void *tsd)) { - DCHECK(destructor == &PlatformTSDDtor); -} - -void PlatformTSDDtor(void *tsd) { UNREACHABLE(__func__); } - -// -// Thread registration. We provide an API similar to the Fushia port. -// - -struct AsanThread::InitOptions { - uptr stack_bottom, stack_size, tls_bottom, tls_size; -}; - -// Shared setup between thread creation and startup for the initial thread. -static AsanThread *CreateAsanThread(StackTrace *stack, u32 parent_tid, - uptr user_id, bool detached, - uptr stack_bottom, uptr stack_size, - uptr tls_bottom, uptr tls_size) { - // In lieu of AsanThread::Create. - AsanThread *thread = (AsanThread *)MmapOrDie(sizeof(AsanThread), __func__); - AsanThreadContext::CreateThreadContextArgs args = {thread, stack}; - asanThreadRegistry().CreateThread(user_id, detached, parent_tid, &args); - - // On other systems, AsanThread::Init() is called from the new - // thread itself. But on RTEMS we already know the stack address - // range beforehand, so we can do most of the setup right now. - const AsanThread::InitOptions options = {stack_bottom, stack_size, - tls_bottom, tls_size}; - thread->Init(&options); - return thread; -} - -// This gets the same arguments passed to Init by CreateAsanThread, above. -// We're in the creator thread before the new thread is actually started, but -// its stack and tls address range are already known. -void AsanThread::SetThreadStackAndTls(const AsanThread::InitOptions *options) { - DCHECK_NE(GetCurrentThread(), this); - DCHECK_NE(GetCurrentThread(), nullptr); - CHECK_NE(options->stack_bottom, 0); - CHECK_NE(options->stack_size, 0); - stack_bottom_ = options->stack_bottom; - stack_top_ = options->stack_bottom + options->stack_size; - tls_begin_ = options->tls_bottom; - tls_end_ = options->tls_bottom + options->tls_size; -} - -// Called by __asan::AsanInitInternal (asan_rtl.c). Unlike other ports, the -// main thread on RTEMS does not require special treatment; its AsanThread is -// already created by the provided hooks. This function simply looks up and -// returns the created thread. -AsanThread *CreateMainThread() { - return GetThreadContextByTidLocked(0)->thread; -} - -// This is called before each thread creation is attempted. So, in -// its first call, the calling thread is the initial and sole thread. -static void *BeforeThreadCreateHook(uptr user_id, bool detached, - uptr stack_bottom, uptr stack_size, - uptr tls_bottom, uptr tls_size) { - EnsureMainThreadIDIsCorrect(); - // Strict init-order checking is thread-hostile. - if (flags()->strict_init_order) StopInitOrderChecking(); - - GET_STACK_TRACE_THREAD; - u32 parent_tid = GetCurrentTidOrInvalid(); - - return CreateAsanThread(&stack, parent_tid, user_id, detached, - stack_bottom, stack_size, tls_bottom, tls_size); -} - -// This is called after creating a new thread (in the creating thread), -// with the pointer returned by BeforeThreadCreateHook (above). -static void ThreadCreateHook(void *hook, bool aborted) { - AsanThread *thread = static_cast<AsanThread *>(hook); - if (!aborted) { - // The thread was created successfully. - // ThreadStartHook is already running in the new thread. - } else { - // The thread wasn't created after all. - // Clean up everything we set up in BeforeThreadCreateHook. - asanThreadRegistry().FinishThread(thread->tid()); - UnmapOrDie(thread, sizeof(AsanThread)); - } -} - -// This is called (1) in the newly-created thread before it runs anything else, -// with the pointer returned by BeforeThreadCreateHook (above). (2) before a -// thread restart. -static void ThreadStartHook(void *hook, uptr os_id) { - if (!hook) - return; - - AsanThread *thread = static_cast<AsanThread *>(hook); - SetCurrentThread(thread); - - ThreadStatus status = - asanThreadRegistry().GetThreadLocked(thread->tid())->status; - DCHECK(status == ThreadStatusCreated || status == ThreadStatusRunning); - // Determine whether we are starting or restarting the thread. - if (status == ThreadStatusCreated) { - // In lieu of AsanThread::ThreadStart. - asanThreadRegistry().StartThread(thread->tid(), os_id, ThreadType::Regular, - nullptr); - } else { - // In a thread restart, a thread may resume execution at an - // arbitrary function entry point, with its stack and TLS state - // reset. We unpoison the stack in that case. - PoisonShadow(thread->stack_bottom(), thread->stack_size(), 0); - } -} - -// Each thread runs this just before it exits, -// with the pointer returned by BeforeThreadCreateHook (above). -// All per-thread destructors have already been called. -static void ThreadExitHook(void *hook, uptr os_id) { - AsanThread *thread = static_cast<AsanThread *>(hook); - if (thread) - AsanThread::TSDDtor(thread->context()); -} - -static void HandleExit() { - // Disable ASan by setting it to uninitialized. Also reset the - // shadow memory to avoid reporting errors after the run-time has - // been desroyed. - if (asan_inited) { - asan_inited = false; - ResetShadowMemory(); - } -} - -bool HandleDlopenInit() { - // Not supported on this platform. - static_assert(!SANITIZER_SUPPORTS_INIT_FOR_DLOPEN, - "Expected SANITIZER_SUPPORTS_INIT_FOR_DLOPEN to be false"); - return false; -} -} // namespace __asan - -// These are declared (in extern "C") by <some_path/sanitizer.h>. -// The system runtime will call our definitions directly. - -extern "C" { -void __sanitizer_early_init() { - __asan::EarlyInit(); -} - -void *__sanitizer_before_thread_create_hook(uptr thread, bool detached, - const char *name, - void *stack_base, size_t stack_size, - void *tls_base, size_t tls_size) { - return __asan::BeforeThreadCreateHook( - thread, detached, - reinterpret_cast<uptr>(stack_base), stack_size, - reinterpret_cast<uptr>(tls_base), tls_size); -} - -void __sanitizer_thread_create_hook(void *handle, uptr thread, int status) { - __asan::ThreadCreateHook(handle, status != 0); -} - -void __sanitizer_thread_start_hook(void *handle, uptr self) { - __asan::ThreadStartHook(handle, self); -} - -void __sanitizer_thread_exit_hook(void *handle, uptr self) { - __asan::ThreadExitHook(handle, self); -} - -void __sanitizer_exit() { - __asan::HandleExit(); -} -} // "C" - -#endif // SANITIZER_RTEMS diff --git a/libsanitizer/asan/asan_rtl.cpp b/libsanitizer/asan/asan_rtl.cpp index e715d77..e06a111 100644 --- a/libsanitizer/asan/asan_rtl.cpp +++ b/libsanitizer/asan/asan_rtl.cpp @@ -13,6 +13,7 @@ #include "asan_activation.h" #include "asan_allocator.h" +#include "asan_fake_stack.h" #include "asan_interceptors.h" #include "asan_interface_internal.h" #include "asan_internal.h" @@ -23,11 +24,11 @@ #include "asan_stats.h" #include "asan_suppressions.h" #include "asan_thread.h" +#include "lsan/lsan_common.h" #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_symbolizer.h" -#include "lsan/lsan_common.h" #include "ubsan/ubsan_init.h" #include "ubsan/ubsan_platform.h" @@ -137,24 +138,21 @@ ASAN_REPORT_ERROR_N(load, false) ASAN_REPORT_ERROR_N(store, true) #define ASAN_MEMORY_ACCESS_CALLBACK_BODY(type, is_write, size, exp_arg, fatal) \ - if (SANITIZER_MYRIAD2 && !AddrIsInMem(addr) && !AddrIsInShadow(addr)) \ - return; \ - uptr sp = MEM_TO_SHADOW(addr); \ - uptr s = size <= SHADOW_GRANULARITY ? *reinterpret_cast<u8 *>(sp) \ - : *reinterpret_cast<u16 *>(sp); \ - if (UNLIKELY(s)) { \ - if (UNLIKELY(size >= SHADOW_GRANULARITY || \ - ((s8)((addr & (SHADOW_GRANULARITY - 1)) + size - 1)) >= \ - (s8)s)) { \ - if (__asan_test_only_reported_buggy_pointer) { \ - *__asan_test_only_reported_buggy_pointer = addr; \ - } else { \ - GET_CALLER_PC_BP_SP; \ - ReportGenericError(pc, bp, sp, addr, is_write, size, exp_arg, \ - fatal); \ - } \ + uptr sp = MEM_TO_SHADOW(addr); \ + uptr s = size <= SHADOW_GRANULARITY ? *reinterpret_cast<u8 *>(sp) \ + : *reinterpret_cast<u16 *>(sp); \ + if (UNLIKELY(s)) { \ + if (UNLIKELY(size >= SHADOW_GRANULARITY || \ + ((s8)((addr & (SHADOW_GRANULARITY - 1)) + size - 1)) >= \ + (s8)s)) { \ + if (__asan_test_only_reported_buggy_pointer) { \ + *__asan_test_only_reported_buggy_pointer = addr; \ + } else { \ + GET_CALLER_PC_BP_SP; \ + ReportGenericError(pc, bp, sp, addr, is_write, size, exp_arg, fatal); \ } \ - } + } \ + } #define ASAN_MEMORY_ACCESS_CALLBACK(type, is_write, size) \ extern "C" NOINLINE INTERFACE_ATTRIBUTE \ @@ -305,7 +303,6 @@ static void asan_atexit() { } static void InitializeHighMemEnd() { -#if !SANITIZER_MYRIAD2 #if !ASAN_FIXED_MAPPING kHighMemEnd = GetMaxUserVirtualAddress(); // Increase kHighMemEnd to make sure it's properly @@ -313,7 +310,6 @@ static void InitializeHighMemEnd() { kHighMemEnd |= (GetMmapGranularity() << SHADOW_SCALE) - 1; #endif // !ASAN_FIXED_MAPPING CHECK_EQ((kHighMemBeg % GetMmapGranularity()), 0); -#endif // !SANITIZER_MYRIAD2 } void PrintAddressSpaceLayout() { @@ -569,9 +565,6 @@ static void UnpoisonDefaultStack() { const uptr page_size = GetPageSizeCached(); top = curr_thread->stack_top(); bottom = ((uptr)&local_stack - page_size) & ~(page_size - 1); - } else if (SANITIZER_RTEMS) { - // Give up On RTEMS. - return; } else { CHECK(!SANITIZER_FUCHSIA); // If we haven't seen this thread, try asking the OS for stack bounds. @@ -586,8 +579,12 @@ static void UnpoisonDefaultStack() { static void UnpoisonFakeStack() { AsanThread *curr_thread = GetCurrentThread(); - if (curr_thread && curr_thread->has_fake_stack()) - curr_thread->fake_stack()->HandleNoReturn(); + if (!curr_thread) + return; + FakeStack *stack = curr_thread->get_fake_stack(); + if (!stack) + return; + stack->HandleNoReturn(); } } // namespace __asan diff --git a/libsanitizer/asan/asan_shadow_setup.cpp b/libsanitizer/asan/asan_shadow_setup.cpp index 2ead442..6e6260d 100644 --- a/libsanitizer/asan/asan_shadow_setup.cpp +++ b/libsanitizer/asan/asan_shadow_setup.cpp @@ -13,12 +13,11 @@ #include "sanitizer_common/sanitizer_platform.h" -// asan_fuchsia.cpp and asan_rtems.cpp have their own -// InitializeShadowMemory implementation. -#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS +// asan_fuchsia.cpp has their own InitializeShadowMemory implementation. +#if !SANITIZER_FUCHSIA -#include "asan_internal.h" -#include "asan_mapping.h" +# include "asan_internal.h" +# include "asan_mapping.h" namespace __asan { @@ -123,4 +122,4 @@ void InitializeShadowMemory() { } // namespace __asan -#endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS +#endif // !SANITIZER_FUCHSIA diff --git a/libsanitizer/asan/asan_stack.cpp b/libsanitizer/asan/asan_stack.cpp index b7f4e6a..048295d 100644 --- a/libsanitizer/asan/asan_stack.cpp +++ b/libsanitizer/asan/asan_stack.cpp @@ -74,7 +74,8 @@ void __sanitizer::BufferedStackTrace::UnwindImpl( if (SANITIZER_MIPS && t && !IsValidFrame(bp, t->stack_top(), t->stack_bottom())) return; - Unwind(max_depth, pc, bp, context, 0, 0, false); + Unwind(max_depth, pc, bp, context, t ? t->stack_top() : 0, + t ? t->stack_bottom() : 0, false); } // ------------------ Interface -------------- {{{1 diff --git a/libsanitizer/asan/asan_thread.cpp b/libsanitizer/asan/asan_thread.cpp index 9c3c86f..35d4467 100644 --- a/libsanitizer/asan/asan_thread.cpp +++ b/libsanitizer/asan/asan_thread.cpp @@ -60,8 +60,8 @@ ThreadRegistry &asanThreadRegistry() { // in TSD and can't reliably tell when no more TSD destructors will // be called. It would be wrong to reuse AsanThreadContext for another // thread before all TSD destructors will be called for it. - asan_thread_registry = new(thread_registry_placeholder) ThreadRegistry( - GetAsanThreadContext, kMaxNumberOfThreads, kMaxNumberOfThreads); + asan_thread_registry = + new (thread_registry_placeholder) ThreadRegistry(GetAsanThreadContext); initialized = true; } return *asan_thread_registry; @@ -257,10 +257,9 @@ void AsanThread::Init(const InitOptions *options) { &local); } -// Fuchsia and RTEMS don't use ThreadStart. -// asan_fuchsia.c/asan_rtems.c define CreateMainThread and -// SetThreadStackAndTls. -#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS +// Fuchsia doesn't use ThreadStart. +// asan_fuchsia.c definies CreateMainThread and SetThreadStackAndTls. +#if !SANITIZER_FUCHSIA thread_return_t AsanThread::ThreadStart(tid_t os_id) { Init(); @@ -317,7 +316,7 @@ void AsanThread::SetThreadStackAndTls(const InitOptions *options) { } } -#endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS +#endif // !SANITIZER_FUCHSIA void AsanThread::ClearShadowForThreadStackAndTLS() { if (stack_top_ != stack_bottom_) @@ -339,8 +338,8 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr, uptr bottom = 0; if (AddrIsInStack(addr)) { bottom = stack_bottom(); - } else if (has_fake_stack()) { - bottom = fake_stack()->AddrIsInFakeStack(addr); + } else if (FakeStack *fake_stack = get_fake_stack()) { + bottom = fake_stack->AddrIsInFakeStack(addr); CHECK(bottom); access->offset = addr - bottom; access->frame_pc = ((uptr*)bottom)[2]; @@ -380,8 +379,8 @@ uptr AsanThread::GetStackVariableShadowStart(uptr addr) { uptr bottom = 0; if (AddrIsInStack(addr)) { bottom = stack_bottom(); - } else if (has_fake_stack()) { - bottom = fake_stack()->AddrIsInFakeStack(addr); + } else if (FakeStack *fake_stack = get_fake_stack()) { + bottom = fake_stack->AddrIsInFakeStack(addr); if (bottom == 0) { return 0; } @@ -409,19 +408,19 @@ bool AsanThread::AddrIsInStack(uptr addr) { static bool ThreadStackContainsAddress(ThreadContextBase *tctx_base, void *addr) { - AsanThreadContext *tctx = static_cast<AsanThreadContext*>(tctx_base); + AsanThreadContext *tctx = static_cast<AsanThreadContext *>(tctx_base); AsanThread *t = tctx->thread; - if (!t) return false; - if (t->AddrIsInStack((uptr)addr)) return true; - if (t->has_fake_stack() && t->fake_stack()->AddrIsInFakeStack((uptr)addr)) + if (!t) + return false; + if (t->AddrIsInStack((uptr)addr)) return true; - return false; + FakeStack *fake_stack = t->get_fake_stack(); + if (!fake_stack) + return false; + return fake_stack->AddrIsInFakeStack((uptr)addr); } AsanThread *GetCurrentThread() { - if (SANITIZER_RTEMS && !asan_inited) - return nullptr; - AsanThreadContext *context = reinterpret_cast<AsanThreadContext *>(AsanTSDGet()); if (!context) { @@ -503,8 +502,12 @@ void GetAllThreadAllocatorCachesLocked(InternalMmapVector<uptr> *caches) {} void ForEachExtraStackRange(tid_t os_id, RangeIteratorCallback callback, void *arg) { __asan::AsanThread *t = __asan::GetAsanThreadByOsIDLocked(os_id); - if (t && t->has_fake_stack()) - t->fake_stack()->ForEachFakeFrame(callback, arg); + if (!t) + return; + __asan::FakeStack *fake_stack = t->get_fake_stack(); + if (!fake_stack) + return; + fake_stack->ForEachFakeFrame(callback, arg); } void LockThreadRegistry() { diff --git a/libsanitizer/asan/asan_thread.h b/libsanitizer/asan/asan_thread.h index 200069c..801a396 100644 --- a/libsanitizer/asan/asan_thread.h +++ b/libsanitizer/asan/asan_thread.h @@ -28,8 +28,6 @@ struct DTLS; namespace __asan { -const u32 kMaxNumberOfThreads = (1 << 22); // 4M - class AsanThread; // These objects are created for every thread and are never deleted, @@ -104,17 +102,18 @@ class AsanThread { void FinishSwitchFiber(FakeStack *fake_stack_save, uptr *bottom_old, uptr *size_old); - bool has_fake_stack() { - return !atomic_load(&stack_switching_, memory_order_relaxed) && - (reinterpret_cast<uptr>(fake_stack_) > 1); + FakeStack *get_fake_stack() { + if (atomic_load(&stack_switching_, memory_order_relaxed)) + return nullptr; + if (reinterpret_cast<uptr>(fake_stack_) <= 1) + return nullptr; + return fake_stack_; } - FakeStack *fake_stack() { - if (!__asan_option_detect_stack_use_after_return) - return nullptr; + FakeStack *get_or_create_fake_stack() { if (atomic_load(&stack_switching_, memory_order_relaxed)) return nullptr; - if (!has_fake_stack()) + if (reinterpret_cast<uptr>(fake_stack_) <= 1) return AsyncSignalSafeLazyInitFakeStack(); return fake_stack_; } diff --git a/libsanitizer/hwasan/Makefile.am b/libsanitizer/hwasan/Makefile.am index 2226f68..5e3a0f1 100644 --- a/libsanitizer/hwasan/Makefile.am +++ b/libsanitizer/hwasan/Makefile.am @@ -13,11 +13,13 @@ ACLOCAL_AMFLAGS = -I $(top_srcdir) -I $(top_srcdir)/config toolexeclib_LTLIBRARIES = libhwasan.la hwasan_files = \ + hwasan_allocation_functions.cpp \ hwasan_allocator.cpp \ hwasan.cpp \ hwasan_dynamic_shadow.cpp \ hwasan_exceptions.cpp \ hwasan_flags.inc \ + hwasan_fuchsia.cpp \ hwasan_globals.cpp \ hwasan_interceptors.cpp \ hwasan_interceptors_vfork.S \ diff --git a/libsanitizer/hwasan/Makefile.in b/libsanitizer/hwasan/Makefile.in index 542af8f..22c5266 100644 --- a/libsanitizer/hwasan/Makefile.in +++ b/libsanitizer/hwasan/Makefile.in @@ -146,8 +146,9 @@ am__DEPENDENCIES_1 = libhwasan_la_DEPENDENCIES = \ $(top_builddir)/sanitizer_common/libsanitizer_common.la \ $(am__append_1) $(am__append_2) $(am__DEPENDENCIES_1) -am__objects_1 = hwasan_allocator.lo hwasan.lo hwasan_dynamic_shadow.lo \ - hwasan_exceptions.lo hwasan_globals.lo hwasan_interceptors.lo \ +am__objects_1 = hwasan_allocation_functions.lo hwasan_allocator.lo \ + hwasan.lo hwasan_dynamic_shadow.lo hwasan_exceptions.lo \ + hwasan_fuchsia.lo hwasan_globals.lo hwasan_interceptors.lo \ hwasan_interceptors_vfork.lo hwasan_linux.lo \ hwasan_memintrinsics.lo hwasan_new_delete.lo \ hwasan_poisoning.lo hwasan_report.lo hwasan_setjmp.lo \ @@ -411,11 +412,13 @@ AM_CXXFLAGS = -Wall -W -Wno-unused-parameter -Wwrite-strings -pedantic \ ACLOCAL_AMFLAGS = -I $(top_srcdir) -I $(top_srcdir)/config toolexeclib_LTLIBRARIES = libhwasan.la hwasan_files = \ + hwasan_allocation_functions.cpp \ hwasan_allocator.cpp \ hwasan.cpp \ hwasan_dynamic_shadow.cpp \ hwasan_exceptions.cpp \ hwasan_flags.inc \ + hwasan_fuchsia.cpp \ hwasan_globals.cpp \ hwasan_interceptors.cpp \ hwasan_interceptors_vfork.S \ @@ -554,9 +557,11 @@ distclean-compile: -rm -f *.tab.c @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_allocation_functions.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_allocator.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_dynamic_shadow.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_exceptions.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_fuchsia.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_globals.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_interceptors.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/hwasan_interceptors_vfork.Plo@am__quote@ diff --git a/libsanitizer/hwasan/hwasan.cpp b/libsanitizer/hwasan/hwasan.cpp index 8d6c252..cbe0dee 100644 --- a/libsanitizer/hwasan/hwasan.cpp +++ b/libsanitizer/hwasan/hwasan.cpp @@ -50,6 +50,11 @@ bool hwasan_init_is_running; int hwasan_report_count = 0; +uptr kLowShadowStart; +uptr kLowShadowEnd; +uptr kHighShadowStart; +uptr kHighShadowEnd; + void Flags::SetDefaults() { #define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue; #include "hwasan_flags.inc" @@ -177,6 +182,65 @@ void UpdateMemoryUsage() { void UpdateMemoryUsage() {} #endif +void HwasanAtExit() { + if (common_flags()->print_module_map) + DumpProcessMap(); + if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) + ReportStats(); + if (hwasan_report_count > 0) { + // ReportAtExitStatistics(); + if (common_flags()->exitcode) + internal__exit(common_flags()->exitcode); + } +} + +void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc, + uptr *registers_frame) { + InternalMmapVector<BufferedStackTrace> stack_buffer(1); + BufferedStackTrace *stack = stack_buffer.data(); + stack->Reset(); + stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal); + + // The second stack frame contains the failure __hwasan_check function, as + // we have a stack frame for the registers saved in __hwasan_tag_mismatch that + // we wish to ignore. This (currently) only occurs on AArch64, as x64 + // implementations use SIGTRAP to implement the failure, and thus do not go + // through the stack saver. + if (registers_frame && stack->trace && stack->size > 0) { + stack->trace++; + stack->size--; + } + + bool fatal = flags()->halt_on_error || !ai.recover; + ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal, + registers_frame); +} + +void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame, + size_t outsize) { + __hwasan::AccessInfo ai; + ai.is_store = access_info & 0x10; + ai.is_load = !ai.is_store; + ai.recover = access_info & 0x20; + ai.addr = addr; + if ((access_info & 0xf) == 0xf) + ai.size = outsize; + else + ai.size = 1 << (access_info & 0xf); + + HandleTagMismatch(ai, (uptr)__builtin_return_address(0), + (uptr)__builtin_frame_address(0), nullptr, registers_frame); + __builtin_unreachable(); +} + +Thread *GetCurrentThread() { + uptr *ThreadLongPtr = GetCurrentThreadLongPtr(); + if (UNLIKELY(*ThreadLongPtr == 0)) + return nullptr; + auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr; + return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next()); +} + } // namespace __hwasan using namespace __hwasan; @@ -216,7 +280,7 @@ static void InitLoadedGlobals() { static void InitInstrumentation() { if (hwasan_instrumentation_inited) return; - InitPrctl(); + InitializeOsSupport(); if (!InitShadow()) { Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n"); @@ -225,7 +289,6 @@ static void InitInstrumentation() { } InitThreads(); - hwasanThreadList().CreateCurrentThread(); hwasan_instrumentation_inited = 1; } @@ -495,7 +558,7 @@ void __hwasan_print_memory_usage() { Printf("%s\n", s.data()); } -static const u8 kFallbackTag = 0xBB; +static const u8 kFallbackTag = 0xBB & kTagMask; u8 __hwasan_generate_tag() { Thread *t = GetCurrentThread(); @@ -516,4 +579,12 @@ void __sanitizer_print_stack_trace() { GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME()); stack.Print(); } + +// Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the +// rest of the mismatch handling code (C++). +void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame, + size_t outsize) { + __hwasan::HwasanTagMismatch(addr, access_info, registers_frame, outsize); +} + } // extern "C" diff --git a/libsanitizer/hwasan/hwasan.h b/libsanitizer/hwasan/hwasan.h index 8515df5..7338b69 100644 --- a/libsanitizer/hwasan/hwasan.h +++ b/libsanitizer/hwasan/hwasan.h @@ -36,7 +36,10 @@ typedef u8 tag_t; -#if defined(__x86_64__) +#if defined(HWASAN_ALIASING_MODE) +# if !defined(__x86_64__) +# error Aliasing mode is only supported on x86_64 +# endif // Tags are done in middle bits using userspace aliasing. constexpr unsigned kAddressTagShift = 39; constexpr unsigned kTagBits = 3; @@ -49,12 +52,16 @@ constexpr unsigned kTagBits = 3; // simpler/faster shadow calculation. constexpr unsigned kTaggableRegionCheckShift = __sanitizer::Max(kAddressTagShift + kTagBits + 1U, 44U); +#elif defined(__x86_64__) +// Tags are done in upper bits using Intel LAM. +constexpr unsigned kAddressTagShift = 57; +constexpr unsigned kTagBits = 6; #else // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in address // translation and can be used to store a tag. constexpr unsigned kAddressTagShift = 56; constexpr unsigned kTagBits = 8; -#endif // defined(__x86_64__) +#endif // defined(HWASAN_ALIASING_MODE) // Mask for extracting tag bits from the lower 8 bits. constexpr uptr kTagMask = (1UL << kTagBits) - 1; @@ -95,7 +102,7 @@ extern bool hwasan_init_is_running; extern int hwasan_report_count; bool InitShadow(); -void InitPrctl(); +void InitializeOsSupport(); void InitThreads(); void InitializeInterceptors(); @@ -129,6 +136,7 @@ void InstallAtExitHandler(); void HwasanTSDInit(); void HwasanTSDThreadInit(); +void HwasanAtExit(); void HwasanOnDeadlySignal(int signo, void *info, void *context); @@ -138,6 +146,26 @@ void AppendToErrorMessageBuffer(const char *buffer); void AndroidTestTlsSlot(); +// This is a compiler-generated struct that can be shared between hwasan +// implementations. +struct AccessInfo { + uptr addr; + uptr size; + bool is_store; + bool is_load; + bool recover; +}; + +// Given access info and frame information, unwind the stack and report the tag +// mismatch. +void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, void *uc, + uptr *registers_frame = nullptr); + +// This dispatches to HandleTagMismatch but sets up the AccessInfo, program +// counter, and frame pointer. +void HwasanTagMismatch(uptr addr, uptr access_info, uptr *registers_frame, + size_t outsize); + } // namespace __hwasan #define HWASAN_MALLOC_HOOK(ptr, size) \ @@ -175,4 +203,12 @@ typedef struct __hw_jmp_buf_struct __hw_jmp_buf[1]; typedef struct __hw_jmp_buf_struct __hw_sigjmp_buf[1]; #endif // HWASAN_WITH_INTERCEPTORS && __aarch64__ +#define ENSURE_HWASAN_INITED() \ + do { \ + CHECK(!hwasan_init_is_running); \ + if (!hwasan_inited) { \ + __hwasan_init(); \ + } \ + } while (0) + #endif // HWASAN_H diff --git a/libsanitizer/hwasan/hwasan_allocation_functions.cpp b/libsanitizer/hwasan/hwasan_allocation_functions.cpp new file mode 100644 index 0000000..6c2a607 --- /dev/null +++ b/libsanitizer/hwasan/hwasan_allocation_functions.cpp @@ -0,0 +1,172 @@ +//===-- hwasan_allocation_functions.cpp -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of HWAddressSanitizer. +// +// Definitions for __sanitizer allocation functions. +// +//===----------------------------------------------------------------------===// + +#include "hwasan.h" +#include "interception/interception.h" +#include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_tls_get_addr.h" + +using namespace __hwasan; + +static uptr allocated_for_dlsym; +static const uptr kDlsymAllocPoolSize = 1024; +static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; + +static bool IsInDlsymAllocPool(const void *ptr) { + uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; + return off < sizeof(alloc_memory_for_dlsym); +} + +static void *AllocateFromLocalPool(uptr size_in_bytes) { + uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; + void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; + allocated_for_dlsym += size_in_words; + CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); + return mem; +} + +int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) { + GET_MALLOC_STACK_TRACE; + CHECK_NE(memptr, 0); + int res = hwasan_posix_memalign(memptr, alignment, size, &stack); + return res; +} + +void *__sanitizer_memalign(uptr alignment, uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_memalign(alignment, size, &stack); +} + +void *__sanitizer_aligned_alloc(uptr alignment, uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_aligned_alloc(alignment, size, &stack); +} + +void *__sanitizer___libc_memalign(uptr alignment, uptr size) { + GET_MALLOC_STACK_TRACE; + void *ptr = hwasan_memalign(alignment, size, &stack); + if (ptr) + DTLS_on_libc_memalign(ptr, size); + return ptr; +} + +void *__sanitizer_valloc(uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_valloc(size, &stack); +} + +void *__sanitizer_pvalloc(uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_pvalloc(size, &stack); +} + +void __sanitizer_free(void *ptr) { + GET_MALLOC_STACK_TRACE; + if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) + return; + hwasan_free(ptr, &stack); +} + +void __sanitizer_cfree(void *ptr) { + GET_MALLOC_STACK_TRACE; + if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) + return; + hwasan_free(ptr, &stack); +} + +uptr __sanitizer_malloc_usable_size(const void *ptr) { + return __sanitizer_get_allocated_size(ptr); +} + +struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() { + __sanitizer_struct_mallinfo sret; + internal_memset(&sret, 0, sizeof(sret)); + return sret; +} + +int __sanitizer_mallopt(int cmd, int value) { return 0; } + +void __sanitizer_malloc_stats(void) { + // FIXME: implement, but don't call REAL(malloc_stats)! +} + +void *__sanitizer_calloc(uptr nmemb, uptr size) { + GET_MALLOC_STACK_TRACE; + if (UNLIKELY(!hwasan_inited)) + // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. + return AllocateFromLocalPool(nmemb * size); + return hwasan_calloc(nmemb, size, &stack); +} + +void *__sanitizer_realloc(void *ptr, uptr size) { + GET_MALLOC_STACK_TRACE; + if (UNLIKELY(IsInDlsymAllocPool(ptr))) { + uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; + uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); + void *new_ptr; + if (UNLIKELY(!hwasan_inited)) { + new_ptr = AllocateFromLocalPool(copy_size); + } else { + copy_size = size; + new_ptr = hwasan_malloc(copy_size, &stack); + } + internal_memcpy(new_ptr, ptr, copy_size); + return new_ptr; + } + return hwasan_realloc(ptr, size, &stack); +} + +void *__sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) { + GET_MALLOC_STACK_TRACE; + return hwasan_reallocarray(ptr, nmemb, size, &stack); +} + +void *__sanitizer_malloc(uptr size) { + GET_MALLOC_STACK_TRACE; + if (UNLIKELY(!hwasan_init_is_running)) + ENSURE_HWASAN_INITED(); + if (UNLIKELY(!hwasan_inited)) + // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. + return AllocateFromLocalPool(size); + return hwasan_malloc(size, &stack); +} + +#if HWASAN_WITH_INTERCEPTORS +# define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \ + ALIAS("__sanitizer_" #FN); \ + extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \ + ARGS) ALIAS("__sanitizer_" #FN) + +INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment, + SIZE_T size); +INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size); +INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size); +INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size); +INTERCEPTOR_ALIAS(void, free, void *ptr); +INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr); +INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size); +INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size); +INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size); +INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size); + +# if !SANITIZER_FREEBSD && !SANITIZER_NETBSD +INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size); +INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size); +INTERCEPTOR_ALIAS(void, cfree, void *ptr); +INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo); +INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value); +INTERCEPTOR_ALIAS(void, malloc_stats, void); +# endif +#endif // #if HWASAN_WITH_INTERCEPTORS diff --git a/libsanitizer/hwasan/hwasan_allocator.cpp b/libsanitizer/hwasan/hwasan_allocator.cpp index a6fc794..ef6d4d6 100644 --- a/libsanitizer/hwasan/hwasan_allocator.cpp +++ b/libsanitizer/hwasan/hwasan_allocator.cpp @@ -80,12 +80,29 @@ void GetAllocatorStats(AllocatorStatCounters s) { allocator.GetStats(s); } +uptr GetAliasRegionStart() { +#if defined(HWASAN_ALIASING_MODE) + constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1); + uptr AliasRegionStart = + __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset; + + CHECK_EQ(AliasRegionStart >> kTaggableRegionCheckShift, + __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift); + CHECK_EQ( + (AliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift, + __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift); + return AliasRegionStart; +#else + return 0; +#endif +} + void HwasanAllocatorInit() { atomic_store_relaxed(&hwasan_allocator_tagging_enabled, !flags()->disable_allocator_tagging); SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null); allocator.Init(common_flags()->allocator_release_to_os_interval_ms, - kAliasRegionStart); + GetAliasRegionStart()); for (uptr i = 0; i < sizeof(tail_magic); i++) tail_magic[i] = GetCurrentThread()->GenerateRandomTag(); } @@ -196,6 +213,7 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { : tagged_ptr; void *aligned_ptr = reinterpret_cast<void *>( RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment)); + tag_t pointer_tag = GetTagFromPointer(reinterpret_cast<uptr>(tagged_ptr)); Metadata *meta = reinterpret_cast<Metadata *>(allocator.GetMetaData(aligned_ptr)); uptr orig_size = meta->get_requested_size(); @@ -229,7 +247,20 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) { flags()->tag_in_free && malloc_bisect(stack, 0) && atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) { // Always store full 8-bit tags on free to maximize UAF detection. - tag_t tag = t ? t->GenerateRandomTag(/*num_bits=*/8) : kFallbackFreeTag; + tag_t tag; + if (t) { + // Make sure we are not using a short granule tag as a poison tag. This + // would make us attempt to read the memory on a UaF. + // The tag can be zero if tagging is disabled on this thread. + do { + tag = t->GenerateRandomTag(/*num_bits=*/8); + } while ( + UNLIKELY((tag < kShadowAlignment || tag == pointer_tag) && tag != 0)); + } else { + static_assert(kFallbackFreeTag >= kShadowAlignment, + "fallback tag must not be a short granule tag."); + tag = kFallbackFreeTag; + } TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size), tag); } diff --git a/libsanitizer/hwasan/hwasan_allocator.h b/libsanitizer/hwasan/hwasan_allocator.h index 03bbcff..35c3d6b 100644 --- a/libsanitizer/hwasan/hwasan_allocator.h +++ b/libsanitizer/hwasan/hwasan_allocator.h @@ -15,6 +15,7 @@ #include "hwasan.h" #include "hwasan_interface_internal.h" +#include "hwasan_mapping.h" #include "hwasan_poisoning.h" #include "sanitizer_common/sanitizer_allocator.h" #include "sanitizer_common/sanitizer_allocator_checks.h" @@ -58,7 +59,7 @@ static const uptr kMaxAllowedMallocSize = 1UL << 40; // 1T struct AP64 { static const uptr kSpaceBeg = ~0ULL; -#if defined(__x86_64__) +#if defined(HWASAN_ALIASING_MODE) static const uptr kSpaceSize = 1ULL << kAddressTagShift; #else static const uptr kSpaceSize = 0x2000000000ULL; @@ -110,11 +111,11 @@ typedef RingBuffer<HeapAllocationRecord> HeapAllocationsRingBuffer; void GetAllocatorStats(AllocatorStatCounters s); inline bool InTaggableRegion(uptr addr) { -#if defined(__x86_64__) +#if defined(HWASAN_ALIASING_MODE) // Aliases are mapped next to shadow so that the upper bits match the shadow // base. return (addr >> kTaggableRegionCheckShift) == - (__hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift); + (GetShadowOffset() >> kTaggableRegionCheckShift); #endif return true; } diff --git a/libsanitizer/hwasan/hwasan_dynamic_shadow.cpp b/libsanitizer/hwasan/hwasan_dynamic_shadow.cpp index f53276e..bde22df 100644 --- a/libsanitizer/hwasan/hwasan_dynamic_shadow.cpp +++ b/libsanitizer/hwasan/hwasan_dynamic_shadow.cpp @@ -119,12 +119,12 @@ namespace __hwasan { void InitShadowGOT() {} uptr FindDynamicShadowStart(uptr shadow_size_bytes) { -#if defined(__x86_64__) +# if defined(HWASAN_ALIASING_MODE) constexpr uptr kAliasSize = 1ULL << kAddressTagShift; constexpr uptr kNumAliases = 1ULL << kTagBits; return MapDynamicShadowAndAliases(shadow_size_bytes, kAliasSize, kNumAliases, RingBufferSize()); -#endif +# endif return MapDynamicShadow(shadow_size_bytes, kShadowScale, kShadowBaseAlignment, kHighMemEnd); } diff --git a/libsanitizer/hwasan/hwasan_fuchsia.cpp b/libsanitizer/hwasan/hwasan_fuchsia.cpp new file mode 100644 index 0000000..e61f6ad --- /dev/null +++ b/libsanitizer/hwasan/hwasan_fuchsia.cpp @@ -0,0 +1,192 @@ +//===-- hwasan_fuchsia.cpp --------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file is a part of HWAddressSanitizer and contains Fuchsia-specific +/// code. +/// +//===----------------------------------------------------------------------===// + +#include "sanitizer_common/sanitizer_fuchsia.h" +#if SANITIZER_FUCHSIA + +#include "hwasan.h" +#include "hwasan_interface_internal.h" +#include "hwasan_report.h" +#include "hwasan_thread.h" +#include "hwasan_thread_list.h" + +// This TLS variable contains the location of the stack ring buffer and can be +// used to always find the hwasan thread object associated with the current +// running thread. +[[gnu::tls_model("initial-exec")]] +SANITIZER_INTERFACE_ATTRIBUTE +THREADLOCAL uptr __hwasan_tls; + +namespace __hwasan { + +bool InitShadow() { + __sanitizer::InitShadowBounds(); + CHECK_NE(__sanitizer::ShadowBounds.shadow_limit, 0); + + return true; +} + +bool MemIsApp(uptr p) { + CHECK(GetTagFromPointer(p) == 0); + return __sanitizer::ShadowBounds.shadow_limit <= p && + p <= (__sanitizer::ShadowBounds.memory_limit - 1); +} + +// These are known parameters passed to the hwasan runtime on thread creation. +struct Thread::InitState { + uptr stack_bottom, stack_top; +}; + +static void FinishThreadInitialization(Thread *thread); + +void InitThreads() { + // This is the minimal alignment needed for the storage where hwasan threads + // and their stack ring buffers are placed. This alignment is necessary so the + // stack ring buffer can perform a simple calculation to get the next element + // in the RB. The instructions for this calculation are emitted by the + // compiler. (Full explanation in hwasan_thread_list.h.) + uptr alloc_size = UINT64_C(1) << kShadowBaseAlignment; + uptr thread_start = reinterpret_cast<uptr>( + MmapAlignedOrDieOnFatalError(alloc_size, alloc_size, __func__)); + + InitThreadList(thread_start, alloc_size); + + // Create the hwasan thread object for the current (main) thread. Stack info + // for this thread is known from information passed via + // __sanitizer_startup_hook. + const Thread::InitState state = { + .stack_bottom = __sanitizer::MainThreadStackBase, + .stack_top = + __sanitizer::MainThreadStackBase + __sanitizer::MainThreadStackSize, + }; + FinishThreadInitialization(hwasanThreadList().CreateCurrentThread(&state)); +} + +uptr *GetCurrentThreadLongPtr() { return &__hwasan_tls; } + +// This is called from the parent thread before the new thread is created. Here +// we can propagate known info like the stack bounds to Thread::Init before +// jumping into the thread. We cannot initialize the stack ring buffer yet since +// we have not entered the new thread. +static void *BeforeThreadCreateHook(uptr user_id, bool detached, + const char *name, uptr stack_bottom, + uptr stack_size) { + const Thread::InitState state = { + .stack_bottom = stack_bottom, + .stack_top = stack_bottom + stack_size, + }; + return hwasanThreadList().CreateCurrentThread(&state); +} + +// This sets the stack top and bottom according to the InitState passed to +// CreateCurrentThread above. +void Thread::InitStackAndTls(const InitState *state) { + CHECK_NE(state->stack_bottom, 0); + CHECK_NE(state->stack_top, 0); + stack_bottom_ = state->stack_bottom; + stack_top_ = state->stack_top; + tls_end_ = tls_begin_ = 0; +} + +// This is called after creating a new thread with the pointer returned by +// BeforeThreadCreateHook. We are still in the creating thread and should check +// if it was actually created correctly. +static void ThreadCreateHook(void *hook, bool aborted) { + Thread *thread = static_cast<Thread *>(hook); + if (!aborted) { + // The thread was created successfully. + // ThreadStartHook can already be running in the new thread. + } else { + // The thread wasn't created after all. + // Clean up everything we set up in BeforeThreadCreateHook. + atomic_signal_fence(memory_order_seq_cst); + hwasanThreadList().ReleaseThread(thread); + } +} + +// This is called in the newly-created thread before it runs anything else, +// with the pointer returned by BeforeThreadCreateHook (above). Here we can +// setup the stack ring buffer. +static void ThreadStartHook(void *hook, thrd_t self) { + Thread *thread = static_cast<Thread *>(hook); + FinishThreadInitialization(thread); + thread->InitRandomState(); +} + +// This is the function that sets up the stack ring buffer and enables us to use +// GetCurrentThread. This function should only be called while IN the thread +// that we want to create the hwasan thread object for so __hwasan_tls can be +// properly referenced. +static void FinishThreadInitialization(Thread *thread) { + CHECK_NE(thread, nullptr); + + // The ring buffer is located immediately before the thread object. + uptr stack_buffer_size = hwasanThreadList().GetRingBufferSize(); + uptr stack_buffer_start = reinterpret_cast<uptr>(thread) - stack_buffer_size; + thread->InitStackRingBuffer(stack_buffer_start, stack_buffer_size); +} + +static void ThreadExitHook(void *hook, thrd_t self) { + Thread *thread = static_cast<Thread *>(hook); + atomic_signal_fence(memory_order_seq_cst); + hwasanThreadList().ReleaseThread(thread); +} + +// Not implemented because Fuchsia does not use signal handlers. +void HwasanOnDeadlySignal(int signo, void *info, void *context) {} + +// Not implemented because Fuchsia does not use interceptors. +void InitializeInterceptors() {} + +// Not implemented because this is only relevant for Android. +void AndroidTestTlsSlot() {} + +// TSD was normally used on linux as a means of calling the hwasan thread exit +// handler passed to pthread_key_create. This is not needed on Fuchsia because +// we will be using __sanitizer_thread_exit_hook. +void HwasanTSDInit() {} +void HwasanTSDThreadInit() {} + +// On linux, this just would call `atexit(HwasanAtExit)`. The functions in +// HwasanAtExit are unimplemented for Fuchsia and effectively no-ops, so this +// function is unneeded. +void InstallAtExitHandler() {} + +} // namespace __hwasan + +extern "C" { + +void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached, + const char *name, void *stack_base, + size_t stack_size) { + return __hwasan::BeforeThreadCreateHook( + reinterpret_cast<uptr>(thread), detached, name, + reinterpret_cast<uptr>(stack_base), stack_size); +} + +void __sanitizer_thread_create_hook(void *hook, thrd_t thread, int error) { + __hwasan::ThreadCreateHook(hook, error != thrd_success); +} + +void __sanitizer_thread_start_hook(void *hook, thrd_t self) { + __hwasan::ThreadStartHook(hook, reinterpret_cast<uptr>(self)); +} + +void __sanitizer_thread_exit_hook(void *hook, thrd_t self) { + __hwasan::ThreadExitHook(hook, self); +} + +} // extern "C" + +#endif // SANITIZER_FUCHSIA diff --git a/libsanitizer/hwasan/hwasan_interceptors.cpp b/libsanitizer/hwasan/hwasan_interceptors.cpp index ad67e27..68f8ade 100644 --- a/libsanitizer/hwasan/hwasan_interceptors.cpp +++ b/libsanitizer/hwasan/hwasan_interceptors.cpp @@ -16,192 +16,14 @@ #include "interception/interception.h" #include "hwasan.h" -#include "hwasan_allocator.h" -#include "hwasan_mapping.h" #include "hwasan_thread.h" -#include "hwasan_poisoning.h" -#include "hwasan_report.h" -#include "sanitizer_common/sanitizer_platform_limits_posix.h" -#include "sanitizer_common/sanitizer_allocator.h" -#include "sanitizer_common/sanitizer_allocator_interface.h" -#include "sanitizer_common/sanitizer_allocator_internal.h" -#include "sanitizer_common/sanitizer_atomic.h" -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_stackdepot.h" -#include "sanitizer_common/sanitizer_libc.h" -#include "sanitizer_common/sanitizer_linux.h" -#include "sanitizer_common/sanitizer_tls_get_addr.h" -#include <stdarg.h> -// ACHTUNG! No other system header includes in this file. -// Ideally, we should get rid of stdarg.h as well. +#if !SANITIZER_FUCHSIA using namespace __hwasan; -using __sanitizer::memory_order; -using __sanitizer::atomic_load; -using __sanitizer::atomic_store; -using __sanitizer::atomic_uintptr_t; - -static uptr allocated_for_dlsym; -static const uptr kDlsymAllocPoolSize = 1024; -static uptr alloc_memory_for_dlsym[kDlsymAllocPoolSize]; - -static bool IsInDlsymAllocPool(const void *ptr) { - uptr off = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - return off < sizeof(alloc_memory_for_dlsym); -} - -static void *AllocateFromLocalPool(uptr size_in_bytes) { - uptr size_in_words = RoundUpTo(size_in_bytes, kWordSize) / kWordSize; - void *mem = (void *)&alloc_memory_for_dlsym[allocated_for_dlsym]; - allocated_for_dlsym += size_in_words; - CHECK_LT(allocated_for_dlsym, kDlsymAllocPoolSize); - return mem; -} - -#define ENSURE_HWASAN_INITED() do { \ - CHECK(!hwasan_init_is_running); \ - if (!hwasan_inited) { \ - __hwasan_init(); \ - } \ -} while (0) - - -int __sanitizer_posix_memalign(void **memptr, uptr alignment, uptr size) { - GET_MALLOC_STACK_TRACE; - CHECK_NE(memptr, 0); - int res = hwasan_posix_memalign(memptr, alignment, size, &stack); - return res; -} - -void * __sanitizer_memalign(uptr alignment, uptr size) { - GET_MALLOC_STACK_TRACE; - return hwasan_memalign(alignment, size, &stack); -} - -void * __sanitizer_aligned_alloc(uptr alignment, uptr size) { - GET_MALLOC_STACK_TRACE; - return hwasan_aligned_alloc(alignment, size, &stack); -} - -void * __sanitizer___libc_memalign(uptr alignment, uptr size) { - GET_MALLOC_STACK_TRACE; - void *ptr = hwasan_memalign(alignment, size, &stack); - if (ptr) - DTLS_on_libc_memalign(ptr, size); - return ptr; -} - -void * __sanitizer_valloc(uptr size) { - GET_MALLOC_STACK_TRACE; - return hwasan_valloc(size, &stack); -} - -void * __sanitizer_pvalloc(uptr size) { - GET_MALLOC_STACK_TRACE; - return hwasan_pvalloc(size, &stack); -} - -void __sanitizer_free(void *ptr) { - GET_MALLOC_STACK_TRACE; - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; - hwasan_free(ptr, &stack); -} - -void __sanitizer_cfree(void *ptr) { - GET_MALLOC_STACK_TRACE; - if (!ptr || UNLIKELY(IsInDlsymAllocPool(ptr))) return; - hwasan_free(ptr, &stack); -} - -uptr __sanitizer_malloc_usable_size(const void *ptr) { - return __sanitizer_get_allocated_size(ptr); -} - -struct __sanitizer_struct_mallinfo __sanitizer_mallinfo() { - __sanitizer_struct_mallinfo sret; - internal_memset(&sret, 0, sizeof(sret)); - return sret; -} - -int __sanitizer_mallopt(int cmd, int value) { - return 0; -} - -void __sanitizer_malloc_stats(void) { - // FIXME: implement, but don't call REAL(malloc_stats)! -} - -void * __sanitizer_calloc(uptr nmemb, uptr size) { - GET_MALLOC_STACK_TRACE; - if (UNLIKELY(!hwasan_inited)) - // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym. - return AllocateFromLocalPool(nmemb * size); - return hwasan_calloc(nmemb, size, &stack); -} - -void * __sanitizer_realloc(void *ptr, uptr size) { - GET_MALLOC_STACK_TRACE; - if (UNLIKELY(IsInDlsymAllocPool(ptr))) { - uptr offset = (uptr)ptr - (uptr)alloc_memory_for_dlsym; - uptr copy_size = Min(size, kDlsymAllocPoolSize - offset); - void *new_ptr; - if (UNLIKELY(!hwasan_inited)) { - new_ptr = AllocateFromLocalPool(copy_size); - } else { - copy_size = size; - new_ptr = hwasan_malloc(copy_size, &stack); - } - internal_memcpy(new_ptr, ptr, copy_size); - return new_ptr; - } - return hwasan_realloc(ptr, size, &stack); -} - -void * __sanitizer_reallocarray(void *ptr, uptr nmemb, uptr size) { - GET_MALLOC_STACK_TRACE; - return hwasan_reallocarray(ptr, nmemb, size, &stack); -} - -void * __sanitizer_malloc(uptr size) { - GET_MALLOC_STACK_TRACE; - if (UNLIKELY(!hwasan_init_is_running)) - ENSURE_HWASAN_INITED(); - if (UNLIKELY(!hwasan_inited)) - // Hack: dlsym calls malloc before REAL(malloc) is retrieved from dlsym. - return AllocateFromLocalPool(size); - return hwasan_malloc(size, &stack); -} - #if HWASAN_WITH_INTERCEPTORS -#define INTERCEPTOR_ALIAS(RET, FN, ARGS...) \ - extern "C" SANITIZER_INTERFACE_ATTRIBUTE RET WRAP(FN)(ARGS) \ - ALIAS("__sanitizer_" #FN); \ - extern "C" SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE RET FN( \ - ARGS) ALIAS("__sanitizer_" #FN) - -INTERCEPTOR_ALIAS(int, posix_memalign, void **memptr, SIZE_T alignment, - SIZE_T size); -INTERCEPTOR_ALIAS(void *, aligned_alloc, SIZE_T alignment, SIZE_T size); -INTERCEPTOR_ALIAS(void *, __libc_memalign, SIZE_T alignment, SIZE_T size); -INTERCEPTOR_ALIAS(void *, valloc, SIZE_T size); -INTERCEPTOR_ALIAS(void, free, void *ptr); -INTERCEPTOR_ALIAS(uptr, malloc_usable_size, const void *ptr); -INTERCEPTOR_ALIAS(void *, calloc, SIZE_T nmemb, SIZE_T size); -INTERCEPTOR_ALIAS(void *, realloc, void *ptr, SIZE_T size); -INTERCEPTOR_ALIAS(void *, reallocarray, void *ptr, SIZE_T nmemb, SIZE_T size); -INTERCEPTOR_ALIAS(void *, malloc, SIZE_T size); - -#if !SANITIZER_FREEBSD && !SANITIZER_NETBSD -INTERCEPTOR_ALIAS(void *, memalign, SIZE_T alignment, SIZE_T size); -INTERCEPTOR_ALIAS(void *, pvalloc, SIZE_T size); -INTERCEPTOR_ALIAS(void, cfree, void *ptr); -INTERCEPTOR_ALIAS(__sanitizer_struct_mallinfo, mallinfo); -INTERCEPTOR_ALIAS(int, mallopt, int cmd, int value); -INTERCEPTOR_ALIAS(void, malloc_stats, void); -#endif struct ThreadStartArg { thread_callback_t callback; @@ -346,3 +168,5 @@ void InitializeInterceptors() { inited = 1; } } // namespace __hwasan + +#endif // #if !SANITIZER_FUCHSIA diff --git a/libsanitizer/hwasan/hwasan_linux.cpp b/libsanitizer/hwasan/hwasan_linux.cpp index 8ce0ff7..e227235 100644 --- a/libsanitizer/hwasan/hwasan_linux.cpp +++ b/libsanitizer/hwasan/hwasan_linux.cpp @@ -69,15 +69,9 @@ static void ProtectGap(uptr addr, uptr size) { uptr kLowMemStart; uptr kLowMemEnd; -uptr kLowShadowEnd; -uptr kLowShadowStart; -uptr kHighShadowStart; -uptr kHighShadowEnd; uptr kHighMemStart; uptr kHighMemEnd; -uptr kAliasRegionStart; // Always 0 on non-x86. - static void PrintRange(uptr start, uptr end, const char *name) { Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name); } @@ -116,7 +110,7 @@ static void InitializeShadowBaseAddress(uptr shadow_size_bytes) { FindDynamicShadowStart(shadow_size_bytes); } -void InitPrctl() { +void InitializeOsSupport() { #define PR_SET_TAGGED_ADDR_CTRL 55 #define PR_GET_TAGGED_ADDR_CTRL 56 #define PR_TAGGED_ADDR_ENABLE (1UL << 0) @@ -125,33 +119,50 @@ void InitPrctl() { if (internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0), &local_errno) && local_errno == EINVAL) { -#if SANITIZER_ANDROID || defined(__x86_64__) +# if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE) // Some older Android kernels have the tagged pointer ABI on // unconditionally, and hence don't have the tagged-addr prctl while still // allow the ABI. // If targeting Android and the prctl is not around we assume this is the // case. return; -#else +# else if (flags()->fail_without_syscall_abi) { Printf( "FATAL: " "HWAddressSanitizer requires a kernel with tagged address ABI.\n"); Die(); } -#endif +# endif } // Turn on the tagged address ABI. if ((internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, 0, 0, 0)) || - !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0)) && - flags()->fail_without_syscall_abi) { - Printf( - "FATAL: HWAddressSanitizer failed to enable tagged address syscall " - "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` " - "configuration.\n"); - Die(); + !internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0))) { +# if defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE) + // Try the new prctl API for Intel LAM. The API is based on a currently + // unsubmitted patch to the Linux kernel (as of May 2021) and is thus + // subject to change. Patch is here: + // https://lore.kernel.org/linux-mm/20210205151631.43511-12-kirill.shutemov@linux.intel.com/ + int tag_bits = kTagBits; + int tag_shift = kAddressTagShift; + if (!internal_iserror( + internal_prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE, + reinterpret_cast<unsigned long>(&tag_bits), + reinterpret_cast<unsigned long>(&tag_shift), 0))) { + CHECK_EQ(tag_bits, kTagBits); + CHECK_EQ(tag_shift, kAddressTagShift); + return; + } +# endif // defined(__x86_64__) && !defined(HWASAN_ALIASING_MODE) + if (flags()->fail_without_syscall_abi) { + Printf( + "FATAL: HWAddressSanitizer failed to enable tagged address syscall " + "ABI.\nSuggest check `sysctl abi.tagged_addr_disabled` " + "configuration.\n"); + Die(); + } } #undef PR_SET_TAGGED_ADDR_CTRL #undef PR_GET_TAGGED_ADDR_CTRL @@ -181,18 +192,6 @@ bool InitShadow() { // High memory starts where allocated shadow allows. kHighMemStart = ShadowToMem(kHighShadowStart); -#if defined(__x86_64__) - constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1); - kAliasRegionStart = - __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset; - - CHECK_EQ(kAliasRegionStart >> kTaggableRegionCheckShift, - __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift); - CHECK_EQ( - (kAliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift, - __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift); -#endif - // Check the sanity of the defined memory ranges (there might be gaps). CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0); CHECK_GT(kHighMemStart, kHighShadowEnd); @@ -233,25 +232,16 @@ void InitThreads() { ProtectGap(thread_space_end, __hwasan_shadow_memory_dynamic_address - thread_space_end); InitThreadList(thread_space_start, thread_space_end - thread_space_start); + hwasanThreadList().CreateCurrentThread(); } bool MemIsApp(uptr p) { -#if !defined(__x86_64__) // Memory outside the alias range has non-zero tags. +// Memory outside the alias range has non-zero tags. +# if !defined(HWASAN_ALIASING_MODE) CHECK(GetTagFromPointer(p) == 0); -#endif - return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd); -} +# endif -static void HwasanAtExit(void) { - if (common_flags()->print_module_map) - DumpProcessMap(); - if (flags()->print_stats && (flags()->atexit || hwasan_report_count > 0)) - ReportStats(); - if (hwasan_report_count > 0) { - // ReportAtExitStatistics(); - if (common_flags()->exitcode) - internal__exit(common_flags()->exitcode); - } + return p >= kHighMemStart || (p >= kLowMemStart && p <= kLowMemEnd); } void InstallAtExitHandler() { @@ -330,22 +320,6 @@ void AndroidTestTlsSlot() { void AndroidTestTlsSlot() {} #endif -Thread *GetCurrentThread() { - uptr *ThreadLongPtr = GetCurrentThreadLongPtr(); - if (UNLIKELY(*ThreadLongPtr == 0)) - return nullptr; - auto *R = (StackAllocationsRingBuffer *)ThreadLongPtr; - return hwasanThreadList().GetThreadByBufferAddress((uptr)R->Next()); -} - -struct AccessInfo { - uptr addr; - uptr size; - bool is_store; - bool is_load; - bool recover; -}; - static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { // Access type is passed in a platform dependent way (see below) and encoded // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is @@ -396,28 +370,6 @@ static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) { return AccessInfo{addr, size, is_store, !is_store, recover}; } -static void HandleTagMismatch(AccessInfo ai, uptr pc, uptr frame, - ucontext_t *uc, uptr *registers_frame = nullptr) { - InternalMmapVector<BufferedStackTrace> stack_buffer(1); - BufferedStackTrace *stack = stack_buffer.data(); - stack->Reset(); - stack->Unwind(pc, frame, uc, common_flags()->fast_unwind_on_fatal); - - // The second stack frame contains the failure __hwasan_check function, as - // we have a stack frame for the registers saved in __hwasan_tag_mismatch that - // we wish to ignore. This (currently) only occurs on AArch64, as x64 - // implementations use SIGTRAP to implement the failure, and thus do not go - // through the stack saver. - if (registers_frame && stack->trace && stack->size > 0) { - stack->trace++; - stack->size--; - } - - bool fatal = flags()->halt_on_error || !ai.recover; - ReportTagMismatch(stack, ai.addr, ai.size, ai.is_store, fatal, - registers_frame); -} - static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) { AccessInfo ai = GetAccessInfo(info, uc); if (!ai.is_store && !ai.is_load) @@ -450,27 +402,39 @@ void HwasanOnDeadlySignal(int signo, void *info, void *context) { HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr); } +void Thread::InitStackAndTls(const InitState *) { + uptr tls_size; + uptr stack_size; + GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_, + &tls_size); + stack_top_ = stack_bottom_ + stack_size; + tls_end_ = tls_begin_ + tls_size; +} -} // namespace __hwasan - -// Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the -// rest of the mismatch handling code (C++). -void __hwasan_tag_mismatch4(uptr addr, uptr access_info, uptr *registers_frame, - size_t outsize) { - __hwasan::AccessInfo ai; - ai.is_store = access_info & 0x10; - ai.is_load = !ai.is_store; - ai.recover = access_info & 0x20; - ai.addr = addr; - if ((access_info & 0xf) == 0xf) - ai.size = outsize; - else - ai.size = 1 << (access_info & 0xf); - - __hwasan::HandleTagMismatch(ai, (uptr)__builtin_return_address(0), - (uptr)__builtin_frame_address(0), nullptr, - registers_frame); - __builtin_unreachable(); +uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { + CHECK(IsAligned(p, kShadowAlignment)); + CHECK(IsAligned(size, kShadowAlignment)); + uptr shadow_start = MemToShadow(p); + uptr shadow_size = MemToShadowSize(size); + + uptr page_size = GetPageSizeCached(); + uptr page_start = RoundUpTo(shadow_start, page_size); + uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size); + uptr threshold = common_flags()->clear_shadow_mmap_threshold; + if (SANITIZER_LINUX && + UNLIKELY(page_end >= page_start + threshold && tag == 0)) { + internal_memset((void *)shadow_start, tag, page_start - shadow_start); + internal_memset((void *)page_end, tag, + shadow_start + shadow_size - page_end); + // For an anonymous private mapping MADV_DONTNEED will return a zero page on + // Linux. + ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end); + } else { + internal_memset((void *)shadow_start, tag, shadow_size); + } + return AddTagToPointer(p, tag); } +} // namespace __hwasan + #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD diff --git a/libsanitizer/hwasan/hwasan_mapping.h b/libsanitizer/hwasan/hwasan_mapping.h index 8243d1e..79a1436 100644 --- a/libsanitizer/hwasan/hwasan_mapping.h +++ b/libsanitizer/hwasan/hwasan_mapping.h @@ -48,14 +48,14 @@ extern uptr kHighShadowEnd; extern uptr kHighMemStart; extern uptr kHighMemEnd; -extern uptr kAliasRegionStart; - +inline uptr GetShadowOffset() { + return SANITIZER_FUCHSIA ? 0 : __hwasan_shadow_memory_dynamic_address; +} inline uptr MemToShadow(uptr untagged_addr) { - return (untagged_addr >> kShadowScale) + - __hwasan_shadow_memory_dynamic_address; + return (untagged_addr >> kShadowScale) + GetShadowOffset(); } inline uptr ShadowToMem(uptr shadow_addr) { - return (shadow_addr - __hwasan_shadow_memory_dynamic_address) << kShadowScale; + return (shadow_addr - GetShadowOffset()) << kShadowScale; } inline uptr MemToShadowSize(uptr size) { return size >> kShadowScale; @@ -63,6 +63,13 @@ inline uptr MemToShadowSize(uptr size) { bool MemIsApp(uptr p); +inline bool MemIsShadow(uptr p) { + return (kLowShadowStart <= p && p <= kLowShadowEnd) || + (kHighShadowStart <= p && p <= kHighShadowEnd); +} + +uptr GetAliasRegionStart(); + } // namespace __hwasan #endif // HWASAN_MAPPING_H diff --git a/libsanitizer/hwasan/hwasan_new_delete.cpp b/libsanitizer/hwasan/hwasan_new_delete.cpp index 69cddda..4e057a6 100644 --- a/libsanitizer/hwasan/hwasan_new_delete.cpp +++ b/libsanitizer/hwasan/hwasan_new_delete.cpp @@ -56,7 +56,6 @@ using namespace __hwasan; // Fake std::nothrow_t to avoid including <new>. namespace std { struct nothrow_t {}; - enum class align_val_t : size_t {}; } // namespace std @@ -73,6 +72,32 @@ INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[](size_t size, std::nothrow_t const&) { OPERATOR_NEW_BODY(true /*nothrow*/); } + +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete(void *ptr) + NOEXCEPT { + OPERATOR_DELETE_BODY; +} +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( + void *ptr) NOEXCEPT { + OPERATOR_DELETE_BODY; +} +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( + void *ptr, std::nothrow_t const &) { + OPERATOR_DELETE_BODY; +} +INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( + void *ptr, std::nothrow_t const &) { + OPERATOR_DELETE_BODY; +} + +#endif // OPERATOR_NEW_BODY + +#ifdef OPERATOR_NEW_ALIGN_BODY + +namespace std { +enum class align_val_t : size_t {}; +} // namespace std + INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new( size_t size, std::align_val_t align) { OPERATOR_NEW_ALIGN_BODY(false /*nothrow*/); @@ -90,16 +115,6 @@ INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void *operator new[]( OPERATOR_NEW_ALIGN_BODY(true /*nothrow*/); } -INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -void operator delete(void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } -INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -void operator delete[](void *ptr) NOEXCEPT { OPERATOR_DELETE_BODY; } -INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -void operator delete(void *ptr, std::nothrow_t const&) { OPERATOR_DELETE_BODY; } -INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -void operator delete[](void *ptr, std::nothrow_t const&) { - OPERATOR_DELETE_BODY; -} INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete( void *ptr, std::align_val_t align) NOEXCEPT { OPERATOR_DELETE_BODY; @@ -117,4 +132,4 @@ INTERCEPTOR_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE void operator delete[]( OPERATOR_DELETE_BODY; } -#endif // OPERATOR_NEW_BODY +#endif // OPERATOR_NEW_ALIGN_BODY diff --git a/libsanitizer/hwasan/hwasan_poisoning.cpp b/libsanitizer/hwasan/hwasan_poisoning.cpp index 2a08164..5aafdb1 100644 --- a/libsanitizer/hwasan/hwasan_poisoning.cpp +++ b/libsanitizer/hwasan/hwasan_poisoning.cpp @@ -19,30 +19,6 @@ namespace __hwasan { -uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) { - CHECK(IsAligned(p, kShadowAlignment)); - CHECK(IsAligned(size, kShadowAlignment)); - uptr shadow_start = MemToShadow(p); - uptr shadow_size = MemToShadowSize(size); - - uptr page_size = GetPageSizeCached(); - uptr page_start = RoundUpTo(shadow_start, page_size); - uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size); - uptr threshold = common_flags()->clear_shadow_mmap_threshold; - if (SANITIZER_LINUX && - UNLIKELY(page_end >= page_start + threshold && tag == 0)) { - internal_memset((void *)shadow_start, tag, page_start - shadow_start); - internal_memset((void *)page_end, tag, - shadow_start + shadow_size - page_end); - // For an anonymous private mapping MADV_DONTNEED will return a zero page on - // Linux. - ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end); - } else { - internal_memset((void *)shadow_start, tag, shadow_size); - } - return AddTagToPointer(p, tag); -} - uptr TagMemory(uptr p, uptr size, tag_t tag) { uptr start = RoundDownTo(p, kShadowAlignment); uptr end = RoundUpTo(p + size, kShadowAlignment); diff --git a/libsanitizer/hwasan/hwasan_report.cpp b/libsanitizer/hwasan/hwasan_report.cpp index c021779..44047c9 100644 --- a/libsanitizer/hwasan/hwasan_report.cpp +++ b/libsanitizer/hwasan/hwasan_report.cpp @@ -236,12 +236,12 @@ static void PrintStackAllocations(StackAllocationsRingBuffer *sa, frame_desc.append(" record_addr:0x%zx record:0x%zx", reinterpret_cast<uptr>(record_addr), record); if (SymbolizedStack *frame = Symbolizer::GetOrInit()->SymbolizePC(pc)) { - RenderFrame(&frame_desc, " %F %L\n", 0, frame->info.address, &frame->info, + RenderFrame(&frame_desc, " %F %L", 0, frame->info.address, &frame->info, common_flags()->symbolize_vs_style, common_flags()->strip_path_prefix); frame->ClearAll(); } - Printf("%s", frame_desc.data()); + Printf("%s\n", frame_desc.data()); frame_desc.clear(); } } @@ -296,6 +296,75 @@ static uptr GetGlobalSizeFromDescriptor(uptr ptr) { return 0; } +static void ShowHeapOrGlobalCandidate(uptr untagged_addr, tag_t *candidate, + tag_t *left, tag_t *right) { + Decorator d; + uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate)); + HwasanChunkView chunk = FindHeapChunkByAddress(mem); + if (chunk.IsAllocated()) { + uptr offset; + const char *whence; + if (untagged_addr < chunk.End() && untagged_addr >= chunk.Beg()) { + offset = untagged_addr - chunk.Beg(); + whence = "inside"; + } else if (candidate == left) { + offset = untagged_addr - chunk.End(); + whence = "to the right of"; + } else { + offset = chunk.Beg() - untagged_addr; + whence = "to the left of"; + } + Printf("%s", d.Error()); + Printf("\nCause: heap-buffer-overflow\n"); + Printf("%s", d.Default()); + Printf("%s", d.Location()); + Printf("%p is located %zd bytes %s %zd-byte region [%p,%p)\n", + untagged_addr, offset, whence, chunk.UsedSize(), chunk.Beg(), + chunk.End()); + Printf("%s", d.Allocation()); + Printf("allocated here:\n"); + Printf("%s", d.Default()); + GetStackTraceFromId(chunk.GetAllocStackId()).Print(); + return; + } + // Check whether the address points into a loaded library. If so, this is + // most likely a global variable. + const char *module_name; + uptr module_address; + Symbolizer *sym = Symbolizer::GetOrInit(); + if (sym->GetModuleNameAndOffsetForPC(mem, &module_name, &module_address)) { + Printf("%s", d.Error()); + Printf("\nCause: global-overflow\n"); + Printf("%s", d.Default()); + DataInfo info; + Printf("%s", d.Location()); + if (sym->SymbolizeData(mem, &info) && info.start) { + Printf( + "%p is located %zd bytes to the %s of %zd-byte global variable " + "%s [%p,%p) in %s\n", + untagged_addr, + candidate == left ? untagged_addr - (info.start + info.size) + : info.start - untagged_addr, + candidate == left ? "right" : "left", info.size, info.name, + info.start, info.start + info.size, module_name); + } else { + uptr size = GetGlobalSizeFromDescriptor(mem); + if (size == 0) + // We couldn't find the size of the global from the descriptors. + Printf("%p is located to the %s of a global variable in (%s+0x%x)\n", + untagged_addr, candidate == left ? "right" : "left", module_name, + module_address); + else + Printf( + "%p is located to the %s of a %zd-byte global variable in " + "(%s+0x%x)\n", + untagged_addr, candidate == left ? "right" : "left", size, + module_name, module_address); + } + Printf("%s", d.Default()); + } +} + void PrintAddressDescription( uptr tagged_addr, uptr access_size, StackAllocationsRingBuffer *current_stack_allocations) { @@ -317,78 +386,59 @@ void PrintAddressDescription( d.Default()); } + tag_t addr_tag = GetTagFromPointer(tagged_addr); + + bool on_stack = false; + // Check stack first. If the address is on the stack of a live thread, we + // know it cannot be a heap / global overflow. + hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { + if (t->AddrIsInStack(untagged_addr)) { + on_stack = true; + // TODO(fmayer): figure out how to distinguish use-after-return and + // stack-buffer-overflow. + Printf("%s", d.Error()); + Printf("\nCause: stack tag-mismatch\n"); + Printf("%s", d.Location()); + Printf("Address %p is located in stack of thread T%zd\n", untagged_addr, + t->unique_id()); + Printf("%s", d.Default()); + t->Announce(); + + auto *sa = (t == GetCurrentThread() && current_stack_allocations) + ? current_stack_allocations + : t->stack_allocations(); + PrintStackAllocations(sa, addr_tag, untagged_addr); + num_descriptions_printed++; + } + }); + // Check if this looks like a heap buffer overflow by scanning // the shadow left and right and looking for the first adjacent // object with a different memory tag. If that tag matches addr_tag, // check the allocator if it has a live chunk there. - tag_t addr_tag = GetTagFromPointer(tagged_addr); tag_t *tag_ptr = reinterpret_cast<tag_t*>(MemToShadow(untagged_addr)); tag_t *candidate = nullptr, *left = tag_ptr, *right = tag_ptr; - for (int i = 0; i < 1000; i++) { - if (TagsEqual(addr_tag, left)) { + uptr candidate_distance = 0; + for (; candidate_distance < 1000; candidate_distance++) { + if (MemIsShadow(reinterpret_cast<uptr>(left)) && + TagsEqual(addr_tag, left)) { candidate = left; break; } --left; - if (TagsEqual(addr_tag, right)) { + if (MemIsShadow(reinterpret_cast<uptr>(right)) && + TagsEqual(addr_tag, right)) { candidate = right; break; } ++right; } - if (candidate) { - uptr mem = ShadowToMem(reinterpret_cast<uptr>(candidate)); - HwasanChunkView chunk = FindHeapChunkByAddress(mem); - if (chunk.IsAllocated()) { - Printf("%s", d.Location()); - Printf("%p is located %zd bytes to the %s of %zd-byte region [%p,%p)\n", - untagged_addr, - candidate == left ? untagged_addr - chunk.End() - : chunk.Beg() - untagged_addr, - candidate == left ? "right" : "left", chunk.UsedSize(), - chunk.Beg(), chunk.End()); - Printf("%s", d.Allocation()); - Printf("allocated here:\n"); - Printf("%s", d.Default()); - GetStackTraceFromId(chunk.GetAllocStackId()).Print(); - num_descriptions_printed++; - } else { - // Check whether the address points into a loaded library. If so, this is - // most likely a global variable. - const char *module_name; - uptr module_address; - Symbolizer *sym = Symbolizer::GetOrInit(); - if (sym->GetModuleNameAndOffsetForPC(mem, &module_name, - &module_address)) { - DataInfo info; - if (sym->SymbolizeData(mem, &info) && info.start) { - Printf( - "%p is located %zd bytes to the %s of %zd-byte global variable " - "%s [%p,%p) in %s\n", - untagged_addr, - candidate == left ? untagged_addr - (info.start + info.size) - : info.start - untagged_addr, - candidate == left ? "right" : "left", info.size, info.name, - info.start, info.start + info.size, module_name); - } else { - uptr size = GetGlobalSizeFromDescriptor(mem); - if (size == 0) - // We couldn't find the size of the global from the descriptors. - Printf( - "%p is located to the %s of a global variable in (%s+0x%x)\n", - untagged_addr, candidate == left ? "right" : "left", - module_name, module_address); - else - Printf( - "%p is located to the %s of a %zd-byte global variable in " - "(%s+0x%x)\n", - untagged_addr, candidate == left ? "right" : "left", size, - module_name, module_address); - } - num_descriptions_printed++; - } - } + constexpr auto kCloseCandidateDistance = 1; + + if (!on_stack && candidate && candidate_distance <= kCloseCandidateDistance) { + ShowHeapOrGlobalCandidate(untagged_addr, candidate, left, right); + num_descriptions_printed++; } hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { @@ -398,6 +448,8 @@ void PrintAddressDescription( if (FindHeapAllocation(t->heap_allocations(), tagged_addr, &har, &ring_index, &num_matching_addrs, &num_matching_addrs_4b)) { + Printf("%s", d.Error()); + Printf("\nCause: use-after-free\n"); Printf("%s", d.Location()); Printf("%p is located %zd bytes inside of %zd-byte region [%p,%p)\n", untagged_addr, untagged_addr - UntagAddr(har.tagged_addr), @@ -424,29 +476,25 @@ void PrintAddressDescription( t->Announce(); num_descriptions_printed++; } - - // Very basic check for stack memory. - if (t->AddrIsInStack(untagged_addr)) { - Printf("%s", d.Location()); - Printf("Address %p is located in stack of thread T%zd\n", untagged_addr, - t->unique_id()); - Printf("%s", d.Default()); - t->Announce(); - - auto *sa = (t == GetCurrentThread() && current_stack_allocations) - ? current_stack_allocations - : t->stack_allocations(); - PrintStackAllocations(sa, addr_tag, untagged_addr); - num_descriptions_printed++; - } }); + if (candidate && num_descriptions_printed == 0) { + ShowHeapOrGlobalCandidate(untagged_addr, candidate, left, right); + num_descriptions_printed++; + } + // Print the remaining threads, as an extra information, 1 line per thread. hwasanThreadList().VisitAllLiveThreads([&](Thread *t) { t->Announce(); }); if (!num_descriptions_printed) // We exhausted our possibilities. Bail out. Printf("HWAddressSanitizer can not describe address in more detail.\n"); + if (num_descriptions_printed > 1) { + Printf( + "There are %d potential causes, printed above in order " + "of likeliness.\n", + num_descriptions_printed); + } } void ReportStats() {} @@ -538,6 +586,12 @@ void ReportTailOverwritten(StackTrace *stack, uptr tagged_addr, uptr orig_size, Report("ERROR: %s: %s; heap object [%p,%p) of size %zd\n", SanitizerToolName, bug_type, untagged_addr, untagged_addr + orig_size, orig_size); Printf("\n%s", d.Default()); + Printf( + "Stack of invalid access unknown. Issue detected at deallocation " + "time.\n"); + Printf("%s", d.Allocation()); + Printf("deallocated here:\n"); + Printf("%s", d.Default()); stack->Print(); HwasanChunkView chunk = FindHeapChunkByAddress(untagged_addr); if (chunk.Beg()) { @@ -657,8 +711,10 @@ void ReportRegisters(uptr *frame, uptr pc) { frame[20], frame[21], frame[22], frame[23]); Printf(" x24 %016llx x25 %016llx x26 %016llx x27 %016llx\n", frame[24], frame[25], frame[26], frame[27]); - Printf(" x28 %016llx x29 %016llx x30 %016llx\n", - frame[28], frame[29], frame[30]); + // hwasan_check* reduces the stack pointer by 256, then __hwasan_tag_mismatch + // passes it to this function. + Printf(" x28 %016llx x29 %016llx x30 %016llx sp %016llx\n", frame[28], + frame[29], frame[30], reinterpret_cast<u8 *>(frame) + 256); } } // namespace __hwasan diff --git a/libsanitizer/hwasan/hwasan_thread.cpp b/libsanitizer/hwasan/hwasan_thread.cpp index bb4d56a..ee747a3 100644 --- a/libsanitizer/hwasan/hwasan_thread.cpp +++ b/libsanitizer/hwasan/hwasan_thread.cpp @@ -34,7 +34,8 @@ void Thread::InitRandomState() { stack_allocations_->push(0); } -void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) { +void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size, + const InitState *state) { CHECK_EQ(0, unique_id_); // try to catch bad stack reuse CHECK_EQ(0, stack_top_); CHECK_EQ(0, stack_bottom_); @@ -44,6 +45,17 @@ void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) { if (auto sz = flags()->heap_history_size) heap_allocations_ = HeapAllocationsRingBuffer::New(sz); + InitStackAndTls(state); +#if !SANITIZER_FUCHSIA + // Do not initialize the stack ring buffer just yet on Fuchsia. Threads will + // be initialized before we enter the thread itself, so we will instead call + // this later. + InitStackRingBuffer(stack_buffer_start, stack_buffer_size); +#endif +} + +void Thread::InitStackRingBuffer(uptr stack_buffer_start, + uptr stack_buffer_size) { HwasanTSDThreadInit(); // Only needed with interceptors. uptr *ThreadLong = GetCurrentThreadLongPtr(); // The following implicitly sets (this) as the current thread. @@ -55,13 +67,6 @@ void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size) { // ScopedTaggingDisable needs GetCurrentThread to be set up. ScopedTaggingDisabler disabler; - uptr tls_size; - uptr stack_size; - GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_, - &tls_size); - stack_top_ = stack_bottom_ + stack_size; - tls_end_ = tls_begin_ + tls_size; - if (stack_bottom_) { int local; CHECK(AddrIsInStack((uptr)&local)); diff --git a/libsanitizer/hwasan/hwasan_thread.h b/libsanitizer/hwasan/hwasan_thread.h index 1c71cab..9f20afe 100644 --- a/libsanitizer/hwasan/hwasan_thread.h +++ b/libsanitizer/hwasan/hwasan_thread.h @@ -23,8 +23,17 @@ typedef __sanitizer::CompactRingBuffer<uptr> StackAllocationsRingBuffer; class Thread { public: - void Init(uptr stack_buffer_start, uptr stack_buffer_size); // Must be called from the thread itself. + // These are optional parameters that can be passed to Init. + struct InitState; + + void Init(uptr stack_buffer_start, uptr stack_buffer_size, + const InitState *state = nullptr); void InitRandomState(); + void InitStackAndTls(const InitState *state = nullptr); + + // Must be called from the thread itself. + void InitStackRingBuffer(uptr stack_buffer_start, uptr stack_buffer_size); + void Destroy(); uptr stack_top() { return stack_top_; } diff --git a/libsanitizer/hwasan/hwasan_thread_list.cpp b/libsanitizer/hwasan/hwasan_thread_list.cpp index a31eee8..fa46e65 100644 --- a/libsanitizer/hwasan/hwasan_thread_list.cpp +++ b/libsanitizer/hwasan/hwasan_thread_list.cpp @@ -12,4 +12,4 @@ void InitThreadList(uptr storage, uptr size) { new (thread_list_placeholder) HwasanThreadList(storage, size); } -} // namespace +} // namespace __hwasan diff --git a/libsanitizer/hwasan/hwasan_thread_list.h b/libsanitizer/hwasan/hwasan_thread_list.h index 11c5863..15916a8 100644 --- a/libsanitizer/hwasan/hwasan_thread_list.h +++ b/libsanitizer/hwasan/hwasan_thread_list.h @@ -85,7 +85,7 @@ class HwasanThreadList { RoundUpTo(ring_buffer_size_ + sizeof(Thread), ring_buffer_size_ * 2); } - Thread *CreateCurrentThread() { + Thread *CreateCurrentThread(const Thread::InitState *state = nullptr) { Thread *t = nullptr; { SpinMutexLock l(&free_list_mutex_); @@ -104,7 +104,7 @@ class HwasanThreadList { SpinMutexLock l(&live_list_mutex_); live_list_.push_back(t); } - t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_); + t->Init((uptr)t - ring_buffer_size_, ring_buffer_size_, state); AddThreadStats(t); return t; } @@ -171,6 +171,8 @@ class HwasanThreadList { return stats_; } + uptr GetRingBufferSize() const { return ring_buffer_size_; } + private: Thread *AllocThread() { SpinMutexLock l(&free_space_mutex_); @@ -200,4 +202,4 @@ class HwasanThreadList { void InitThreadList(uptr storage, uptr size); HwasanThreadList &hwasanThreadList(); -} // namespace +} // namespace __hwasan diff --git a/libsanitizer/include/sanitizer/dfsan_interface.h b/libsanitizer/include/sanitizer/dfsan_interface.h index 40f9379..cd3b6d6 100644 --- a/libsanitizer/include/sanitizer/dfsan_interface.h +++ b/libsanitizer/include/sanitizer/dfsan_interface.h @@ -21,34 +21,15 @@ extern "C" { #endif -typedef uint16_t dfsan_label; +typedef uint8_t dfsan_label; typedef uint32_t dfsan_origin; -/// Stores information associated with a specific label identifier. A label -/// may be a base label created using dfsan_create_label, with associated -/// text description and user data, or an automatically created union label, -/// which represents the union of two label identifiers (which may themselves -/// be base or union labels). -struct dfsan_label_info { - // Fields for union labels, set to 0 for base labels. - dfsan_label l1; - dfsan_label l2; - - // Fields for base labels. - const char *desc; - void *userdata; -}; - /// Signature of the callback argument to dfsan_set_write_callback(). typedef void (*dfsan_write_callback_t)(int fd, const void *buf, size_t count); -/// Computes the union of \c l1 and \c l2, possibly creating a union label in -/// the process. +/// Computes the union of \c l1 and \c l2, resulting in a union label. dfsan_label dfsan_union(dfsan_label l1, dfsan_label l2); -/// Creates and returns a base label with the given description and user data. -dfsan_label dfsan_create_label(const char *desc, void *userdata); - /// Sets the label for each address in [addr,addr+size) to \c label. void dfsan_set_label(dfsan_label label, void *addr, size_t size); @@ -73,19 +54,9 @@ dfsan_origin dfsan_get_origin(long data); /// Retrieves the label associated with the data at the given address. dfsan_label dfsan_read_label(const void *addr, size_t size); -/// Retrieves a pointer to the dfsan_label_info struct for the given label. -const struct dfsan_label_info *dfsan_get_label_info(dfsan_label label); - /// Returns whether the given label label contains the label elem. int dfsan_has_label(dfsan_label label, dfsan_label elem); -/// If the given label label contains a label with the description desc, returns -/// that label, else returns 0. -dfsan_label dfsan_has_label_with_desc(dfsan_label label, const char *desc); - -/// Returns the number of labels allocated. -size_t dfsan_get_label_count(void); - /// Flushes the DFSan shadow, i.e. forgets about all labels currently associated /// with the application memory. Use this call to start over the taint tracking /// within the same process. @@ -99,12 +70,6 @@ void dfsan_flush(void); /// callback executes. Pass in NULL to remove any callback. void dfsan_set_write_callback(dfsan_write_callback_t labeled_write_callback); -/// Writes the labels currently used by the program to the given file -/// descriptor. The lines of the output have the following format: -/// -/// <label> <parent label 1> <parent label 2> <label description if any> -void dfsan_dump_labels(int fd); - /// Interceptor hooks. /// Whenever a dfsan's custom function is called the corresponding /// hook is called it non-zero. The hooks should be defined by the user. @@ -123,9 +88,65 @@ void dfsan_weak_hook_strncmp(void *caller_pc, const char *s1, const char *s2, /// on, or the address is not labeled, it prints nothing. void dfsan_print_origin_trace(const void *addr, const char *description); +/// Prints the origin trace of the label at the address \p addr to a +/// pre-allocated output buffer. If origin tracking is not on, or the address is +/// not labeled, it prints nothing. +/// +/// Typical usage: +/// \code +/// char kDescription[] = "..."; +/// char buf[1024]; +/// dfsan_sprint_origin_trace(&tainted_var, kDescription, buf, sizeof(buf)); +/// \endcode +/// +/// Typical usage that handles truncation: +/// \code +/// char buf[1024]; +/// int len = dfsan_sprint_origin_trace(&var, nullptr, buf, sizeof(buf)); +/// +/// if (len < sizeof(buf)) { +/// ProcessOriginTrace(buf); +/// } else { +/// char *tmpbuf = new char[len + 1]; +/// dfsan_sprint_origin_trace(&var, nullptr, tmpbuf, len + 1); +/// ProcessOriginTrace(tmpbuf); +/// delete[] tmpbuf; +/// } +/// \endcode +/// +/// \param addr The tainted memory address whose origin we are printing. +/// \param description A description printed at the beginning of the trace. +/// \param [out] out_buf The output buffer to write the results to. +/// \param out_buf_size The size of \p out_buf. +/// +/// \returns The number of symbols that should have been written to \p out_buf +/// (not including trailing null byte '\0'). Thus, the string is truncated iff +/// return value is not less than \p out_buf_size. +size_t dfsan_sprint_origin_trace(const void *addr, const char *description, + char *out_buf, size_t out_buf_size); + +/// Prints the stack trace leading to this call to a pre-allocated output +/// buffer. +/// +/// For usage examples, see dfsan_sprint_origin_trace. +/// +/// \param [out] out_buf The output buffer to write the results to. +/// \param out_buf_size The size of \p out_buf. +/// +/// \returns The number of symbols that should have been written to \p out_buf +/// (not including trailing null byte '\0'). Thus, the string is truncated iff +/// return value is not less than \p out_buf_size. +size_t dfsan_sprint_stack_trace(char *out_buf, size_t out_buf_size); + /// Retrieves the very first origin associated with the data at the given /// address. dfsan_origin dfsan_get_init_origin(const void *addr); + +/// Returns the value of -dfsan-track-origins. +/// * 0: do not track origins. +/// * 1: track origins at memory store operations. +/// * 2: track origins at memory load and store operations. +int dfsan_get_track_origins(void); #ifdef __cplusplus } // extern "C" diff --git a/libsanitizer/interception/interception.h b/libsanitizer/interception/interception.h index cb0b528..d8dc092 100644 --- a/libsanitizer/interception/interception.h +++ b/libsanitizer/interception/interception.h @@ -16,10 +16,10 @@ #include "sanitizer_common/sanitizer_internal_defs.h" -#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \ - !SANITIZER_NETBSD && !SANITIZER_WINDOWS && \ - !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_SOLARIS -# error "Interception doesn't work on this operating system." +#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_MAC && \ + !SANITIZER_NETBSD && !SANITIZER_WINDOWS && !SANITIZER_FUCHSIA && \ + !SANITIZER_SOLARIS +# error "Interception doesn't work on this operating system." #endif // These typedefs should be used only in the interceptor definitions to replace @@ -130,11 +130,6 @@ const interpose_substitution substitution_##func_name[] \ extern "C" ret_type func(__VA_ARGS__); # define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \ extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__); -#elif SANITIZER_RTEMS -# define WRAP(x) x -# define WRAPPER_NAME(x) #x -# define INTERCEPTOR_ATTRIBUTE -# define DECLARE_WRAPPER(ret_type, func, ...) #elif SANITIZER_FREEBSD || SANITIZER_NETBSD # define WRAP(x) __interceptor_ ## x # define WRAPPER_NAME(x) "__interceptor_" #x @@ -162,10 +157,6 @@ const interpose_substitution substitution_##func_name[] \ # define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default"))) # define REAL(x) __unsanitized_##x # define DECLARE_REAL(ret_type, func, ...) -#elif SANITIZER_RTEMS -# define REAL(x) __real_ ## x -# define DECLARE_REAL(ret_type, func, ...) \ - extern "C" ret_type REAL(func)(__VA_ARGS__); #elif !SANITIZER_MAC # define PTR_TO_REAL(x) real_##x # define REAL(x) __interception::PTR_TO_REAL(x) @@ -184,10 +175,10 @@ const interpose_substitution substitution_##func_name[] \ # define ASSIGN_REAL(x, y) #endif // SANITIZER_MAC -#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS -# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ - DECLARE_REAL(ret_type, func, __VA_ARGS__) \ - extern "C" ret_type WRAP(func)(__VA_ARGS__); +#if !SANITIZER_FUCHSIA +# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ + DECLARE_REAL(ret_type, func, __VA_ARGS__) \ + extern "C" ret_type WRAP(func)(__VA_ARGS__); // Declare an interceptor and its wrapper defined in a different translation // unit (ex. asm). # define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \ @@ -202,11 +193,11 @@ const interpose_substitution substitution_##func_name[] \ // macros does its job. In exceptional cases you may need to call REAL(foo) // without defining INTERCEPTOR(..., foo, ...). For example, if you override // foo with an interceptor for other function. -#if !SANITIZER_MAC && !SANITIZER_FUCHSIA && !SANITIZER_RTEMS -# define DEFINE_REAL(ret_type, func, ...) \ +#if !SANITIZER_MAC && !SANITIZER_FUCHSIA +# define DEFINE_REAL(ret_type, func, ...) \ typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \ - namespace __interception { \ - FUNC_TYPE(func) PTR_TO_REAL(func); \ + namespace __interception { \ + FUNC_TYPE(func) PTR_TO_REAL(func); \ } #else # define DEFINE_REAL(ret_type, func, ...) diff --git a/libsanitizer/lsan/lsan.cpp b/libsanitizer/lsan/lsan.cpp index 2c0a3bf..b6adc24 100644 --- a/libsanitizer/lsan/lsan.cpp +++ b/libsanitizer/lsan/lsan.cpp @@ -35,18 +35,14 @@ void __sanitizer::BufferedStackTrace::UnwindImpl( uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { using namespace __lsan; uptr stack_top = 0, stack_bottom = 0; - ThreadContext *t; - if (StackTrace::WillUseFastUnwind(request_fast) && - (t = CurrentThreadContext())) { + if (ThreadContext *t = CurrentThreadContext()) { stack_top = t->stack_end(); stack_bottom = t->stack_begin(); } - if (!SANITIZER_MIPS || IsValidFrame(bp, stack_top, stack_bottom)) { - if (StackTrace::WillUseFastUnwind(request_fast)) - Unwind(max_depth, pc, bp, nullptr, stack_top, stack_bottom, true); - else - Unwind(max_depth, pc, 0, context, 0, 0, false); - } + if (SANITIZER_MIPS && !IsValidFrame(bp, stack_top, stack_bottom)) + return; + bool fast = StackTrace::WillUseFastUnwind(request_fast); + Unwind(max_depth, pc, bp, context, stack_top, stack_bottom, fast); } using namespace __lsan; diff --git a/libsanitizer/lsan/lsan_common.h b/libsanitizer/lsan/lsan_common.h index fe855cf..776ca60 100644 --- a/libsanitizer/lsan/lsan_common.h +++ b/libsanitizer/lsan/lsan_common.h @@ -221,8 +221,8 @@ void UnlockAllocator(); // Returns true if [addr, addr + sizeof(void *)) is poisoned. bool WordIsPoisoned(uptr addr); // Wrappers for ThreadRegistry access. -void LockThreadRegistry(); -void UnlockThreadRegistry(); +void LockThreadRegistry() NO_THREAD_SAFETY_ANALYSIS; +void UnlockThreadRegistry() NO_THREAD_SAFETY_ANALYSIS; ThreadRegistry *GetThreadRegistryLocked(); bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end, uptr *tls_begin, uptr *tls_end, uptr *cache_begin, diff --git a/libsanitizer/lsan/lsan_thread.cpp b/libsanitizer/lsan/lsan_thread.cpp index 8efb54a..1d224eb 100644 --- a/libsanitizer/lsan/lsan_thread.cpp +++ b/libsanitizer/lsan/lsan_thread.cpp @@ -30,13 +30,10 @@ static ThreadContextBase *CreateThreadContext(u32 tid) { return new (mem) ThreadContext(tid); } -static const uptr kMaxThreads = 1 << 13; -static const uptr kThreadQuarantineSize = 64; - void InitializeThreadRegistry() { static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)]; - thread_registry = new (thread_registry_placeholder) - ThreadRegistry(CreateThreadContext, kMaxThreads, kThreadQuarantineSize); + thread_registry = + new (thread_registry_placeholder) ThreadRegistry(CreateThreadContext); } ThreadContextLsanBase::ThreadContextLsanBase(int tid) diff --git a/libsanitizer/sanitizer_common/Makefile.am b/libsanitizer/sanitizer_common/Makefile.am index 7cba534..d04f2d8 100644 --- a/libsanitizer/sanitizer_common/Makefile.am +++ b/libsanitizer/sanitizer_common/Makefile.am @@ -41,6 +41,7 @@ sanitizer_common_files = \ sanitizer_linux_s390.cpp \ sanitizer_mac.cpp \ sanitizer_mac_libcdep.cpp \ + sanitizer_mutex.cpp \ sanitizer_netbsd.cpp \ sanitizer_openbsd.cpp \ sanitizer_persistent_allocator.cpp \ @@ -57,7 +58,6 @@ sanitizer_common_files = \ sanitizer_procmaps_linux.cpp \ sanitizer_procmaps_mac.cpp \ sanitizer_procmaps_solaris.cpp \ - sanitizer_rtems.cpp \ sanitizer_solaris.cpp \ sanitizer_stackdepot.cpp \ sanitizer_stacktrace.cpp \ diff --git a/libsanitizer/sanitizer_common/Makefile.in b/libsanitizer/sanitizer_common/Makefile.in index 7e5555c..2856894 100644 --- a/libsanitizer/sanitizer_common/Makefile.in +++ b/libsanitizer/sanitizer_common/Makefile.in @@ -128,8 +128,9 @@ am__objects_1 = sancov_flags.lo sanitizer_allocator.lo \ sanitizer_file.lo sanitizer_flags.lo sanitizer_flag_parser.lo \ sanitizer_libc.lo sanitizer_libignore.lo sanitizer_linux.lo \ sanitizer_linux_libcdep.lo sanitizer_linux_s390.lo \ - sanitizer_mac.lo sanitizer_mac_libcdep.lo sanitizer_netbsd.lo \ - sanitizer_openbsd.lo sanitizer_persistent_allocator.lo \ + sanitizer_mac.lo sanitizer_mac_libcdep.lo sanitizer_mutex.lo \ + sanitizer_netbsd.lo sanitizer_openbsd.lo \ + sanitizer_persistent_allocator.lo \ sanitizer_platform_limits_freebsd.lo \ sanitizer_platform_limits_linux.lo \ sanitizer_platform_limits_openbsd.lo \ @@ -138,11 +139,11 @@ am__objects_1 = sancov_flags.lo sanitizer_allocator.lo \ sanitizer_posix_libcdep.lo sanitizer_printf.lo \ sanitizer_procmaps_bsd.lo sanitizer_procmaps_common.lo \ sanitizer_procmaps_linux.lo sanitizer_procmaps_mac.lo \ - sanitizer_procmaps_solaris.lo sanitizer_rtems.lo \ - sanitizer_solaris.lo sanitizer_stackdepot.lo \ - sanitizer_stacktrace.lo sanitizer_stacktrace_libcdep.lo \ - sanitizer_stacktrace_sparc.lo sanitizer_symbolizer_mac.lo \ - sanitizer_symbolizer_report.lo sanitizer_stacktrace_printer.lo \ + sanitizer_procmaps_solaris.lo sanitizer_solaris.lo \ + sanitizer_stackdepot.lo sanitizer_stacktrace.lo \ + sanitizer_stacktrace_libcdep.lo sanitizer_stacktrace_sparc.lo \ + sanitizer_symbolizer_mac.lo sanitizer_symbolizer_report.lo \ + sanitizer_stacktrace_printer.lo \ sanitizer_stoptheworld_linux_libcdep.lo \ sanitizer_stoptheworld_mac.lo sanitizer_suppressions.lo \ sanitizer_symbolizer.lo sanitizer_symbolizer_libbacktrace.lo \ @@ -400,6 +401,7 @@ sanitizer_common_files = \ sanitizer_linux_s390.cpp \ sanitizer_mac.cpp \ sanitizer_mac_libcdep.cpp \ + sanitizer_mutex.cpp \ sanitizer_netbsd.cpp \ sanitizer_openbsd.cpp \ sanitizer_persistent_allocator.cpp \ @@ -416,7 +418,6 @@ sanitizer_common_files = \ sanitizer_procmaps_linux.cpp \ sanitizer_procmaps_mac.cpp \ sanitizer_procmaps_solaris.cpp \ - sanitizer_rtems.cpp \ sanitizer_solaris.cpp \ sanitizer_stackdepot.cpp \ sanitizer_stacktrace.cpp \ @@ -557,6 +558,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_linux_s390.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_mac.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_mac_libcdep.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_mutex.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_netbsd.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_openbsd.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_persistent_allocator.Plo@am__quote@ @@ -573,7 +575,6 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_procmaps_linux.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_procmaps_mac.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_procmaps_solaris.Plo@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_rtems.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_solaris.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_stackdepot.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/sanitizer_stacktrace.Plo@am__quote@ diff --git a/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h b/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h index a033e78..15f81a0 100644 --- a/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h +++ b/libsanitizer/sanitizer_common/sanitizer_addrhashmap.h @@ -162,8 +162,8 @@ AddrHashMap<T, kSize>::AddrHashMap() { table_ = (Bucket*)MmapOrDie(kSize * sizeof(table_[0]), "AddrHashMap"); } -template<typename T, uptr kSize> -void AddrHashMap<T, kSize>::acquire(Handle *h) { +template <typename T, uptr kSize> +void AddrHashMap<T, kSize>::acquire(Handle *h) NO_THREAD_SAFETY_ANALYSIS { uptr addr = h->addr_; uptr hash = calcHash(addr); Bucket *b = &table_[hash]; @@ -289,57 +289,57 @@ void AddrHashMap<T, kSize>::acquire(Handle *h) { CHECK_EQ(atomic_load(&c->addr, memory_order_relaxed), 0); h->addidx_ = i; h->cell_ = c; -} - -template<typename T, uptr kSize> -void AddrHashMap<T, kSize>::release(Handle *h) { - if (!h->cell_) - return; - Bucket *b = h->bucket_; - Cell *c = h->cell_; - uptr addr1 = atomic_load(&c->addr, memory_order_relaxed); - if (h->created_) { - // Denote completion of insertion. - CHECK_EQ(addr1, 0); - // After the following store, the element becomes available - // for lock-free reads. - atomic_store(&c->addr, h->addr_, memory_order_release); - b->mtx.Unlock(); - } else if (h->remove_) { - // Denote that the cell is empty now. - CHECK_EQ(addr1, h->addr_); - atomic_store(&c->addr, 0, memory_order_release); - // See if we need to compact the bucket. - AddBucket *add = (AddBucket*)atomic_load(&b->add, memory_order_relaxed); - if (h->addidx_ == -1U) { - // Removed from embed array, move an add element into the freed cell. - if (add && add->size != 0) { - uptr last = --add->size; - Cell *c1 = &add->cells[last]; - c->val = c1->val; - uptr addr1 = atomic_load(&c1->addr, memory_order_relaxed); - atomic_store(&c->addr, addr1, memory_order_release); - atomic_store(&c1->addr, 0, memory_order_release); - } - } else { - // Removed from add array, compact it. - uptr last = --add->size; - Cell *c1 = &add->cells[last]; - if (c != c1) { - *c = *c1; - atomic_store(&c1->addr, 0, memory_order_relaxed); - } - } - if (add && add->size == 0) { - // FIXME(dvyukov): free add? - } - b->mtx.Unlock(); - } else { - CHECK_EQ(addr1, h->addr_); - if (h->addidx_ != -1U) - b->mtx.ReadUnlock(); - } -} + } + + template <typename T, uptr kSize> + void AddrHashMap<T, kSize>::release(Handle *h) NO_THREAD_SAFETY_ANALYSIS { + if (!h->cell_) + return; + Bucket *b = h->bucket_; + Cell *c = h->cell_; + uptr addr1 = atomic_load(&c->addr, memory_order_relaxed); + if (h->created_) { + // Denote completion of insertion. + CHECK_EQ(addr1, 0); + // After the following store, the element becomes available + // for lock-free reads. + atomic_store(&c->addr, h->addr_, memory_order_release); + b->mtx.Unlock(); + } else if (h->remove_) { + // Denote that the cell is empty now. + CHECK_EQ(addr1, h->addr_); + atomic_store(&c->addr, 0, memory_order_release); + // See if we need to compact the bucket. + AddBucket *add = (AddBucket *)atomic_load(&b->add, memory_order_relaxed); + if (h->addidx_ == -1U) { + // Removed from embed array, move an add element into the freed cell. + if (add && add->size != 0) { + uptr last = --add->size; + Cell *c1 = &add->cells[last]; + c->val = c1->val; + uptr addr1 = atomic_load(&c1->addr, memory_order_relaxed); + atomic_store(&c->addr, addr1, memory_order_release); + atomic_store(&c1->addr, 0, memory_order_release); + } + } else { + // Removed from add array, compact it. + uptr last = --add->size; + Cell *c1 = &add->cells[last]; + if (c != c1) { + *c = *c1; + atomic_store(&c1->addr, 0, memory_order_relaxed); + } + } + if (add && add->size == 0) { + // FIXME(dvyukov): free add? + } + b->mtx.Unlock(); + } else { + CHECK_EQ(addr1, h->addr_); + if (h->addidx_ != -1U) + b->mtx.ReadUnlock(); + } + } template<typename T, uptr kSize> uptr AddrHashMap<T, kSize>::calcHash(uptr addr) { diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator.cpp b/libsanitizer/sanitizer_common/sanitizer_allocator.cpp index 3157b35..bcb7370 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_allocator.cpp @@ -137,14 +137,6 @@ static void RawInternalFree(void *ptr, InternalAllocatorCache *cache) { #endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC) -namespace { -const u64 kBlockMagic = 0x6A6CB03ABCEBC041ull; - -struct BlockHeader { - u64 magic; -}; -} // namespace - static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) { SetAllocatorOutOfMemory(); Report("FATAL: %s: internal allocator is out of memory trying to allocate " @@ -153,28 +145,17 @@ static void NORETURN ReportInternalAllocatorOutOfMemory(uptr requested_size) { } void *InternalAlloc(uptr size, InternalAllocatorCache *cache, uptr alignment) { - uptr s = size + sizeof(BlockHeader); - if (s < size) - return nullptr; - BlockHeader *p = (BlockHeader *)RawInternalAlloc(s, cache, alignment); + void *p = RawInternalAlloc(size, cache, alignment); if (UNLIKELY(!p)) - ReportInternalAllocatorOutOfMemory(s); - p->magic = kBlockMagic; - return p + 1; + ReportInternalAllocatorOutOfMemory(size); + return p; } void *InternalRealloc(void *addr, uptr size, InternalAllocatorCache *cache) { - if (!addr) - return InternalAlloc(size, cache); - uptr s = size + sizeof(BlockHeader); - if (s < size) - return nullptr; - BlockHeader *p = (BlockHeader *)addr - 1; - CHECK_EQ(kBlockMagic, p->magic); - p = (BlockHeader *)RawInternalRealloc(p, s, cache); + void *p = RawInternalRealloc(addr, size, cache); if (UNLIKELY(!p)) - ReportInternalAllocatorOutOfMemory(s); - return p + 1; + ReportInternalAllocatorOutOfMemory(size); + return p; } void *InternalReallocArray(void *addr, uptr count, uptr size, @@ -203,12 +184,7 @@ void *InternalCalloc(uptr count, uptr size, InternalAllocatorCache *cache) { } void InternalFree(void *addr, InternalAllocatorCache *cache) { - if (!addr) - return; - BlockHeader *p = (BlockHeader *)addr - 1; - CHECK_EQ(kBlockMagic, p->magic); - p->magic = 0; - RawInternalFree(p, cache); + RawInternalFree(addr, cache); } // LowLevelAllocator diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h b/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h index eb836bc..0e81e67 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_combined.h @@ -177,12 +177,12 @@ class CombinedAllocator { // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone // introspection API. - void ForceLock() { + void ForceLock() NO_THREAD_SAFETY_ANALYSIS { primary_.ForceLock(); secondary_.ForceLock(); } - void ForceUnlock() { + void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS { secondary_.ForceUnlock(); primary_.ForceUnlock(); } diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h b/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h index 108dfc2..e495c56 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_local_cache.h @@ -17,6 +17,7 @@ template <class SizeClassAllocator> struct SizeClassAllocator64LocalCache { typedef SizeClassAllocator Allocator; + typedef MemoryMapper<Allocator> MemoryMapperT; void Init(AllocatorGlobalStats *s) { stats_.Init(); @@ -53,7 +54,7 @@ struct SizeClassAllocator64LocalCache { PerClass *c = &per_class_[class_id]; InitCache(c); if (UNLIKELY(c->count == c->max_count)) - Drain(c, allocator, class_id, c->max_count / 2); + DrainHalfMax(c, allocator, class_id); CompactPtrT chunk = allocator->PointerToCompactPtr( allocator->GetRegionBeginBySizeClass(class_id), reinterpret_cast<uptr>(p)); @@ -62,10 +63,10 @@ struct SizeClassAllocator64LocalCache { } void Drain(SizeClassAllocator *allocator) { + MemoryMapperT memory_mapper(*allocator); for (uptr i = 1; i < kNumClasses; i++) { PerClass *c = &per_class_[i]; - while (c->count > 0) - Drain(c, allocator, i, c->count); + while (c->count > 0) Drain(&memory_mapper, c, allocator, i, c->count); } } @@ -106,12 +107,18 @@ struct SizeClassAllocator64LocalCache { return true; } - NOINLINE void Drain(PerClass *c, SizeClassAllocator *allocator, uptr class_id, - uptr count) { + NOINLINE void DrainHalfMax(PerClass *c, SizeClassAllocator *allocator, + uptr class_id) { + MemoryMapperT memory_mapper(*allocator); + Drain(&memory_mapper, c, allocator, class_id, c->max_count / 2); + } + + void Drain(MemoryMapperT *memory_mapper, PerClass *c, + SizeClassAllocator *allocator, uptr class_id, uptr count) { CHECK_GE(c->count, count); const uptr first_idx_to_drain = c->count - count; c->count -= count; - allocator->ReturnToAllocator(&stats_, class_id, + allocator->ReturnToAllocator(memory_mapper, &stats_, class_id, &c->chunks[first_idx_to_drain], count); } }; diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h b/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h index fb5394c..38d2a7d 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_primary32.h @@ -237,13 +237,13 @@ class SizeClassAllocator32 { // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone // introspection API. - void ForceLock() { + void ForceLock() NO_THREAD_SAFETY_ANALYSIS { for (uptr i = 0; i < kNumClasses; i++) { GetSizeClassInfo(i)->mutex.Lock(); } } - void ForceUnlock() { + void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS { for (int i = kNumClasses - 1; i >= 0; i--) { GetSizeClassInfo(i)->mutex.Unlock(); } diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h b/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h index db30e13..b142ee0 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_primary64.h @@ -42,6 +42,44 @@ struct SizeClassAllocator64FlagMasks { // Bit masks. }; }; +template <typename Allocator> +class MemoryMapper { + public: + typedef typename Allocator::CompactPtrT CompactPtrT; + + explicit MemoryMapper(const Allocator &allocator) : allocator_(allocator) {} + + bool GetAndResetStats(uptr &ranges, uptr &bytes) { + ranges = released_ranges_count_; + released_ranges_count_ = 0; + bytes = released_bytes_; + released_bytes_ = 0; + return ranges != 0; + } + + u64 *MapPackedCounterArrayBuffer(uptr count) { + buffer_.clear(); + buffer_.resize(count); + return buffer_.data(); + } + + // Releases [from, to) range of pages back to OS. + void ReleasePageRangeToOS(uptr class_id, CompactPtrT from, CompactPtrT to) { + const uptr region_base = allocator_.GetRegionBeginBySizeClass(class_id); + const uptr from_page = allocator_.CompactPtrToPointer(region_base, from); + const uptr to_page = allocator_.CompactPtrToPointer(region_base, to); + ReleaseMemoryPagesToOS(from_page, to_page); + released_ranges_count_++; + released_bytes_ += to_page - from_page; + } + + private: + const Allocator &allocator_; + uptr released_ranges_count_ = 0; + uptr released_bytes_ = 0; + InternalMmapVector<u64> buffer_; +}; + template <class Params> class SizeClassAllocator64 { public: @@ -57,6 +95,7 @@ class SizeClassAllocator64 { typedef SizeClassAllocator64<Params> ThisT; typedef SizeClassAllocator64LocalCache<ThisT> AllocatorCache; + typedef MemoryMapper<ThisT> MemoryMapperT; // When we know the size class (the region base) we can represent a pointer // as a 4-byte integer (offset from the region start shifted right by 4). @@ -120,9 +159,10 @@ class SizeClassAllocator64 { } void ForceReleaseToOS() { + MemoryMapperT memory_mapper(*this); for (uptr class_id = 1; class_id < kNumClasses; class_id++) { BlockingMutexLock l(&GetRegionInfo(class_id)->mutex); - MaybeReleaseToOS(class_id, true /*force*/); + MaybeReleaseToOS(&memory_mapper, class_id, true /*force*/); } } @@ -131,7 +171,8 @@ class SizeClassAllocator64 { alignment <= SizeClassMap::kMaxSize; } - NOINLINE void ReturnToAllocator(AllocatorStats *stat, uptr class_id, + NOINLINE void ReturnToAllocator(MemoryMapperT *memory_mapper, + AllocatorStats *stat, uptr class_id, const CompactPtrT *chunks, uptr n_chunks) { RegionInfo *region = GetRegionInfo(class_id); uptr region_beg = GetRegionBeginBySizeClass(class_id); @@ -154,7 +195,7 @@ class SizeClassAllocator64 { region->num_freed_chunks = new_num_freed_chunks; region->stats.n_freed += n_chunks; - MaybeReleaseToOS(class_id, false /*force*/); + MaybeReleaseToOS(memory_mapper, class_id, false /*force*/); } NOINLINE bool GetFromAllocator(AllocatorStats *stat, uptr class_id, @@ -312,13 +353,13 @@ class SizeClassAllocator64 { // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone // introspection API. - void ForceLock() { + void ForceLock() NO_THREAD_SAFETY_ANALYSIS { for (uptr i = 0; i < kNumClasses; i++) { GetRegionInfo(i)->mutex.Lock(); } } - void ForceUnlock() { + void ForceUnlock() NO_THREAD_SAFETY_ANALYSIS { for (int i = (int)kNumClasses - 1; i >= 0; i--) { GetRegionInfo(i)->mutex.Unlock(); } @@ -362,11 +403,11 @@ class SizeClassAllocator64 { // For the performance sake, none of the accessors check the validity of the // arguments, it is assumed that index is always in [0, n) range and the value // is not incremented past max_value. - template<class MemoryMapperT> class PackedCounterArray { public: - PackedCounterArray(u64 num_counters, u64 max_value, MemoryMapperT *mapper) - : n(num_counters), memory_mapper(mapper) { + template <typename MemoryMapper> + PackedCounterArray(u64 num_counters, u64 max_value, MemoryMapper *mapper) + : n(num_counters) { CHECK_GT(num_counters, 0); CHECK_GT(max_value, 0); constexpr u64 kMaxCounterBits = sizeof(*buffer) * 8ULL; @@ -383,16 +424,8 @@ class SizeClassAllocator64 { packing_ratio_log = Log2(packing_ratio); bit_offset_mask = packing_ratio - 1; - buffer_size = - (RoundUpTo(n, 1ULL << packing_ratio_log) >> packing_ratio_log) * - sizeof(*buffer); - buffer = reinterpret_cast<u64*>( - memory_mapper->MapPackedCounterArrayBuffer(buffer_size)); - } - ~PackedCounterArray() { - if (buffer) { - memory_mapper->UnmapPackedCounterArrayBuffer(buffer, buffer_size); - } + buffer = mapper->MapPackedCounterArrayBuffer( + RoundUpTo(n, 1ULL << packing_ratio_log) >> packing_ratio_log); } bool IsAllocated() const { @@ -429,19 +462,16 @@ class SizeClassAllocator64 { u64 counter_mask; u64 packing_ratio_log; u64 bit_offset_mask; - - MemoryMapperT* const memory_mapper; - u64 buffer_size; u64* buffer; }; - template<class MemoryMapperT> + template <class MemoryMapperT> class FreePagesRangeTracker { public: - explicit FreePagesRangeTracker(MemoryMapperT* mapper) + FreePagesRangeTracker(MemoryMapperT *mapper, uptr class_id) : memory_mapper(mapper), - page_size_scaled_log(Log2(GetPageSizeCached() >> kCompactPtrScale)), - in_the_range(false), current_page(0), current_range_start_page(0) {} + class_id(class_id), + page_size_scaled_log(Log2(GetPageSizeCached() >> kCompactPtrScale)) {} void NextPage(bool freed) { if (freed) { @@ -463,28 +493,30 @@ class SizeClassAllocator64 { void CloseOpenedRange() { if (in_the_range) { memory_mapper->ReleasePageRangeToOS( - current_range_start_page << page_size_scaled_log, + class_id, current_range_start_page << page_size_scaled_log, current_page << page_size_scaled_log); in_the_range = false; } } - MemoryMapperT* const memory_mapper; - const uptr page_size_scaled_log; - bool in_the_range; - uptr current_page; - uptr current_range_start_page; + MemoryMapperT *const memory_mapper = nullptr; + const uptr class_id = 0; + const uptr page_size_scaled_log = 0; + bool in_the_range = false; + uptr current_page = 0; + uptr current_range_start_page = 0; }; // Iterates over the free_array to identify memory pages containing freed // chunks only and returns these pages back to OS. // allocated_pages_count is the total number of pages allocated for the // current bucket. - template<class MemoryMapperT> + template <typename MemoryMapper> static void ReleaseFreeMemoryToOS(CompactPtrT *free_array, uptr free_array_count, uptr chunk_size, uptr allocated_pages_count, - MemoryMapperT *memory_mapper) { + MemoryMapper *memory_mapper, + uptr class_id) { const uptr page_size = GetPageSizeCached(); // Figure out the number of chunks per page and whether we can take a fast @@ -520,9 +552,8 @@ class SizeClassAllocator64 { UNREACHABLE("All chunk_size/page_size ratios must be handled."); } - PackedCounterArray<MemoryMapperT> counters(allocated_pages_count, - full_pages_chunk_count_max, - memory_mapper); + PackedCounterArray counters(allocated_pages_count, + full_pages_chunk_count_max, memory_mapper); if (!counters.IsAllocated()) return; @@ -547,7 +578,7 @@ class SizeClassAllocator64 { // Iterate over pages detecting ranges of pages with chunk counters equal // to the expected number of chunks for the particular page. - FreePagesRangeTracker<MemoryMapperT> range_tracker(memory_mapper); + FreePagesRangeTracker<MemoryMapper> range_tracker(memory_mapper, class_id); if (same_chunk_count_per_page) { // Fast path, every page has the same number of chunks affecting it. for (uptr i = 0; i < counters.GetCount(); i++) @@ -586,7 +617,7 @@ class SizeClassAllocator64 { } private: - friend class MemoryMapper; + friend class MemoryMapper<ThisT>; ReservedAddressRange address_range; @@ -820,57 +851,13 @@ class SizeClassAllocator64 { return true; } - class MemoryMapper { - public: - MemoryMapper(const ThisT& base_allocator, uptr class_id) - : allocator(base_allocator), - region_base(base_allocator.GetRegionBeginBySizeClass(class_id)), - released_ranges_count(0), - released_bytes(0) { - } - - uptr GetReleasedRangesCount() const { - return released_ranges_count; - } - - uptr GetReleasedBytes() const { - return released_bytes; - } - - void *MapPackedCounterArrayBuffer(uptr buffer_size) { - // TODO(alekseyshl): The idea to explore is to check if we have enough - // space between num_freed_chunks*sizeof(CompactPtrT) and - // mapped_free_array to fit buffer_size bytes and use that space instead - // of mapping a temporary one. - return MmapOrDieOnFatalError(buffer_size, "ReleaseToOSPageCounters"); - } - - void UnmapPackedCounterArrayBuffer(void *buffer, uptr buffer_size) { - UnmapOrDie(buffer, buffer_size); - } - - // Releases [from, to) range of pages back to OS. - void ReleasePageRangeToOS(CompactPtrT from, CompactPtrT to) { - const uptr from_page = allocator.CompactPtrToPointer(region_base, from); - const uptr to_page = allocator.CompactPtrToPointer(region_base, to); - ReleaseMemoryPagesToOS(from_page, to_page); - released_ranges_count++; - released_bytes += to_page - from_page; - } - - private: - const ThisT& allocator; - const uptr region_base; - uptr released_ranges_count; - uptr released_bytes; - }; - // Attempts to release RAM occupied by freed chunks back to OS. The region is // expected to be locked. // // TODO(morehouse): Support a callback on memory release so HWASan can release // aliases as well. - void MaybeReleaseToOS(uptr class_id, bool force) { + void MaybeReleaseToOS(MemoryMapperT *memory_mapper, uptr class_id, + bool force) { RegionInfo *region = GetRegionInfo(class_id); const uptr chunk_size = ClassIdToSize(class_id); const uptr page_size = GetPageSizeCached(); @@ -894,17 +881,16 @@ class SizeClassAllocator64 { } } - MemoryMapper memory_mapper(*this, class_id); - - ReleaseFreeMemoryToOS<MemoryMapper>( + ReleaseFreeMemoryToOS( GetFreeArray(GetRegionBeginBySizeClass(class_id)), n, chunk_size, - RoundUpTo(region->allocated_user, page_size) / page_size, - &memory_mapper); + RoundUpTo(region->allocated_user, page_size) / page_size, memory_mapper, + class_id); - if (memory_mapper.GetReleasedRangesCount() > 0) { + uptr ranges, bytes; + if (memory_mapper->GetAndResetStats(ranges, bytes)) { region->rtoi.n_freed_at_last_release = region->stats.n_freed; - region->rtoi.num_releases += memory_mapper.GetReleasedRangesCount(); - region->rtoi.last_released_bytes = memory_mapper.GetReleasedBytes(); + region->rtoi.num_releases += ranges; + region->rtoi.last_released_bytes = bytes; } region->rtoi.last_release_at_ns = MonotonicNanoTime(); } diff --git a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h index 61fb987..dd34fe8 100644 --- a/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h +++ b/libsanitizer/sanitizer_common/sanitizer_allocator_secondary.h @@ -267,13 +267,9 @@ class LargeMmapAllocator { // ForceLock() and ForceUnlock() are needed to implement Darwin malloc zone // introspection API. - void ForceLock() { - mutex_.Lock(); - } + void ForceLock() ACQUIRE(mutex_) { mutex_.Lock(); } - void ForceUnlock() { - mutex_.Unlock(); - } + void ForceUnlock() RELEASE(mutex_) { mutex_.Unlock(); } // Iterate over all existing chunks. // The allocator must be locked when calling this function. diff --git a/libsanitizer/sanitizer_common/sanitizer_common.cpp b/libsanitizer/sanitizer_common/sanitizer_common.cpp index 33960d9..5fae8e3 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_common.cpp @@ -37,10 +37,9 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, const char *mmap_type, error_t err, bool raw_report) { static int recursion_count; - if (SANITIZER_RTEMS || raw_report || recursion_count) { - // If we are on RTEMS or raw report is requested or we went into recursion, - // just die. The Report() and CHECK calls below may call mmap recursively - // and fail. + if (raw_report || recursion_count) { + // If raw report is requested or we went into recursion just die. The + // Report() and CHECK calls below may call mmap recursively and fail. RawWrite("ERROR: Failed to mmap\n"); Die(); } @@ -331,6 +330,14 @@ static int InstallMallocFreeHooks(void (*malloc_hook)(const void *, uptr), return 0; } +void internal_sleep(unsigned seconds) { + internal_usleep((u64)seconds * 1000 * 1000); +} +void SleepForSeconds(unsigned seconds) { + internal_usleep((u64)seconds * 1000 * 1000); +} +void SleepForMillis(unsigned millis) { internal_usleep((u64)millis * 1000); } + } // namespace __sanitizer using namespace __sanitizer; diff --git a/libsanitizer/sanitizer_common/sanitizer_common.h b/libsanitizer/sanitizer_common/sanitizer_common.h index 7b65dd7..cbdbb0c 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common.h +++ b/libsanitizer/sanitizer_common/sanitizer_common.h @@ -237,10 +237,16 @@ void SetPrintfAndReportCallback(void (*callback)(const char *)); // Lock sanitizer error reporting and protects against nested errors. class ScopedErrorReportLock { public: - ScopedErrorReportLock(); - ~ScopedErrorReportLock(); + ScopedErrorReportLock() ACQUIRE(mutex_) { Lock(); } + ~ScopedErrorReportLock() RELEASE(mutex_) { Unlock(); } - static void CheckLocked(); + static void Lock() ACQUIRE(mutex_); + static void Unlock() RELEASE(mutex_); + static void CheckLocked() CHECK_LOCKED(mutex_); + + private: + static atomic_uintptr_t reporting_thread_; + static StaticSpinMutex mutex_; }; extern uptr stoptheworld_tracer_pid; @@ -288,8 +294,8 @@ void InitTlsSize(); uptr GetTlsSize(); // Other -void SleepForSeconds(int seconds); -void SleepForMillis(int millis); +void SleepForSeconds(unsigned seconds); +void SleepForMillis(unsigned millis); u64 NanoTime(); u64 MonotonicNanoTime(); int Atexit(void (*function)(void)); @@ -1057,6 +1063,13 @@ class ArrayRef { T *end_ = nullptr; }; +#define PRINTF_128(v) \ + (*((u8 *)&v + 0)), (*((u8 *)&v + 1)), (*((u8 *)&v + 2)), (*((u8 *)&v + 3)), \ + (*((u8 *)&v + 4)), (*((u8 *)&v + 5)), (*((u8 *)&v + 6)), \ + (*((u8 *)&v + 7)), (*((u8 *)&v + 8)), (*((u8 *)&v + 9)), \ + (*((u8 *)&v + 10)), (*((u8 *)&v + 11)), (*((u8 *)&v + 12)), \ + (*((u8 *)&v + 13)), (*((u8 *)&v + 14)), (*((u8 *)&v + 15)) + } // namespace __sanitizer inline void *operator new(__sanitizer::operator_new_size_type size, diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc index 7867fcc..6205d85 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc +++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc @@ -134,11 +134,11 @@ extern const short *_tolower_tab_; // Platform-specific options. #if SANITIZER_MAC -#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE false +#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0 #elif SANITIZER_WINDOWS64 -#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE false +#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0 #else -#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE true +#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1 #endif // SANITIZER_MAC #ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE @@ -823,11 +823,11 @@ INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) { // N.B.: If we switch this to internal_ we'll have to use internal_memmove // due to memcpy being an alias of memmove on OS X. void *ctx; - if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { +#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size); - } else { +#else COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); - } +#endif } #define INIT_MEMCPY \ @@ -957,6 +957,7 @@ INTERCEPTOR(double, frexp, double x, int *exp) { // Assuming frexp() always writes to |exp|. COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp)); double res = REAL(frexp)(x, exp); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(exp, sizeof(*exp)); return res; } @@ -969,22 +970,18 @@ INTERCEPTOR(double, frexp, double x, int *exp) { INTERCEPTOR(float, frexpf, float x, int *exp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, frexpf, x, exp); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. - float res = REAL(frexpf)(x, exp); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp)); + float res = REAL(frexpf)(x, exp); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(exp, sizeof(*exp)); return res; } INTERCEPTOR(long double, frexpl, long double x, int *exp) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, frexpl, x, exp); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. - long double res = REAL(frexpl)(x, exp); COMMON_INTERCEPTOR_WRITE_RANGE(ctx, exp, sizeof(*exp)); + long double res = REAL(frexpl)(x, exp); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(exp, sizeof(*exp)); return res; } @@ -5303,6 +5300,12 @@ INTERCEPTOR(__sanitizer_clock_t, times, void *tms) { #define INIT_TIMES #endif +#if SANITIZER_S390 && \ + (SANITIZER_INTERCEPT_TLS_GET_ADDR || SANITIZER_INTERCEPT_TLS_GET_OFFSET) +extern "C" uptr __tls_get_offset_wrapper(void *arg, uptr (*fn)(void *arg)); +DEFINE_REAL(uptr, __tls_get_offset, void *arg) +#endif + #if SANITIZER_INTERCEPT_TLS_GET_ADDR #if !SANITIZER_S390 #define INIT_TLS_GET_ADDR COMMON_INTERCEPT_FUNCTION(__tls_get_addr) @@ -5342,11 +5345,7 @@ void *__tls_get_addr_opt(void *arg); // descriptor offset as an argument instead of a pointer. GOT address // is passed in r12, so it's necessary to write it in assembly. This is // the function used by the compiler. -extern "C" uptr __tls_get_offset_wrapper(void *arg, uptr (*fn)(void *arg)); #define INIT_TLS_GET_ADDR COMMON_INTERCEPT_FUNCTION(__tls_get_offset) -DEFINE_REAL(uptr, __tls_get_offset, void *arg) -extern "C" uptr __tls_get_offset(void *arg); -extern "C" uptr __interceptor___tls_get_offset(void *arg); INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, __tls_get_addr_internal, arg); @@ -5362,6 +5361,15 @@ INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) { } return res; } +#endif // SANITIZER_S390 +#else +#define INIT_TLS_GET_ADDR +#endif + +#if SANITIZER_S390 && \ + (SANITIZER_INTERCEPT_TLS_GET_ADDR || SANITIZER_INTERCEPT_TLS_GET_OFFSET) +extern "C" uptr __tls_get_offset(void *arg); +extern "C" uptr __interceptor___tls_get_offset(void *arg); // We need a hidden symbol aliasing the above, so that we can jump // directly to it from the assembly below. extern "C" __attribute__((alias("__interceptor___tls_get_addr_internal"), @@ -5400,9 +5408,6 @@ asm( "br %r3\n" ".size __tls_get_offset_wrapper, .-__tls_get_offset_wrapper\n" ); -#endif // SANITIZER_S390 -#else -#define INIT_TLS_GET_ADDR #endif #if SANITIZER_INTERCEPT_LISTXATTR @@ -6099,6 +6104,40 @@ INTERCEPTOR(__sanitizer_FILE *, freopen, const char *path, const char *mode, #define INIT_FOPEN #endif +#if SANITIZER_INTERCEPT_FLOPEN +INTERCEPTOR(int, flopen, const char *path, int flags, ...) { + void *ctx; + va_list ap; + va_start(ap, flags); + u16 mode = static_cast<u16>(va_arg(ap, u32)); + va_end(ap); + COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode); + if (path) { + COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + } + return REAL(flopen)(path, flags, mode); +} + +INTERCEPTOR(int, flopenat, int dirfd, const char *path, int flags, ...) { + void *ctx; + va_list ap; + va_start(ap, flags); + u16 mode = static_cast<u16>(va_arg(ap, u32)); + va_end(ap); + COMMON_INTERCEPTOR_ENTER(ctx, flopen, path, flags, mode); + if (path) { + COMMON_INTERCEPTOR_READ_RANGE(ctx, path, REAL(strlen)(path) + 1); + } + return REAL(flopenat)(dirfd, path, flags, mode); +} + +#define INIT_FLOPEN \ + COMMON_INTERCEPT_FUNCTION(flopen); \ + COMMON_INTERCEPT_FUNCTION(flopenat); +#else +#define INIT_FLOPEN +#endif + #if SANITIZER_INTERCEPT_FOPEN64 INTERCEPTOR(__sanitizer_FILE *, fopen64, const char *path, const char *mode) { void *ctx; @@ -6463,7 +6502,7 @@ INTERCEPTOR(int, sem_wait, __sanitizer_sem_t *s) { INTERCEPTOR(int, sem_trywait, __sanitizer_sem_t *s) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, sem_trywait, s); - int res = COMMON_INTERCEPTOR_BLOCK_REAL(sem_trywait)(s); + int res = REAL(sem_trywait)(s); if (res == 0) { COMMON_INTERCEPTOR_ACQUIRE(ctx, (uptr)s); } @@ -10264,6 +10303,7 @@ static void InitializeCommonInterceptors() { INIT_LIBIO_INTERNALS; INIT_FOPEN; INIT_FOPEN64; + INIT_FLOPEN; INIT_OPEN_MEMSTREAM; INIT_OBSTACK; INIT_FFLUSH; diff --git a/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cpp index 1037938..01ccacc 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_common_libcdep.cpp @@ -138,7 +138,7 @@ uptr ReservedAddressRange::InitAligned(uptr size, uptr align, return start; } -#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS +#if !SANITIZER_FUCHSIA // Reserve memory range [beg, end]. // We need to use inclusive range because end+1 may not be representable. @@ -189,7 +189,7 @@ void ProtectGap(uptr addr, uptr size, uptr zero_base_shadow_start, Die(); } -#endif // !SANITIZER_FUCHSIA && !SANITIZER_RTEMS +#endif // !SANITIZER_FUCHSIA } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cpp b/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cpp index 487a634..9a4e538 100644 --- a/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_common_nolibc.cpp @@ -25,7 +25,6 @@ void LogMessageOnPrintf(const char *str) {} #endif void WriteToSyslog(const char *buffer) {} void Abort() { internal__exit(1); } -void SleepForSeconds(int seconds) { internal_sleep(seconds); } #endif // !SANITIZER_WINDOWS #if !SANITIZER_WINDOWS && !SANITIZER_MAC diff --git a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cpp b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cpp index 2c924f5..ccb7065 100644 --- a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector1.cpp @@ -136,7 +136,7 @@ void DD::ReportDeadlock(DDCallback *cb, DDMutex *m) { DDMutex *m0 = (DDMutex*)dd.getData(from); DDMutex *m1 = (DDMutex*)dd.getData(to); - u32 stk_from = -1U, stk_to = -1U; + u32 stk_from = 0, stk_to = 0; int unique_tid = 0; dd.findEdge(from, to, &stk_from, &stk_to, &unique_tid); // Printf("Edge: %zd=>%zd: %u/%u T%d\n", from, to, stk_from, stk_to, diff --git a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector2.cpp b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector2.cpp index e3f8e1b..1fbbbcc 100644 --- a/libsanitizer/sanitizer_common/sanitizer_deadlock_detector2.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_deadlock_detector2.cpp @@ -73,7 +73,7 @@ struct DDLogicalThread { int nlocked; }; -struct Mutex { +struct MutexState { StaticSpinMutex mtx; u32 seq; int nlink; @@ -101,12 +101,12 @@ struct DD final : public DDetector { void CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt, DDMutex *mtx); void Report(DDPhysicalThread *pt, DDLogicalThread *lt, int npath); u32 allocateId(DDCallback *cb); - Mutex *getMutex(u32 id); - u32 getMutexId(Mutex *m); + MutexState *getMutex(u32 id); + u32 getMutexId(MutexState *m); DDFlags flags; - Mutex* mutex[kL1Size]; + MutexState *mutex[kL1Size]; SpinMutex mtx; InternalMmapVector<u32> free_id; @@ -152,13 +152,11 @@ void DD::MutexInit(DDCallback *cb, DDMutex *m) { atomic_store(&m->owner, 0, memory_order_relaxed); } -Mutex *DD::getMutex(u32 id) { - return &mutex[id / kL2Size][id % kL2Size]; -} +MutexState *DD::getMutex(u32 id) { return &mutex[id / kL2Size][id % kL2Size]; } -u32 DD::getMutexId(Mutex *m) { +u32 DD::getMutexId(MutexState *m) { for (int i = 0; i < kL1Size; i++) { - Mutex *tab = mutex[i]; + MutexState *tab = mutex[i]; if (tab == 0) break; if (m >= tab && m < tab + kL2Size) @@ -176,8 +174,8 @@ u32 DD::allocateId(DDCallback *cb) { } else { CHECK_LT(id_gen, kMaxMutex); if ((id_gen % kL2Size) == 0) { - mutex[id_gen / kL2Size] = (Mutex*)MmapOrDie(kL2Size * sizeof(Mutex), - "deadlock detector (mutex table)"); + mutex[id_gen / kL2Size] = (MutexState *)MmapOrDie( + kL2Size * sizeof(MutexState), "deadlock detector (mutex table)"); } id = id_gen++; } @@ -216,11 +214,11 @@ void DD::MutexBeforeLock(DDCallback *cb, DDMutex *m, bool wlock) { } bool added = false; - Mutex *mtx = getMutex(m->id); + MutexState *mtx = getMutex(m->id); for (int i = 0; i < lt->nlocked - 1; i++) { u32 id1 = lt->locked[i].id; u32 stk1 = lt->locked[i].stk; - Mutex *mtx1 = getMutex(id1); + MutexState *mtx1 = getMutex(id1); SpinMutexLock l(&mtx1->mtx); if (mtx1->nlink == kMaxLink) { // FIXME(dvyukov): check stale links @@ -342,7 +340,7 @@ void DD::MutexDestroy(DDCallback *cb, DDMutex *m) { // Clear and invalidate the mutex descriptor. { - Mutex *mtx = getMutex(m->id); + MutexState *mtx = getMutex(m->id); SpinMutexLock l(&mtx->mtx); mtx->seq++; mtx->nlink = 0; @@ -361,7 +359,7 @@ void DD::CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt, int npath = 0; int npending = 0; { - Mutex *mtx = getMutex(m->id); + MutexState *mtx = getMutex(m->id); SpinMutexLock l(&mtx->mtx); for (int li = 0; li < mtx->nlink; li++) pt->pending[npending++] = mtx->link[li]; @@ -374,7 +372,7 @@ void DD::CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt, } if (pt->visited[link.id]) continue; - Mutex *mtx1 = getMutex(link.id); + MutexState *mtx1 = getMutex(link.id); SpinMutexLock l(&mtx1->mtx); if (mtx1->seq != link.seq) continue; @@ -387,7 +385,7 @@ void DD::CycleCheck(DDPhysicalThread *pt, DDLogicalThread *lt, return Report(pt, lt, npath); // Bingo! for (int li = 0; li < mtx1->nlink; li++) { Link *link1 = &mtx1->link[li]; - // Mutex *mtx2 = getMutex(link->id); + // MutexState *mtx2 = getMutex(link->id); // FIXME(dvyukov): fast seq check // FIXME(dvyukov): fast nlink != 0 check // FIXME(dvyukov): fast pending check? diff --git a/libsanitizer/sanitizer_common/sanitizer_errno.h b/libsanitizer/sanitizer_common/sanitizer_errno.h index 94f16b6..70a6e88 100644 --- a/libsanitizer/sanitizer_common/sanitizer_errno.h +++ b/libsanitizer/sanitizer_common/sanitizer_errno.h @@ -23,8 +23,7 @@ #if SANITIZER_FREEBSD || SANITIZER_MAC # define __errno_location __error -#elif SANITIZER_ANDROID || SANITIZER_NETBSD || \ - SANITIZER_RTEMS +#elif SANITIZER_ANDROID || SANITIZER_NETBSD # define __errno_location __errno #elif SANITIZER_SOLARIS # define __errno_location ___errno diff --git a/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp b/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp index 4f692f9..65bc398 100644 --- a/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_fuchsia.cpp @@ -36,16 +36,11 @@ uptr internal_sched_yield() { return 0; // Why doesn't this return void? } -static void internal_nanosleep(zx_time_t ns) { - zx_status_t status = _zx_nanosleep(_zx_deadline_after(ns)); +void internal_usleep(u64 useconds) { + zx_status_t status = _zx_nanosleep(_zx_deadline_after(ZX_USEC(useconds))); CHECK_EQ(status, ZX_OK); } -unsigned int internal_sleep(unsigned int seconds) { - internal_nanosleep(ZX_SEC(seconds)); - return 0; -} - u64 NanoTime() { zx_handle_t utc_clock = _zx_utc_reference_get(); CHECK_NE(utc_clock, ZX_HANDLE_INVALID); @@ -78,10 +73,6 @@ void Abort() { abort(); } int Atexit(void (*function)(void)) { return atexit(function); } -void SleepForSeconds(int seconds) { internal_sleep(seconds); } - -void SleepForMillis(int millis) { internal_nanosleep(ZX_MSEC(millis)); } - void GetThreadStackTopAndBottom(bool, uptr *stack_top, uptr *stack_bottom) { pthread_attr_t attr; CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0); @@ -109,6 +100,18 @@ bool SignalContext::IsStackOverflow() const { return false; } void SignalContext::DumpAllRegisters(void *context) { UNIMPLEMENTED(); } const char *SignalContext::Describe() const { UNIMPLEMENTED(); } +void FutexWait(atomic_uint32_t *p, u32 cmp) { + zx_status_t status = _zx_futex_wait(reinterpret_cast<zx_futex_t *>(p), cmp, + ZX_HANDLE_INVALID, ZX_TIME_INFINITE); + if (status != ZX_ERR_BAD_STATE) // Normal race. + CHECK_EQ(status, ZX_OK); +} + +void FutexWake(atomic_uint32_t *p, u32 count) { + zx_status_t status = _zx_futex_wake(reinterpret_cast<zx_futex_t *>(p), count); + CHECK_EQ(status, ZX_OK); +} + enum MutexState : int { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 }; BlockingMutex::BlockingMutex() { @@ -145,8 +148,8 @@ void BlockingMutex::Unlock() { } } -void BlockingMutex::CheckLocked() { - atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_); +void BlockingMutex::CheckLocked() const { + auto m = reinterpret_cast<atomic_uint32_t const *>(&opaque_storage_); CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed)); } @@ -156,8 +159,10 @@ uptr GetMmapGranularity() { return _zx_system_get_page_size(); } sanitizer_shadow_bounds_t ShadowBounds; +void InitShadowBounds() { ShadowBounds = __sanitizer_shadow_bounds(); } + uptr GetMaxUserVirtualAddress() { - ShadowBounds = __sanitizer_shadow_bounds(); + InitShadowBounds(); return ShadowBounds.memory_limit - 1; } diff --git a/libsanitizer/sanitizer_common/sanitizer_fuchsia.h b/libsanitizer/sanitizer_common/sanitizer_fuchsia.h index 96f9cde..26c1dea 100644 --- a/libsanitizer/sanitizer_common/sanitizer_fuchsia.h +++ b/libsanitizer/sanitizer_common/sanitizer_fuchsia.h @@ -30,6 +30,8 @@ struct MemoryMappingLayoutData { size_t current; // Current index into the vector. }; +void InitShadowBounds(); + } // namespace __sanitizer #endif // SANITIZER_FUCHSIA diff --git a/libsanitizer/sanitizer_common/sanitizer_libc.h b/libsanitizer/sanitizer_common/sanitizer_libc.h index ec0a6de..bcb81eb 100644 --- a/libsanitizer/sanitizer_common/sanitizer_libc.h +++ b/libsanitizer/sanitizer_common/sanitizer_libc.h @@ -67,7 +67,8 @@ uptr internal_ftruncate(fd_t fd, uptr size); // OS void NORETURN internal__exit(int exitcode); -unsigned int internal_sleep(unsigned int seconds); +void internal_sleep(unsigned seconds); +void internal_usleep(u64 useconds); uptr internal_getpid(); uptr internal_getppid(); diff --git a/libsanitizer/sanitizer_common/sanitizer_libignore.cpp b/libsanitizer/sanitizer_common/sanitizer_libignore.cpp index a65d3d8..431efc5 100644 --- a/libsanitizer/sanitizer_common/sanitizer_libignore.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_libignore.cpp @@ -84,6 +84,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) { ignored_code_ranges_[idx].begin = range.beg; ignored_code_ranges_[idx].end = range.end; atomic_store(&ignored_ranges_count_, idx + 1, memory_order_release); + atomic_store(&enabled_, 1, memory_order_release); break; } } @@ -114,6 +115,7 @@ void LibIgnore::OnLibraryLoaded(const char *name) { instrumented_code_ranges_[idx].end = range.end; atomic_store(&instrumented_ranges_count_, idx + 1, memory_order_release); + atomic_store(&enabled_, 1, memory_order_release); } } } @@ -123,6 +125,29 @@ void LibIgnore::OnLibraryUnloaded() { OnLibraryLoaded(nullptr); } +bool LibIgnore::IsIgnoredSlow(uptr pc, bool *pc_in_ignored_lib) const { + const uptr n = atomic_load(&ignored_ranges_count_, memory_order_acquire); + for (uptr i = 0; i < n; i++) { + if (IsInRange(pc, ignored_code_ranges_[i])) { + *pc_in_ignored_lib = true; + return true; + } + } + *pc_in_ignored_lib = false; + if (track_instrumented_libs_ && !IsPcInstrumented(pc)) + return true; + return false; +} + +bool LibIgnore::IsPcInstrumented(uptr pc) const { + const uptr n = atomic_load(&instrumented_ranges_count_, memory_order_acquire); + for (uptr i = 0; i < n; i++) { + if (IsInRange(pc, instrumented_code_ranges_[i])) + return true; + } + return false; +} + } // namespace __sanitizer #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || diff --git a/libsanitizer/sanitizer_common/sanitizer_libignore.h b/libsanitizer/sanitizer_common/sanitizer_libignore.h index 256f685..85452e5 100644 --- a/libsanitizer/sanitizer_common/sanitizer_libignore.h +++ b/libsanitizer/sanitizer_common/sanitizer_libignore.h @@ -45,9 +45,6 @@ class LibIgnore { // "pc_in_ignored_lib" if the PC is in an ignored library, false otherwise. bool IsIgnored(uptr pc, bool *pc_in_ignored_lib) const; - // Checks whether the provided PC belongs to an instrumented module. - bool IsPcInstrumented(uptr pc) const; - private: struct Lib { char *templ; @@ -61,6 +58,10 @@ class LibIgnore { uptr end; }; + // Checks whether the provided PC belongs to an instrumented module. + bool IsPcInstrumented(uptr pc) const; + bool IsIgnoredSlow(uptr pc, bool *pc_in_ignored_lib) const; + inline bool IsInRange(uptr pc, const LibCodeRange &range) const { return (pc >= range.begin && pc < range.end); } @@ -70,6 +71,8 @@ class LibIgnore { static const uptr kMaxLibs = 1024; // Hot part: + atomic_uintptr_t enabled_; + atomic_uintptr_t ignored_ranges_count_; LibCodeRange ignored_code_ranges_[kMaxIgnoredRanges]; @@ -87,27 +90,11 @@ class LibIgnore { void operator = (const LibIgnore&); // not implemented }; -inline bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const { - const uptr n = atomic_load(&ignored_ranges_count_, memory_order_acquire); - for (uptr i = 0; i < n; i++) { - if (IsInRange(pc, ignored_code_ranges_[i])) { - *pc_in_ignored_lib = true; - return true; - } - } - *pc_in_ignored_lib = false; - if (track_instrumented_libs_ && !IsPcInstrumented(pc)) - return true; - return false; -} - -inline bool LibIgnore::IsPcInstrumented(uptr pc) const { - const uptr n = atomic_load(&instrumented_ranges_count_, memory_order_acquire); - for (uptr i = 0; i < n; i++) { - if (IsInRange(pc, instrumented_code_ranges_[i])) - return true; - } - return false; +ALWAYS_INLINE +bool LibIgnore::IsIgnored(uptr pc, bool *pc_in_ignored_lib) const { + if (LIKELY(atomic_load(&enabled_, memory_order_acquire) == 0)) + return false; + return IsIgnoredSlow(pc, pc_in_ignored_lib); } } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_linux.cpp b/libsanitizer/sanitizer_common/sanitizer_linux.cpp index b371477..9b7d87e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_linux.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_linux.cpp @@ -430,13 +430,11 @@ uptr internal_sched_yield() { return internal_syscall(SYSCALL(sched_yield)); } -unsigned int internal_sleep(unsigned int seconds) { +void internal_usleep(u64 useconds) { struct timespec ts; - ts.tv_sec = seconds; - ts.tv_nsec = 0; - int res = internal_syscall(SYSCALL(nanosleep), &ts, &ts); - if (res) return ts.tv_sec; - return 0; + ts.tv_sec = useconds / 1000000; + ts.tv_nsec = (useconds % 1000000) * 1000; + internal_syscall(SYSCALL(nanosleep), &ts, &ts); } uptr internal_execve(const char *filename, char *const argv[], @@ -641,11 +639,27 @@ char **GetEnviron() { } #if !SANITIZER_SOLARIS -enum MutexState { - MtxUnlocked = 0, - MtxLocked = 1, - MtxSleeping = 2 -}; +void FutexWait(atomic_uint32_t *p, u32 cmp) { +# if SANITIZER_FREEBSD + _umtx_op(p, UMTX_OP_WAIT_UINT, cmp, 0, 0); +# elif SANITIZER_NETBSD + sched_yield(); /* No userspace futex-like synchronization */ +# else + internal_syscall(SYSCALL(futex), (uptr)p, FUTEX_WAIT_PRIVATE, cmp, 0, 0, 0); +# endif +} + +void FutexWake(atomic_uint32_t *p, u32 count) { +# if SANITIZER_FREEBSD + _umtx_op(p, UMTX_OP_WAKE, count, 0, 0); +# elif SANITIZER_NETBSD + /* No userspace futex-like synchronization */ +# else + internal_syscall(SYSCALL(futex), (uptr)p, FUTEX_WAKE_PRIVATE, count, 0, 0, 0); +# endif +} + +enum { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 }; BlockingMutex::BlockingMutex() { internal_memset(this, 0, sizeof(*this)); @@ -683,11 +697,11 @@ void BlockingMutex::Unlock() { } } -void BlockingMutex::CheckLocked() { - atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_); +void BlockingMutex::CheckLocked() const { + auto m = reinterpret_cast<atomic_uint32_t const *>(&opaque_storage_); CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed)); } -#endif // !SANITIZER_SOLARIS +# endif // !SANITIZER_SOLARIS // ----------------- sanitizer_linux.h // The actual size of this structure is specified by d_reclen. @@ -884,7 +898,7 @@ void internal_sigdelset(__sanitizer_sigset_t *set, int signum) { __sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set; const uptr idx = signum / (sizeof(k_set->sig[0]) * 8); const uptr bit = signum % (sizeof(k_set->sig[0]) * 8); - k_set->sig[idx] &= ~(1 << bit); + k_set->sig[idx] &= ~((uptr)1 << bit); } bool internal_sigismember(__sanitizer_sigset_t *set, int signum) { @@ -894,7 +908,7 @@ bool internal_sigismember(__sanitizer_sigset_t *set, int signum) { __sanitizer_kernel_sigset_t *k_set = (__sanitizer_kernel_sigset_t *)set; const uptr idx = signum / (sizeof(k_set->sig[0]) * 8); const uptr bit = signum % (sizeof(k_set->sig[0]) * 8); - return k_set->sig[idx] & (1 << bit); + return k_set->sig[idx] & ((uptr)1 << bit); } #elif SANITIZER_FREEBSD void internal_sigdelset(__sanitizer_sigset_t *set, int signum) { diff --git a/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp index 4f9577a..7ce9e25 100644 --- a/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_linux_libcdep.cpp @@ -203,7 +203,7 @@ void InitTlsSize() { g_use_dlpi_tls_data = GetLibcVersion(&major, &minor, &patch) && major == 2 && minor >= 25; -#if defined(__x86_64__) || defined(__powerpc64__) +#if defined(__aarch64__) || defined(__x86_64__) || defined(__powerpc64__) void *get_tls_static_info = dlsym(RTLD_NEXT, "_dl_get_tls_static_info"); size_t tls_align; ((void (*)(size_t *, size_t *))get_tls_static_info)(&g_tls_size, &tls_align); @@ -317,21 +317,44 @@ struct TlsBlock { }; } // namespace +#ifdef __s390__ +extern "C" uptr __tls_get_offset(void *arg); + +static uptr TlsGetOffset(uptr ti_module, uptr ti_offset) { + // The __tls_get_offset ABI requires %r12 to point to GOT and %r2 to be an + // offset of a struct tls_index inside GOT. We don't possess either of the + // two, so violate the letter of the "ELF Handling For Thread-Local + // Storage" document and assume that the implementation just dereferences + // %r2 + %r12. + uptr tls_index[2] = {ti_module, ti_offset}; + register uptr r2 asm("2") = 0; + register void *r12 asm("12") = tls_index; + asm("basr %%r14, %[__tls_get_offset]" + : "+r"(r2) + : [__tls_get_offset] "r"(__tls_get_offset), "r"(r12) + : "memory", "cc", "0", "1", "3", "4", "5", "14"); + return r2; +} +#else extern "C" void *__tls_get_addr(size_t *); +#endif static int CollectStaticTlsBlocks(struct dl_phdr_info *info, size_t size, void *data) { if (!info->dlpi_tls_modid) return 0; uptr begin = (uptr)info->dlpi_tls_data; -#ifndef __s390__ if (!g_use_dlpi_tls_data) { // Call __tls_get_addr as a fallback. This forces TLS allocation on glibc // and FreeBSD. +#ifdef __s390__ + begin = (uptr)__builtin_thread_pointer() + + TlsGetOffset(info->dlpi_tls_modid, 0); +#else size_t mod_and_off[2] = {info->dlpi_tls_modid, 0}; begin = (uptr)__tls_get_addr(mod_and_off); - } #endif + } for (unsigned i = 0; i != info->dlpi_phnum; ++i) if (info->dlpi_phdr[i].p_type == PT_TLS) { static_cast<InternalMmapVector<TlsBlock> *>(data)->push_back( @@ -427,12 +450,16 @@ static void GetTls(uptr *addr, uptr *size) { *size = 0; } #elif SANITIZER_GLIBC && defined(__x86_64__) - // For x86-64, use an O(1) approach which requires precise - // ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize. + // For aarch64 and x86-64, use an O(1) approach which requires relatively + // precise ThreadDescriptorSize. g_tls_size was initialized in InitTlsSize. asm("mov %%fs:16,%0" : "=r"(*addr)); *size = g_tls_size; *addr -= *size; *addr += ThreadDescriptorSize(); +#elif SANITIZER_GLIBC && defined(__aarch64__) + *addr = reinterpret_cast<uptr>(__builtin_thread_pointer()) - + ThreadDescriptorSize(); + *size = g_tls_size + ThreadDescriptorSize(); #elif SANITIZER_GLIBC && defined(__powerpc64__) // Workaround for glibc<2.25(?). 2.27 is known to not need this. uptr tp; @@ -732,13 +759,9 @@ u32 GetNumberOfCPUs() { #elif SANITIZER_SOLARIS return sysconf(_SC_NPROCESSORS_ONLN); #else -#if defined(CPU_COUNT) cpu_set_t CPUs; CHECK_EQ(sched_getaffinity(0, sizeof(cpu_set_t), &CPUs), 0); return CPU_COUNT(&CPUs); -#else - return 1; -#endif #endif } diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.cpp b/libsanitizer/sanitizer_common/sanitizer_mac.cpp index 30a94fc..125ecac 100644 --- a/libsanitizer/sanitizer_common/sanitizer_mac.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_mac.cpp @@ -37,7 +37,7 @@ extern char **environ; #endif -#if defined(__has_include) && __has_include(<os/trace.h>) && defined(__BLOCKS__) +#if defined(__has_include) && __has_include(<os/trace.h>) #define SANITIZER_OS_TRACE 1 #include <os/trace.h> #else @@ -70,15 +70,7 @@ extern "C" { #include <mach/mach_time.h> #include <mach/vm_statistics.h> #include <malloc/malloc.h> -#if defined(__has_builtin) && __has_builtin(__builtin_os_log_format) -# include <os/log.h> -#else - /* Without support for __builtin_os_log_format, fall back to the older - method. */ -# define OS_LOG_DEFAULT 0 -# define os_log_error(A,B,C) \ - asl_log(nullptr, nullptr, ASL_LEVEL_ERR, "%s", (C)); -#endif +#include <os/log.h> #include <pthread.h> #include <sched.h> #include <signal.h> @@ -227,9 +219,7 @@ void internal__exit(int exitcode) { _exit(exitcode); } -unsigned int internal_sleep(unsigned int seconds) { - return sleep(seconds); -} +void internal_usleep(u64 useconds) { usleep(useconds); } uptr internal_getpid() { return getpid(); @@ -519,6 +509,13 @@ void MprotectMallocZones(void *addr, int prot) { } } +void FutexWait(atomic_uint32_t *p, u32 cmp) { + // FIXME: implement actual blocking. + sched_yield(); +} + +void FutexWake(atomic_uint32_t *p, u32 count) {} + BlockingMutex::BlockingMutex() { internal_memset(this, 0, sizeof(*this)); } @@ -534,7 +531,7 @@ void BlockingMutex::Unlock() { OSSpinLockUnlock((OSSpinLock*)&opaque_storage_); } -void BlockingMutex::CheckLocked() { +void BlockingMutex::CheckLocked() const { CHECK_NE(*(OSSpinLock*)&opaque_storage_, 0); } diff --git a/libsanitizer/sanitizer_common/sanitizer_mac.h b/libsanitizer/sanitizer_common/sanitizer_mac.h index 96a5986..0b6af5a 100644 --- a/libsanitizer/sanitizer_common/sanitizer_mac.h +++ b/libsanitizer/sanitizer_common/sanitizer_mac.h @@ -14,26 +14,6 @@ #include "sanitizer_common.h" #include "sanitizer_platform.h" - -/* TARGET_OS_OSX is not present in SDKs before Darwin16 (macOS 10.12) use - TARGET_OS_MAC (we have no support for iOS in any form for these versions, - so there's no ambiguity). */ -#if !defined(TARGET_OS_OSX) && TARGET_OS_MAC -# define TARGET_OS_OSX 1 -#endif - -/* Other TARGET_OS_xxx are not present on earlier versions, define them to - 0 (we have no support for them; they are not valid targets anyway). */ -#ifndef TARGET_OS_IOS -#define TARGET_OS_IOS 0 -#endif -#ifndef TARGET_OS_TV -#define TARGET_OS_TV 0 -#endif -#ifndef TARGET_OS_WATCH -#define TARGET_OS_WATCH 0 -#endif - #if SANITIZER_MAC #include "sanitizer_posix.h" diff --git a/libsanitizer/sanitizer_common/sanitizer_mutex.cpp b/libsanitizer/sanitizer_common/sanitizer_mutex.cpp new file mode 100644 index 0000000..bc2d83c --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_mutex.cpp @@ -0,0 +1,39 @@ +//===-- sanitizer_mutex.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is shared between AddressSanitizer and ThreadSanitizer +// run-time libraries. +//===----------------------------------------------------------------------===// + +#include "sanitizer_mutex.h" + +#include "sanitizer_common.h" + +namespace __sanitizer { + +void Semaphore::Wait() { + u32 count = atomic_load(&state_, memory_order_relaxed); + for (;;) { + if (count == 0) { + FutexWait(&state_, 0); + count = atomic_load(&state_, memory_order_relaxed); + continue; + } + if (atomic_compare_exchange_weak(&state_, &count, count - 1, + memory_order_acquire)) + break; + } +} + +void Semaphore::Post(u32 count) { + CHECK_NE(count, 0); + atomic_fetch_add(&state_, count, memory_order_release); + FutexWake(&state_, count); +} + +} // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_mutex.h b/libsanitizer/sanitizer_common/sanitizer_mutex.h index 40a6591..e3ff650 100644 --- a/libsanitizer/sanitizer_common/sanitizer_mutex.h +++ b/libsanitizer/sanitizer_common/sanitizer_mutex.h @@ -16,30 +16,29 @@ #include "sanitizer_atomic.h" #include "sanitizer_internal_defs.h" #include "sanitizer_libc.h" +#include "sanitizer_thread_safety.h" namespace __sanitizer { -class StaticSpinMutex { +class MUTEX StaticSpinMutex { public: void Init() { atomic_store(&state_, 0, memory_order_relaxed); } - void Lock() { + void Lock() ACQUIRE() { if (TryLock()) return; LockSlow(); } - bool TryLock() { + bool TryLock() TRY_ACQUIRE(true) { return atomic_exchange(&state_, 1, memory_order_acquire) == 0; } - void Unlock() { - atomic_store(&state_, 0, memory_order_release); - } + void Unlock() RELEASE() { atomic_store(&state_, 0, memory_order_release); } - void CheckLocked() { + void CheckLocked() const CHECK_LOCKED() { CHECK_EQ(atomic_load(&state_, memory_order_relaxed), 1); } @@ -59,24 +58,223 @@ class StaticSpinMutex { } }; -class SpinMutex : public StaticSpinMutex { +class MUTEX SpinMutex : public StaticSpinMutex { public: SpinMutex() { Init(); } private: - SpinMutex(const SpinMutex&); - void operator=(const SpinMutex&); + SpinMutex(const SpinMutex &) = delete; + void operator=(const SpinMutex &) = delete; +}; + +// Semaphore provides an OS-dependent way to park/unpark threads. +// The last thread returned from Wait can destroy the object +// (destruction-safety). +class Semaphore { + public: + constexpr Semaphore() {} + Semaphore(const Semaphore &) = delete; + void operator=(const Semaphore &) = delete; + + void Wait(); + void Post(u32 count = 1); + + private: + atomic_uint32_t state_ = {0}; +}; + +// Reader-writer mutex. +class MUTEX Mutex2 { + public: + constexpr Mutex2() {} + + void Lock() ACQUIRE() { + u64 reset_mask = ~0ull; + u64 state = atomic_load_relaxed(&state_); + const uptr kMaxSpinIters = 1500; + for (uptr spin_iters = 0;; spin_iters++) { + u64 new_state; + bool locked = (state & (kWriterLock | kReaderLockMask)) != 0; + if (LIKELY(!locked)) { + // The mutex is not read-/write-locked, try to lock. + new_state = (state | kWriterLock) & reset_mask; + } else if (spin_iters > kMaxSpinIters) { + // We've spun enough, increment waiting writers count and block. + // The counter will be decremented by whoever wakes us. + new_state = (state + kWaitingWriterInc) & reset_mask; + } else if ((state & kWriterSpinWait) == 0) { + // Active spinning, but denote our presence so that unlocking + // thread does not wake up other threads. + new_state = state | kWriterSpinWait; + } else { + // Active spinning. + state = atomic_load(&state_, memory_order_relaxed); + continue; + } + if (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state, + memory_order_acquire))) + continue; + if (LIKELY(!locked)) + return; // We've locked the mutex. + if (spin_iters > kMaxSpinIters) { + // We've incremented waiting writers, so now block. + writers_.Wait(); + spin_iters = 0; + state = atomic_load(&state_, memory_order_relaxed); + DCHECK_NE(state & kWriterSpinWait, 0); + } else { + // We've set kWriterSpinWait, but we are still in active spinning. + } + // We either blocked and were unblocked, + // or we just spun but set kWriterSpinWait. + // Either way we need to reset kWriterSpinWait + // next time we take the lock or block again. + reset_mask = ~kWriterSpinWait; + } + } + + void Unlock() RELEASE() { + bool wake_writer; + u64 wake_readers; + u64 new_state; + u64 state = atomic_load_relaxed(&state_); + do { + DCHECK_NE(state & kWriterLock, 0); + DCHECK_EQ(state & kReaderLockMask, 0); + new_state = state & ~kWriterLock; + wake_writer = + (state & kWriterSpinWait) == 0 && (state & kWaitingWriterMask) != 0; + if (wake_writer) + new_state = (new_state - kWaitingWriterInc) | kWriterSpinWait; + wake_readers = + (state & (kWriterSpinWait | kWaitingWriterMask)) != 0 + ? 0 + : ((state & kWaitingReaderMask) >> kWaitingReaderShift); + if (wake_readers) + new_state = (new_state & ~kWaitingReaderMask) + + (wake_readers << kReaderLockShift); + } while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state, + memory_order_release))); + if (UNLIKELY(wake_writer)) + writers_.Post(); + else if (UNLIKELY(wake_readers)) + readers_.Post(wake_readers); + } + + void ReadLock() ACQUIRE_SHARED() { + bool locked; + u64 new_state; + u64 state = atomic_load_relaxed(&state_); + do { + locked = + (state & kReaderLockMask) == 0 && + (state & (kWriterLock | kWriterSpinWait | kWaitingWriterMask)) != 0; + if (LIKELY(!locked)) + new_state = state + kReaderLockInc; + else + new_state = state + kWaitingReaderInc; + } while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state, + memory_order_acquire))); + if (UNLIKELY(locked)) + readers_.Wait(); + DCHECK_EQ(atomic_load_relaxed(&state_) & kWriterLock, 0); + DCHECK_NE(atomic_load_relaxed(&state_) & kReaderLockMask, 0); + } + + void ReadUnlock() RELEASE_SHARED() { + bool wake; + u64 new_state; + u64 state = atomic_load_relaxed(&state_); + do { + DCHECK_NE(state & kReaderLockMask, 0); + DCHECK_EQ(state & (kWaitingReaderMask | kWriterLock), 0); + new_state = state - kReaderLockInc; + wake = (new_state & (kReaderLockMask | kWriterSpinWait)) == 0 && + (new_state & kWaitingWriterMask) != 0; + if (wake) + new_state = (new_state - kWaitingWriterInc) | kWriterSpinWait; + } while (UNLIKELY(!atomic_compare_exchange_weak(&state_, &state, new_state, + memory_order_release))); + if (UNLIKELY(wake)) + writers_.Post(); + } + + // This function does not guarantee an explicit check that the calling thread + // is the thread which owns the mutex. This behavior, while more strictly + // correct, causes problems in cases like StopTheWorld, where a parent thread + // owns the mutex but a child checks that it is locked. Rather than + // maintaining complex state to work around those situations, the check only + // checks that the mutex is owned. + void CheckWriteLocked() const CHECK_LOCKED() { + CHECK(atomic_load(&state_, memory_order_relaxed) & kWriterLock); + } + + void CheckLocked() const CHECK_LOCKED() { CheckWriteLocked(); } + + void CheckReadLocked() const CHECK_LOCKED() { + CHECK(atomic_load(&state_, memory_order_relaxed) & kReaderLockMask); + } + + private: + atomic_uint64_t state_ = {0}; + Semaphore writers_; + Semaphore readers_; + + // The state has 3 counters: + // - number of readers holding the lock, + // if non zero, the mutex is read-locked + // - number of waiting readers, + // if not zero, the mutex is write-locked + // - number of waiting writers, + // if non zero, the mutex is read- or write-locked + // And 2 flags: + // - writer lock + // if set, the mutex is write-locked + // - a writer is awake and spin-waiting + // the flag is used to prevent thundering herd problem + // (new writers are not woken if this flag is set) + // + // Writer support active spinning, readers does not. + // But readers are more aggressive and always take the mutex + // if there are any other readers. + // Writers hand off the mutex to readers: after wake up readers + // already assume ownership of the mutex (don't need to do any + // state updates). But the mutex is not handed off to writers, + // after wake up writers compete to lock the mutex again. + // This is needed to allow repeated write locks even in presence + // of other blocked writers. + static constexpr u64 kCounterWidth = 20; + static constexpr u64 kReaderLockShift = 0; + static constexpr u64 kReaderLockInc = 1ull << kReaderLockShift; + static constexpr u64 kReaderLockMask = ((1ull << kCounterWidth) - 1) + << kReaderLockShift; + static constexpr u64 kWaitingReaderShift = kCounterWidth; + static constexpr u64 kWaitingReaderInc = 1ull << kWaitingReaderShift; + static constexpr u64 kWaitingReaderMask = ((1ull << kCounterWidth) - 1) + << kWaitingReaderShift; + static constexpr u64 kWaitingWriterShift = 2 * kCounterWidth; + static constexpr u64 kWaitingWriterInc = 1ull << kWaitingWriterShift; + static constexpr u64 kWaitingWriterMask = ((1ull << kCounterWidth) - 1) + << kWaitingWriterShift; + static constexpr u64 kWriterLock = 1ull << (3 * kCounterWidth); + static constexpr u64 kWriterSpinWait = 1ull << (3 * kCounterWidth + 1); + + Mutex2(const Mutex2 &) = delete; + void operator=(const Mutex2 &) = delete; }; -class BlockingMutex { +void FutexWait(atomic_uint32_t *p, u32 cmp); +void FutexWake(atomic_uint32_t *p, u32 count); + +class MUTEX BlockingMutex { public: explicit constexpr BlockingMutex(LinkerInitialized) : opaque_storage_ {0, }, owner_ {0} {} BlockingMutex(); - void Lock(); - void Unlock(); + void Lock() ACQUIRE(); + void Unlock() RELEASE(); // This function does not guarantee an explicit check that the calling thread // is the thread which owns the mutex. This behavior, while more strictly @@ -85,7 +283,7 @@ class BlockingMutex { // maintaining complex state to work around those situations, the check only // checks that the mutex is owned, and assumes callers to be generally // well-behaved. - void CheckLocked(); + void CheckLocked() const CHECK_LOCKED(); private: // Solaris mutex_t has a member that requires 64-bit alignment. @@ -94,7 +292,7 @@ class BlockingMutex { }; // Reader-writer spin mutex. -class RWMutex { +class MUTEX RWMutex { public: RWMutex() { atomic_store(&state_, kUnlocked, memory_order_relaxed); @@ -104,7 +302,7 @@ class RWMutex { CHECK_EQ(atomic_load(&state_, memory_order_relaxed), kUnlocked); } - void Lock() { + void Lock() ACQUIRE() { u32 cmp = kUnlocked; if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock, memory_order_acquire)) @@ -112,27 +310,27 @@ class RWMutex { LockSlow(); } - void Unlock() { + void Unlock() RELEASE() { u32 prev = atomic_fetch_sub(&state_, kWriteLock, memory_order_release); DCHECK_NE(prev & kWriteLock, 0); (void)prev; } - void ReadLock() { + void ReadLock() ACQUIRE_SHARED() { u32 prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire); if ((prev & kWriteLock) == 0) return; ReadLockSlow(); } - void ReadUnlock() { + void ReadUnlock() RELEASE_SHARED() { u32 prev = atomic_fetch_sub(&state_, kReadLock, memory_order_release); DCHECK_EQ(prev & kWriteLock, 0); DCHECK_GT(prev & ~kWriteLock, 0); (void)prev; } - void CheckLocked() { + void CheckLocked() const CHECK_LOCKED() { CHECK_NE(atomic_load(&state_, memory_order_relaxed), kUnlocked); } @@ -171,46 +369,40 @@ class RWMutex { } } - RWMutex(const RWMutex&); - void operator = (const RWMutex&); + RWMutex(const RWMutex &) = delete; + void operator=(const RWMutex &) = delete; }; -template<typename MutexType> -class GenericScopedLock { +template <typename MutexType> +class SCOPED_LOCK GenericScopedLock { public: - explicit GenericScopedLock(MutexType *mu) - : mu_(mu) { + explicit GenericScopedLock(MutexType *mu) ACQUIRE(mu) : mu_(mu) { mu_->Lock(); } - ~GenericScopedLock() { - mu_->Unlock(); - } + ~GenericScopedLock() RELEASE() { mu_->Unlock(); } private: MutexType *mu_; - GenericScopedLock(const GenericScopedLock&); - void operator=(const GenericScopedLock&); + GenericScopedLock(const GenericScopedLock &) = delete; + void operator=(const GenericScopedLock &) = delete; }; -template<typename MutexType> -class GenericScopedReadLock { +template <typename MutexType> +class SCOPED_LOCK GenericScopedReadLock { public: - explicit GenericScopedReadLock(MutexType *mu) - : mu_(mu) { + explicit GenericScopedReadLock(MutexType *mu) ACQUIRE(mu) : mu_(mu) { mu_->ReadLock(); } - ~GenericScopedReadLock() { - mu_->ReadUnlock(); - } + ~GenericScopedReadLock() RELEASE() { mu_->ReadUnlock(); } private: MutexType *mu_; - GenericScopedReadLock(const GenericScopedReadLock&); - void operator=(const GenericScopedReadLock&); + GenericScopedReadLock(const GenericScopedReadLock &) = delete; + void operator=(const GenericScopedReadLock &) = delete; }; typedef GenericScopedLock<StaticSpinMutex> SpinMutexLock; diff --git a/libsanitizer/sanitizer_common/sanitizer_netbsd.cpp b/libsanitizer/sanitizer_common/sanitizer_netbsd.cpp index ac20f91..5e601bd 100644 --- a/libsanitizer/sanitizer_common/sanitizer_netbsd.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_netbsd.cpp @@ -215,15 +215,12 @@ void internal__exit(int exitcode) { Die(); // Unreachable. } -unsigned int internal_sleep(unsigned int seconds) { +void internal_usleep(u64 useconds) { struct timespec ts; - ts.tv_sec = seconds; - ts.tv_nsec = 0; + ts.tv_sec = useconds / 1000000; + ts.tv_nsec = (useconds % 1000000) * 1000; CHECK(&_sys___nanosleep50); - int res = _sys___nanosleep50(&ts, &ts); - if (res) - return ts.tv_sec; - return 0; + _sys___nanosleep50(&ts, &ts); } uptr internal_execve(const char *filename, char *const argv[], diff --git a/libsanitizer/sanitizer_common/sanitizer_platform.h b/libsanitizer/sanitizer_common/sanitizer_platform.h index 2f64584..4d3c088 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform.h @@ -13,10 +13,9 @@ #define SANITIZER_PLATFORM_H #if !defined(__linux__) && !defined(__FreeBSD__) && !defined(__NetBSD__) && \ - !defined(__APPLE__) && !defined(_WIN32) && \ - !defined(__Fuchsia__) && !defined(__rtems__) && \ - !(defined(__sun__) && defined(__svr4__)) -# error "This operating system is not supported" + !defined(__APPLE__) && !defined(_WIN32) && !defined(__Fuchsia__) && \ + !(defined(__sun__) && defined(__svr4__)) +# error "This operating system is not supported" #endif // Get __GLIBC__ on a glibc platform. Exclude Android: features.h includes C @@ -117,12 +116,6 @@ # define SANITIZER_FUCHSIA 0 #endif -#if defined(__rtems__) -# define SANITIZER_RTEMS 1 -#else -# define SANITIZER_RTEMS 0 -#endif - #define SANITIZER_POSIX \ (SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_MAC || \ SANITIZER_NETBSD || SANITIZER_SOLARIS) @@ -226,12 +219,6 @@ # define SANITIZER_SOLARIS32 0 #endif -#if defined(__myriad2__) -# define SANITIZER_MYRIAD2 1 -#else -# define SANITIZER_MYRIAD2 0 -#endif - #if defined(__riscv) && (__riscv_xlen == 64) #define SANITIZER_RISCV64 1 #else @@ -374,9 +361,9 @@ # define SANITIZER_CACHE_LINE_SIZE 64 #endif -// Enable offline markup symbolizer for Fuchsia and RTEMS. -#if SANITIZER_FUCHSIA || SANITIZER_RTEMS -#define SANITIZER_SYMBOLIZER_MARKUP 1 +// Enable offline markup symbolizer for Fuchsia. +#if SANITIZER_FUCHSIA +# define SANITIZER_SYMBOLIZER_MARKUP 1 #else #define SANITIZER_SYMBOLIZER_MARKUP 0 #endif diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h index 731df71..5b710c2 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform_interceptors.h @@ -114,12 +114,6 @@ #define SI_NOT_FUCHSIA 1 #endif -#if SANITIZER_RTEMS -#define SI_NOT_RTEMS 0 -#else -#define SI_NOT_RTEMS 1 -#endif - #if SANITIZER_SOLARIS #define SI_SOLARIS 1 #else @@ -482,13 +476,12 @@ #define SANITIZER_INTERCEPT_MMAP SI_POSIX #define SANITIZER_INTERCEPT_MMAP64 SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_MALLOPT_AND_MALLINFO (SI_GLIBC || SI_ANDROID) -#define SANITIZER_INTERCEPT_MEMALIGN \ - (!SI_FREEBSD && !SI_MAC && !SI_NETBSD && SI_NOT_RTEMS) +#define SANITIZER_INTERCEPT_MEMALIGN (!SI_FREEBSD && !SI_MAC && !SI_NETBSD) #define SANITIZER_INTERCEPT___LIBC_MEMALIGN SI_GLIBC #define SANITIZER_INTERCEPT_PVALLOC (SI_GLIBC || SI_ANDROID) #define SANITIZER_INTERCEPT_CFREE (SI_GLIBC && !SANITIZER_RISCV64) #define SANITIZER_INTERCEPT_REALLOCARRAY SI_POSIX -#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC && SI_NOT_RTEMS) +#define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC) #define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD) #define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID #define SANITIZER_INTERCEPT_WCSCAT SI_POSIX @@ -584,6 +577,7 @@ (SI_POSIX && !(SANITIZER_MAC && SANITIZER_I386)) #define SANITIZER_INTERCEPT_UNAME (SI_POSIX && !SI_FREEBSD) #define SANITIZER_INTERCEPT___XUNAME SI_FREEBSD +#define SANITIZER_INTERCEPT_FLOPEN SI_FREEBSD // This macro gives a way for downstream users to override the above // interceptor macros irrespective of the platform they are on. They have diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp b/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp index f22f503..c51327e 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_linux.cpp @@ -26,12 +26,9 @@ // With old kernels (and even new kernels on powerpc) asm/stat.h uses types that // are not defined anywhere in userspace headers. Fake them. This seems to work -// fine with newer headers, too. Beware that with <sys/stat.h>, struct stat -// takes the form of struct stat64 on 32-bit platforms if _FILE_OFFSET_BITS=64. -// Also, for some platforms (e.g. mips) there are additional members in the -// <sys/stat.h> struct stat:s. +// fine with newer headers, too. #include <linux/posix_types.h> -#if defined(__x86_64__) +#if defined(__x86_64__) || defined(__mips__) #include <sys/stat.h> #else #define ino_t __kernel_ino_t diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h index cba41ba..4dd2764 100644 --- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h @@ -83,7 +83,7 @@ const unsigned struct_kernel_stat64_sz = 104; #elif defined(__mips__) const unsigned struct_kernel_stat_sz = SANITIZER_ANDROID ? FIRST_32_SECOND_64(104, 128) - : FIRST_32_SECOND_64(144, 216); + : FIRST_32_SECOND_64(160, 216); const unsigned struct_kernel_stat64_sz = 104; #elif defined(__s390__) && !defined(__s390x__) const unsigned struct_kernel_stat_sz = 64; @@ -650,14 +650,14 @@ struct __sanitizer_sigaction { #endif // !SANITIZER_ANDROID #if defined(__mips__) -struct __sanitizer_kernel_sigset_t { - uptr sig[2]; -}; +#define __SANITIZER_KERNEL_NSIG 128 #else +#define __SANITIZER_KERNEL_NSIG 64 +#endif + struct __sanitizer_kernel_sigset_t { - u8 sig[8]; + uptr sig[__SANITIZER_KERNEL_NSIG / (sizeof(uptr) * 8)]; }; -#endif // Linux system headers define the 'sa_handler' and 'sa_sigaction' macros. #if SANITIZER_MIPS diff --git a/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp index d1d8e50..ddf6844 100644 --- a/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_posix_libcdep.cpp @@ -128,14 +128,6 @@ void SetAddressSpaceUnlimited() { CHECK(AddressSpaceIsUnlimited()); } -void SleepForSeconds(int seconds) { - sleep(seconds); -} - -void SleepForMillis(int millis) { - usleep(millis * 1000); -} - void Abort() { #if !SANITIZER_GO // If we are handling SIGABRT, unhandle it first. @@ -166,9 +158,10 @@ bool SupportsColoredOutput(fd_t fd) { #if !SANITIZER_GO // TODO(glider): different tools may require different altstack size. static uptr GetAltStackSize() { - // SIGSTKSZ is not enough. - static const uptr kAltStackSize = SIGSTKSZ * 4; - return kAltStackSize; + // Note: since GLIBC_2.31, SIGSTKSZ may be a function call, so this may be + // more costly that you think. However GetAltStackSize is only call 2-3 times + // per thread so don't cache the evaluation. + return SIGSTKSZ * 4; } void SetAlternateSignalStack() { diff --git a/libsanitizer/sanitizer_common/sanitizer_printf.cpp b/libsanitizer/sanitizer_common/sanitizer_printf.cpp index 5d16dfd..b913c92 100644 --- a/libsanitizer/sanitizer_common/sanitizer_printf.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_printf.cpp @@ -20,6 +20,10 @@ #include <stdio.h> #include <stdarg.h> +#if defined(__x86_64__) +# include <emmintrin.h> +#endif + #if SANITIZER_WINDOWS && defined(_MSC_VER) && _MSC_VER < 1800 && \ !defined(va_copy) # define va_copy(dst, src) ((dst) = (src)) @@ -128,7 +132,7 @@ static int AppendPointer(char **buff, const char *buff_end, u64 ptr_value) { int VSNPrintf(char *buff, int buff_length, const char *format, va_list args) { static const char *kPrintfFormatsHelp = - "Supported Printf formats: %([0-9]*)?(z|ll)?{d,u,x,X}; %p; " + "Supported Printf formats: %([0-9]*)?(z|ll)?{d,u,x,X,V}; %p; " "%[-]([0-9]*)?(\\.\\*)?s; %c\n"; RAW_CHECK(format); RAW_CHECK(buff_length > 0); @@ -162,17 +166,15 @@ int VSNPrintf(char *buff, int buff_length, cur += have_z; bool have_ll = !have_z && (cur[0] == 'l' && cur[1] == 'l'); cur += have_ll * 2; - s64 dval; - u64 uval; const bool have_length = have_z || have_ll; const bool have_flags = have_width || have_length; // At the moment only %s supports precision and left-justification. CHECK(!((precision >= 0 || left_justified) && *cur != 's')); switch (*cur) { case 'd': { - dval = have_ll ? va_arg(args, s64) - : have_z ? va_arg(args, sptr) - : va_arg(args, int); + s64 dval = have_ll ? va_arg(args, s64) + : have_z ? va_arg(args, sptr) + : va_arg(args, int); result += AppendSignedDecimal(&buff, buff_end, dval, width, pad_with_zero); break; @@ -180,14 +182,21 @@ int VSNPrintf(char *buff, int buff_length, case 'u': case 'x': case 'X': { - uval = have_ll ? va_arg(args, u64) - : have_z ? va_arg(args, uptr) - : va_arg(args, unsigned); + u64 uval = have_ll ? va_arg(args, u64) + : have_z ? va_arg(args, uptr) + : va_arg(args, unsigned); bool uppercase = (*cur == 'X'); result += AppendUnsigned(&buff, buff_end, uval, (*cur == 'u') ? 10 : 16, width, pad_with_zero, uppercase); break; } + case 'V': { + for (uptr i = 0; i < 16; i++) { + unsigned x = va_arg(args, unsigned); + result += AppendUnsigned(&buff, buff_end, x, 16, 2, true, false); + } + break; + } case 'p': { RAW_CHECK_MSG(!have_flags, kPrintfFormatsHelp); result += AppendPointer(&buff, buff_end, va_arg(args, uptr)); diff --git a/libsanitizer/sanitizer_common/sanitizer_quarantine.h b/libsanitizer/sanitizer_common/sanitizer_quarantine.h index 992f231..1a074d2 100644 --- a/libsanitizer/sanitizer_common/sanitizer_quarantine.h +++ b/libsanitizer/sanitizer_common/sanitizer_quarantine.h @@ -149,7 +149,8 @@ class Quarantine { Cache cache_; char pad2_[kCacheLineSize]; - void NOINLINE Recycle(uptr min_size, Callback cb) { + void NOINLINE Recycle(uptr min_size, Callback cb) REQUIRES(recycle_mutex_) + RELEASE(recycle_mutex_) { Cache tmp; { SpinMutexLock l(&cache_mutex_); diff --git a/libsanitizer/sanitizer_common/sanitizer_rtems.cpp b/libsanitizer/sanitizer_common/sanitizer_rtems.cpp deleted file mode 100644 index d58bd08..0000000 --- a/libsanitizer/sanitizer_common/sanitizer_rtems.cpp +++ /dev/null @@ -1,281 +0,0 @@ -//===-- sanitizer_rtems.cpp -----------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is shared between various sanitizers' runtime libraries and -// implements RTEMS-specific functions. -//===----------------------------------------------------------------------===// - -#include "sanitizer_rtems.h" -#if SANITIZER_RTEMS - -#define posix_memalign __real_posix_memalign -#define free __real_free -#define memset __real_memset - -#include "sanitizer_file.h" -#include "sanitizer_symbolizer.h" -#include <errno.h> -#include <fcntl.h> -#include <pthread.h> -#include <sched.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> - -// There is no mmap on RTEMS. Use memalign, etc. -#define __mmap_alloc_aligned posix_memalign -#define __mmap_free free -#define __mmap_memset memset - -namespace __sanitizer { - -#include "sanitizer_syscall_generic.inc" - -void NORETURN internal__exit(int exitcode) { - _exit(exitcode); -} - -uptr internal_sched_yield() { - return sched_yield(); -} - -uptr internal_getpid() { - return getpid(); -} - -int internal_dlinfo(void *handle, int request, void *p) { - UNIMPLEMENTED(); -} - -bool FileExists(const char *filename) { - struct stat st; - if (stat(filename, &st)) - return false; - // Sanity check: filename is a regular file. - return S_ISREG(st.st_mode); -} - -uptr GetThreadSelf() { return static_cast<uptr>(pthread_self()); } - -tid_t GetTid() { return GetThreadSelf(); } - -void Abort() { abort(); } - -int Atexit(void (*function)(void)) { return atexit(function); } - -void SleepForSeconds(int seconds) { sleep(seconds); } - -void SleepForMillis(int millis) { usleep(millis * 1000); } - -bool SupportsColoredOutput(fd_t fd) { return false; } - -void GetThreadStackTopAndBottom(bool at_initialization, - uptr *stack_top, uptr *stack_bottom) { - pthread_attr_t attr; - pthread_attr_init(&attr); - CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0); - void *base = nullptr; - size_t size = 0; - CHECK_EQ(pthread_attr_getstack(&attr, &base, &size), 0); - CHECK_EQ(pthread_attr_destroy(&attr), 0); - - *stack_bottom = reinterpret_cast<uptr>(base); - *stack_top = *stack_bottom + size; -} - -void GetThreadStackAndTls(bool main, uptr *stk_addr, uptr *stk_size, - uptr *tls_addr, uptr *tls_size) { - uptr stack_top, stack_bottom; - GetThreadStackTopAndBottom(main, &stack_top, &stack_bottom); - *stk_addr = stack_bottom; - *stk_size = stack_top - stack_bottom; - *tls_addr = *tls_size = 0; -} - -void InitializePlatformEarly() {} -void MaybeReexec() {} -void CheckASLR() {} -void CheckMPROTECT() {} -void DisableCoreDumperIfNecessary() {} -void InstallDeadlySignalHandlers(SignalHandlerType handler) {} -void SetAlternateSignalStack() {} -void UnsetAlternateSignalStack() {} -void InitTlsSize() {} - -void SignalContext::DumpAllRegisters(void *context) {} -const char *DescribeSignalOrException(int signo) { UNIMPLEMENTED(); } - -enum MutexState { MtxUnlocked = 0, MtxLocked = 1, MtxSleeping = 2 }; - -BlockingMutex::BlockingMutex() { - internal_memset(this, 0, sizeof(*this)); -} - -void BlockingMutex::Lock() { - CHECK_EQ(owner_, 0); - atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_); - if (atomic_exchange(m, MtxLocked, memory_order_acquire) == MtxUnlocked) - return; - while (atomic_exchange(m, MtxSleeping, memory_order_acquire) != MtxUnlocked) { - internal_sched_yield(); - } -} - -void BlockingMutex::Unlock() { - atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_); - u32 v = atomic_exchange(m, MtxUnlocked, memory_order_release); - CHECK_NE(v, MtxUnlocked); -} - -void BlockingMutex::CheckLocked() { - atomic_uint32_t *m = reinterpret_cast<atomic_uint32_t *>(&opaque_storage_); - CHECK_NE(MtxUnlocked, atomic_load(m, memory_order_relaxed)); -} - -uptr GetPageSize() { return getpagesize(); } - -uptr GetMmapGranularity() { return GetPageSize(); } - -uptr GetMaxVirtualAddress() { - return (1ULL << 32) - 1; // 0xffffffff -} - -void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { - void* ptr = 0; - int res = __mmap_alloc_aligned(&ptr, GetPageSize(), size); - if (UNLIKELY(res)) - ReportMmapFailureAndDie(size, mem_type, "allocate", res, raw_report); - __mmap_memset(ptr, 0, size); - IncreaseTotalMmap(size); - return ptr; -} - -void *MmapOrDieOnFatalError(uptr size, const char *mem_type) { - void* ptr = 0; - int res = __mmap_alloc_aligned(&ptr, GetPageSize(), size); - if (UNLIKELY(res)) { - if (res == ENOMEM) - return nullptr; - ReportMmapFailureAndDie(size, mem_type, "allocate", false); - } - __mmap_memset(ptr, 0, size); - IncreaseTotalMmap(size); - return ptr; -} - -void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, - const char *mem_type) { - CHECK(IsPowerOfTwo(size)); - CHECK(IsPowerOfTwo(alignment)); - void* ptr = 0; - int res = __mmap_alloc_aligned(&ptr, alignment, size); - if (res) - ReportMmapFailureAndDie(size, mem_type, "align allocate", res, false); - __mmap_memset(ptr, 0, size); - IncreaseTotalMmap(size); - return ptr; -} - -void *MmapNoReserveOrDie(uptr size, const char *mem_type) { - return MmapOrDie(size, mem_type, false); -} - -void UnmapOrDie(void *addr, uptr size) { - if (!addr || !size) return; - __mmap_free(addr); - DecreaseTotalMmap(size); -} - -fd_t OpenFile(const char *filename, FileAccessMode mode, error_t *errno_p) { - int flags; - switch (mode) { - case RdOnly: flags = O_RDONLY; break; - case WrOnly: flags = O_WRONLY | O_CREAT | O_TRUNC; break; - case RdWr: flags = O_RDWR | O_CREAT; break; - } - fd_t res = open(filename, flags, 0660); - if (internal_iserror(res, errno_p)) - return kInvalidFd; - return res; -} - -void CloseFile(fd_t fd) { - close(fd); -} - -bool ReadFromFile(fd_t fd, void *buff, uptr buff_size, uptr *bytes_read, - error_t *error_p) { - uptr res = read(fd, buff, buff_size); - if (internal_iserror(res, error_p)) - return false; - if (bytes_read) - *bytes_read = res; - return true; -} - -bool WriteToFile(fd_t fd, const void *buff, uptr buff_size, uptr *bytes_written, - error_t *error_p) { - uptr res = write(fd, buff, buff_size); - if (internal_iserror(res, error_p)) - return false; - if (bytes_written) - *bytes_written = res; - return true; -} - -void ReleaseMemoryPagesToOS(uptr beg, uptr end) {} -void DumpProcessMap() {} - -// There is no page protection so everything is "accessible." -bool IsAccessibleMemoryRange(uptr beg, uptr size) { - return true; -} - -char **GetArgv() { return nullptr; } -char **GetEnviron() { return nullptr; } - -const char *GetEnv(const char *name) { - return getenv(name); -} - -uptr ReadBinaryName(/*out*/char *buf, uptr buf_len) { - internal_strncpy(buf, "StubBinaryName", buf_len); - return internal_strlen(buf); -} - -uptr ReadLongProcessName(/*out*/ char *buf, uptr buf_len) { - internal_strncpy(buf, "StubProcessName", buf_len); - return internal_strlen(buf); -} - -bool IsPathSeparator(const char c) { - return c == '/'; -} - -bool IsAbsolutePath(const char *path) { - return path != nullptr && IsPathSeparator(path[0]); -} - -void ReportFile::Write(const char *buffer, uptr length) { - SpinMutexLock l(mu); - static const char *kWriteError = - "ReportFile::Write() can't output requested buffer!\n"; - ReopenIfNecessary(); - if (length != write(fd, buffer, length)) { - write(fd, kWriteError, internal_strlen(kWriteError)); - Die(); - } -} - -uptr MainThreadStackBase, MainThreadStackSize; -uptr MainThreadTlsBase, MainThreadTlsSize; - -} // namespace __sanitizer - -#endif // SANITIZER_RTEMS diff --git a/libsanitizer/sanitizer_common/sanitizer_rtems.h b/libsanitizer/sanitizer_common/sanitizer_rtems.h deleted file mode 100644 index e8adfd5..0000000 --- a/libsanitizer/sanitizer_common/sanitizer_rtems.h +++ /dev/null @@ -1,20 +0,0 @@ -//===-- sanitizer_rtems.h ---------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is shared between various sanitizers' runtime libraries and -// provides definitions for RTEMS-specific functions. -//===----------------------------------------------------------------------===// -#ifndef SANITIZER_RTEMS_H -#define SANITIZER_RTEMS_H - -#include "sanitizer_platform.h" -#if SANITIZER_RTEMS -#include "sanitizer_common.h" - -#endif // SANITIZER_RTEMS -#endif // SANITIZER_RTEMS_H diff --git a/libsanitizer/sanitizer_common/sanitizer_solaris.cpp b/libsanitizer/sanitizer_common/sanitizer_solaris.cpp index 8789dcd..cb53eab 100644 --- a/libsanitizer/sanitizer_common/sanitizer_solaris.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_solaris.cpp @@ -160,6 +160,13 @@ DECLARE__REAL_AND_INTERNAL(uptr, sched_yield, void) { return sched_yield(); } +DECLARE__REAL_AND_INTERNAL(void, usleep, u64 useconds) { + struct timespec ts; + ts.tv_sec = useconds / 1000000; + ts.tv_nsec = (useconds % 1000000) * 1000; + nanosleep(&ts, nullptr); +} + DECLARE__REAL_AND_INTERNAL(uptr, execve, const char *filename, char *const argv[], char *const envp[]) { return _REAL(execve)(filename, argv, envp); @@ -211,6 +218,13 @@ uptr internal_clock_gettime(__sanitizer_clockid_t clk_id, void *tp) { } // ----------------- sanitizer_common.h +void FutexWait(atomic_uint32_t *p, u32 cmp) { + // FIXME: implement actual blocking. + sched_yield(); +} + +void FutexWake(atomic_uint32_t *p, u32 count) {} + BlockingMutex::BlockingMutex() { CHECK(sizeof(mutex_t) <= sizeof(opaque_storage_)); internal_memset(this, 0, sizeof(*this)); @@ -231,9 +245,7 @@ void BlockingMutex::Unlock() { CHECK_EQ(mutex_unlock((mutex_t *)&opaque_storage_), 0); } -void BlockingMutex::CheckLocked() { - CHECK_EQ((uptr)thr_self(), owner_); -} +void BlockingMutex::CheckLocked() const { CHECK_EQ((uptr)thr_self(), owner_); } } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp index 515dedd..07e4409 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.cpp @@ -85,8 +85,8 @@ static inline uhwptr *GetCanonicFrame(uptr bp, // Nope, this does not look right either. This means the frame after next does // not have a valid frame pointer, but we can still extract the caller PC. // Unfortunately, there is no way to decide between GCC and LLVM frame - // layouts. Assume GCC. - return bp_prev - 1; + // layouts. Assume LLVM. + return bp_prev; #else return (uhwptr*)bp; #endif @@ -109,21 +109,14 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top, IsAligned((uptr)frame, sizeof(*frame)) && size < max_depth) { #ifdef __powerpc__ - // PowerPC ABIs specify that the return address is saved on the - // *caller's* stack frame. Thus we must dereference the back chain - // to find the caller frame before extracting it. + // PowerPC ABIs specify that the return address is saved at offset + // 16 of the *caller's* stack frame. Thus we must dereference the + // back chain to find the caller frame before extracting it. uhwptr *caller_frame = (uhwptr*)frame[0]; if (!IsValidFrame((uptr)caller_frame, stack_top, bottom) || !IsAligned((uptr)caller_frame, sizeof(uhwptr))) break; - // For most ABIs the offset where the return address is saved is two - // register sizes. The exception is the SVR4 ABI, which uses an - // offset of only one register size. -#ifdef _CALL_SYSV - uhwptr pc1 = caller_frame[1]; -#else uhwptr pc1 = caller_frame[2]; -#endif #elif defined(__s390__) uhwptr pc1 = frame[14]; #elif defined(__riscv) diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace.h b/libsanitizer/sanitizer_common/sanitizer_stacktrace.h index 15616f8..ea330f3 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace.h +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace.h @@ -12,6 +12,7 @@ #ifndef SANITIZER_STACKTRACE_H #define SANITIZER_STACKTRACE_H +#include "sanitizer_common.h" #include "sanitizer_internal_defs.h" #include "sanitizer_platform.h" @@ -32,8 +33,8 @@ static const u32 kStackTraceMax = 256; // Fast unwind is the only option on Mac for now; we will need to // revisit this macro when slow unwind works on Mac, see // https://github.com/google/sanitizers/issues/137 -#if SANITIZER_MAC || SANITIZER_RTEMS -# define SANITIZER_CAN_SLOW_UNWIND 0 +#if SANITIZER_MAC +# define SANITIZER_CAN_SLOW_UNWIND 0 #else # define SANITIZER_CAN_SLOW_UNWIND 1 #endif @@ -56,6 +57,16 @@ struct StackTrace { // Prints a symbolized stacktrace, followed by an empty line. void Print() const; + // Prints a symbolized stacktrace to the output string, followed by an empty + // line. + void PrintTo(InternalScopedString *output) const; + + // Prints a symbolized stacktrace to the output buffer, followed by an empty + // line. Returns the number of symbols that should have been written to buffer + // (not including trailing '\0'). Thus, the string is truncated iff return + // value is not less than "out_buf_size". + uptr PrintTo(char *out_buf, uptr out_buf_size) const; + static bool WillUseFastUnwind(bool request_fast_unwind) { if (!SANITIZER_CAN_FAST_UNWIND) return false; @@ -185,5 +196,26 @@ static inline bool IsValidFrame(uptr frame, uptr stack_top, uptr stack_bottom) { uptr local_stack; \ uptr sp = (uptr)&local_stack +// GET_CURRENT_PC() is equivalent to StackTrace::GetCurrentPc(). +// Optimized x86 version is faster than GetCurrentPc because +// it does not involve a function call, instead it reads RIP register. +// Reads of RIP by an instruction return RIP pointing to the next +// instruction, which is exactly what we want here, thus 0 offset. +// It needs to be a macro because otherwise we will get the name +// of this function on the top of most stacks. Attribute artificial +// does not do what it claims to do, unfortunatley. And attribute +// __nodebug__ is clang-only. If we would have an attribute that +// would remove this function from debug info, we could simply make +// StackTrace::GetCurrentPc() faster. +#if defined(__x86_64__) +# define GET_CURRENT_PC() \ + ({ \ + uptr pc; \ + asm("lea 0(%%rip), %0" : "=r"(pc)); \ + pc; \ + }) +#else +# define GET_CURRENT_PC() StackTrace::GetCurrentPc() +#endif #endif // SANITIZER_STACKTRACE_H diff --git a/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cpp b/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cpp index 7386332..f60ea77 100644 --- a/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_stacktrace_libcdep.cpp @@ -18,46 +18,119 @@ namespace __sanitizer { -void StackTrace::Print() const { +namespace { + +class StackTraceTextPrinter { + public: + StackTraceTextPrinter(const char *stack_trace_fmt, char frame_delimiter, + InternalScopedString *output, + InternalScopedString *dedup_token) + : stack_trace_fmt_(stack_trace_fmt), + frame_delimiter_(frame_delimiter), + output_(output), + dedup_token_(dedup_token), + symbolize_(RenderNeedsSymbolization(stack_trace_fmt)) {} + + bool ProcessAddressFrames(uptr pc) { + SymbolizedStack *frames = symbolize_ + ? Symbolizer::GetOrInit()->SymbolizePC(pc) + : SymbolizedStack::New(pc); + if (!frames) + return false; + + for (SymbolizedStack *cur = frames; cur; cur = cur->next) { + uptr prev_len = output_->length(); + RenderFrame(output_, stack_trace_fmt_, frame_num_++, cur->info.address, + symbolize_ ? &cur->info : nullptr, + common_flags()->symbolize_vs_style, + common_flags()->strip_path_prefix); + + if (prev_len != output_->length()) + output_->append("%c", frame_delimiter_); + + ExtendDedupToken(cur); + } + frames->ClearAll(); + return true; + } + + private: + // Extend the dedup token by appending a new frame. + void ExtendDedupToken(SymbolizedStack *stack) { + if (!dedup_token_) + return; + + if (dedup_frames_-- > 0) { + if (dedup_token_->length()) + dedup_token_->append("--"); + if (stack->info.function != nullptr) + dedup_token_->append(stack->info.function); + } + } + + const char *stack_trace_fmt_; + const char frame_delimiter_; + int dedup_frames_ = common_flags()->dedup_token_length; + uptr frame_num_ = 0; + InternalScopedString *output_; + InternalScopedString *dedup_token_; + const bool symbolize_ = false; +}; + +static void CopyStringToBuffer(const InternalScopedString &str, char *out_buf, + uptr out_buf_size) { + if (!out_buf_size) + return; + + CHECK_GT(out_buf_size, 0); + uptr copy_size = Min(str.length(), out_buf_size - 1); + internal_memcpy(out_buf, str.data(), copy_size); + out_buf[copy_size] = '\0'; +} + +} // namespace + +void StackTrace::PrintTo(InternalScopedString *output) const { + CHECK(output); + + InternalScopedString dedup_token; + StackTraceTextPrinter printer(common_flags()->stack_trace_format, '\n', + output, &dedup_token); + if (trace == nullptr || size == 0) { - Printf(" <empty stack>\n\n"); + output->append(" <empty stack>\n\n"); return; } - InternalScopedString frame_desc; - InternalScopedString dedup_token; - int dedup_frames = common_flags()->dedup_token_length; - bool symbolize = RenderNeedsSymbolization(common_flags()->stack_trace_format); - uptr frame_num = 0; + for (uptr i = 0; i < size && trace[i]; i++) { // PCs in stack traces are actually the return addresses, that is, // addresses of the next instructions after the call. uptr pc = GetPreviousInstructionPc(trace[i]); - SymbolizedStack *frames; - if (symbolize) - frames = Symbolizer::GetOrInit()->SymbolizePC(pc); - else - frames = SymbolizedStack::New(pc); - CHECK(frames); - for (SymbolizedStack *cur = frames; cur; cur = cur->next) { - frame_desc.clear(); - RenderFrame(&frame_desc, common_flags()->stack_trace_format, frame_num++, - cur->info.address, symbolize ? &cur->info : nullptr, - common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix); - Printf("%s\n", frame_desc.data()); - if (dedup_frames-- > 0) { - if (dedup_token.length()) - dedup_token.append("--"); - if (cur->info.function != nullptr) - dedup_token.append(cur->info.function); - } - } - frames->ClearAll(); + CHECK(printer.ProcessAddressFrames(pc)); } - // Always print a trailing empty line after stack trace. - Printf("\n"); + + // Always add a trailing empty line after stack trace. + output->append("\n"); + + // Append deduplication token, if non-empty. if (dedup_token.length()) - Printf("DEDUP_TOKEN: %s\n", dedup_token.data()); + output->append("DEDUP_TOKEN: %s\n", dedup_token.data()); +} + +uptr StackTrace::PrintTo(char *out_buf, uptr out_buf_size) const { + CHECK(out_buf); + + InternalScopedString output; + PrintTo(&output); + CopyStringToBuffer(output, out_buf, out_buf_size); + + return output.length(); +} + +void StackTrace::Print() const { + InternalScopedString output; + PrintTo(&output); + Printf("%s", output.data()); } void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context, @@ -82,12 +155,15 @@ void BufferedStackTrace::Unwind(u32 max_depth, uptr pc, uptr bp, void *context, UnwindSlow(pc, context, max_depth); else UnwindSlow(pc, max_depth); + // If there are too few frames, the program may be built with + // -fno-asynchronous-unwind-tables. Fall back to fast unwinder below. + if (size > 2 || size >= max_depth) + return; #else UNREACHABLE("slow unwind requested but not available"); #endif - } else { - UnwindFast(pc, bp, stack_top, stack_bottom, max_depth); } + UnwindFast(pc, bp, stack_top, stack_bottom, max_depth); } static int GetModuleAndOffsetForPc(uptr pc, char *module_name, @@ -112,41 +188,18 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_symbolize_pc(uptr pc, const char *fmt, char *out_buf, uptr out_buf_size) { - if (!out_buf_size) return; - pc = StackTrace::GetPreviousInstructionPc(pc); - SymbolizedStack *frame; - bool symbolize = RenderNeedsSymbolization(fmt); - if (symbolize) - frame = Symbolizer::GetOrInit()->SymbolizePC(pc); - else - frame = SymbolizedStack::New(pc); - if (!frame) { - internal_strncpy(out_buf, "<can't symbolize>", out_buf_size); - out_buf[out_buf_size - 1] = 0; + if (!out_buf_size) return; + + pc = StackTrace::GetPreviousInstructionPc(pc); + + InternalScopedString output; + StackTraceTextPrinter printer(fmt, '\0', &output, nullptr); + if (!printer.ProcessAddressFrames(pc)) { + output.clear(); + output.append("<can't symbolize>"); } - InternalScopedString frame_desc; - uptr frame_num = 0; - // Reserve one byte for the final 0. - char *out_end = out_buf + out_buf_size - 1; - for (SymbolizedStack *cur = frame; cur && out_buf < out_end; - cur = cur->next) { - frame_desc.clear(); - RenderFrame(&frame_desc, fmt, frame_num++, cur->info.address, - symbolize ? &cur->info : nullptr, - common_flags()->symbolize_vs_style, - common_flags()->strip_path_prefix); - if (!frame_desc.length()) - continue; - // Reserve one byte for the terminating 0. - uptr n = out_end - out_buf - 1; - internal_strncpy(out_buf, frame_desc.data(), n); - out_buf += __sanitizer::Min<uptr>(n, frame_desc.length()); - *out_buf++ = 0; - } - CHECK(out_buf <= out_end); - *out_buf = 0; - frame->ClearAll(); + CopyStringToBuffer(output, out_buf, out_buf_size); } SANITIZER_INTERFACE_ATTRIBUTE diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_markup.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_markup.cpp index 01edef9..9a5b4a8 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_markup.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_markup.cpp @@ -16,14 +16,13 @@ #if SANITIZER_FUCHSIA #include "sanitizer_symbolizer_fuchsia.h" -#elif SANITIZER_RTEMS -#include "sanitizer_symbolizer_rtems.h" -#endif -#include "sanitizer_stacktrace.h" -#include "sanitizer_symbolizer.h" - -#include <limits.h> -#include <unwind.h> +# endif + +# include <limits.h> +# include <unwind.h> + +# include "sanitizer_stacktrace.h" +# include "sanitizer_symbolizer.h" namespace __sanitizer { diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp b/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp index 9287993..f330ed3 100644 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_symbolizer_report.cpp @@ -120,7 +120,7 @@ void ReportMmapWriteExec(int prot) { #endif } -#if !SANITIZER_FUCHSIA && !SANITIZER_RTEMS && !SANITIZER_GO +#if !SANITIZER_FUCHSIA && !SANITIZER_GO void StartReportDeadlySignal() { // Write the first message using fd=2, just in case. // It may actually fail to write in case stderr is closed. @@ -250,17 +250,17 @@ void HandleDeadlySignal(void *siginfo, void *context, u32 tid, #endif // !SANITIZER_FUCHSIA && !SANITIZER_GO -static atomic_uintptr_t reporting_thread = {0}; -static StaticSpinMutex CommonSanitizerReportMutex; +atomic_uintptr_t ScopedErrorReportLock::reporting_thread_ = {0}; +StaticSpinMutex ScopedErrorReportLock::mutex_; -ScopedErrorReportLock::ScopedErrorReportLock() { +void ScopedErrorReportLock::Lock() { uptr current = GetThreadSelf(); for (;;) { uptr expected = 0; - if (atomic_compare_exchange_strong(&reporting_thread, &expected, current, + if (atomic_compare_exchange_strong(&reporting_thread_, &expected, current, memory_order_relaxed)) { // We've claimed reporting_thread so proceed. - CommonSanitizerReportMutex.Lock(); + mutex_.Lock(); return; } @@ -282,13 +282,11 @@ ScopedErrorReportLock::ScopedErrorReportLock() { } } -ScopedErrorReportLock::~ScopedErrorReportLock() { - CommonSanitizerReportMutex.Unlock(); - atomic_store_relaxed(&reporting_thread, 0); +void ScopedErrorReportLock::Unlock() { + mutex_.Unlock(); + atomic_store_relaxed(&reporting_thread_, 0); } -void ScopedErrorReportLock::CheckLocked() { - CommonSanitizerReportMutex.CheckLocked(); -} +void ScopedErrorReportLock::CheckLocked() { mutex_.CheckLocked(); } } // namespace __sanitizer diff --git a/libsanitizer/sanitizer_common/sanitizer_symbolizer_rtems.h b/libsanitizer/sanitizer_common/sanitizer_symbolizer_rtems.h deleted file mode 100644 index 3371092..0000000 --- a/libsanitizer/sanitizer_common/sanitizer_symbolizer_rtems.h +++ /dev/null @@ -1,40 +0,0 @@ -//===-- sanitizer_symbolizer_rtems.h -----------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is shared between various sanitizers' runtime libraries. -// -// Define RTEMS's string formats and limits for the markup symbolizer. -//===----------------------------------------------------------------------===// -#ifndef SANITIZER_SYMBOLIZER_RTEMS_H -#define SANITIZER_SYMBOLIZER_RTEMS_H - -#include "sanitizer_internal_defs.h" - -namespace __sanitizer { - -// The Myriad RTEMS symbolizer currently only parses backtrace lines, -// so use a format that the symbolizer understands. For other -// markups, keep them the same as the Fuchsia's. - -// This is used by UBSan for type names, and by ASan for global variable names. -constexpr const char *kFormatDemangle = "{{{symbol:%s}}}"; -constexpr uptr kFormatDemangleMax = 1024; // Arbitrary. - -// Function name or equivalent from PC location. -constexpr const char *kFormatFunction = "{{{pc:%p}}}"; -constexpr uptr kFormatFunctionMax = 64; // More than big enough for 64-bit hex. - -// Global variable name or equivalent from data memory address. -constexpr const char *kFormatData = "{{{data:%p}}}"; - -// One frame in a backtrace (printed on a line by itself). -constexpr const char *kFormatFrame = " [%u] IP: %p"; - -} // namespace __sanitizer - -#endif // SANITIZER_SYMBOLIZER_RTEMS_H diff --git a/libsanitizer/sanitizer_common/sanitizer_thread_registry.cpp b/libsanitizer/sanitizer_common/sanitizer_thread_registry.cpp index 3273da3..745fbf7 100644 --- a/libsanitizer/sanitizer_common/sanitizer_thread_registry.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_thread_registry.cpp @@ -99,6 +99,9 @@ void ThreadContextBase::Reset() { // ThreadRegistry implementation. +ThreadRegistry::ThreadRegistry(ThreadContextFactory factory) + : ThreadRegistry(factory, UINT32_MAX, UINT32_MAX, 0) {} + ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads, u32 thread_quarantine_size, u32 max_reuse) : context_factory_(factory), @@ -106,13 +109,10 @@ ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads, thread_quarantine_size_(thread_quarantine_size), max_reuse_(max_reuse), mtx_(), - n_contexts_(0), total_threads_(0), alive_threads_(0), max_alive_threads_(0), running_threads_(0) { - threads_ = (ThreadContextBase **)MmapOrDie(max_threads_ * sizeof(threads_[0]), - "ThreadRegistry"); dead_threads_.clear(); invalid_threads_.clear(); } @@ -120,7 +120,8 @@ ThreadRegistry::ThreadRegistry(ThreadContextFactory factory, u32 max_threads, void ThreadRegistry::GetNumberOfThreads(uptr *total, uptr *running, uptr *alive) { BlockingMutexLock l(&mtx_); - if (total) *total = n_contexts_; + if (total) + *total = threads_.size(); if (running) *running = running_threads_; if (alive) *alive = alive_threads_; } @@ -137,11 +138,11 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid, ThreadContextBase *tctx = QuarantinePop(); if (tctx) { tid = tctx->tid; - } else if (n_contexts_ < max_threads_) { + } else if (threads_.size() < max_threads_) { // Allocate new thread context and tid. - tid = n_contexts_++; + tid = threads_.size(); tctx = context_factory_(tid); - threads_[tid] = tctx; + threads_.push_back(tctx); } else { #if !SANITIZER_GO Report("%s: Thread limit (%u threads) exceeded. Dying.\n", @@ -169,7 +170,7 @@ u32 ThreadRegistry::CreateThread(uptr user_id, bool detached, u32 parent_tid, void ThreadRegistry::RunCallbackForEachThreadLocked(ThreadCallback cb, void *arg) { CheckLocked(); - for (u32 tid = 0; tid < n_contexts_; tid++) { + for (u32 tid = 0; tid < threads_.size(); tid++) { ThreadContextBase *tctx = threads_[tid]; if (tctx == 0) continue; @@ -179,7 +180,7 @@ void ThreadRegistry::RunCallbackForEachThreadLocked(ThreadCallback cb, u32 ThreadRegistry::FindThread(FindThreadCallback cb, void *arg) { BlockingMutexLock l(&mtx_); - for (u32 tid = 0; tid < n_contexts_; tid++) { + for (u32 tid = 0; tid < threads_.size(); tid++) { ThreadContextBase *tctx = threads_[tid]; if (tctx != 0 && cb(tctx, arg)) return tctx->tid; @@ -190,7 +191,7 @@ u32 ThreadRegistry::FindThread(FindThreadCallback cb, void *arg) { ThreadContextBase * ThreadRegistry::FindThreadContextLocked(FindThreadCallback cb, void *arg) { CheckLocked(); - for (u32 tid = 0; tid < n_contexts_; tid++) { + for (u32 tid = 0; tid < threads_.size(); tid++) { ThreadContextBase *tctx = threads_[tid]; if (tctx != 0 && cb(tctx, arg)) return tctx; @@ -211,7 +212,6 @@ ThreadContextBase *ThreadRegistry::FindThreadContextByOsIDLocked(tid_t os_id) { void ThreadRegistry::SetThreadName(u32 tid, const char *name) { BlockingMutexLock l(&mtx_); - CHECK_LT(tid, n_contexts_); ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); CHECK_EQ(SANITIZER_FUCHSIA ? ThreadStatusCreated : ThreadStatusRunning, @@ -221,7 +221,7 @@ void ThreadRegistry::SetThreadName(u32 tid, const char *name) { void ThreadRegistry::SetThreadNameByUserId(uptr user_id, const char *name) { BlockingMutexLock l(&mtx_); - for (u32 tid = 0; tid < n_contexts_; tid++) { + for (u32 tid = 0; tid < threads_.size(); tid++) { ThreadContextBase *tctx = threads_[tid]; if (tctx != 0 && tctx->user_id == user_id && tctx->status != ThreadStatusInvalid) { @@ -233,7 +233,6 @@ void ThreadRegistry::SetThreadNameByUserId(uptr user_id, const char *name) { void ThreadRegistry::DetachThread(u32 tid, void *arg) { BlockingMutexLock l(&mtx_); - CHECK_LT(tid, n_contexts_); ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); if (tctx->status == ThreadStatusInvalid) { @@ -254,7 +253,6 @@ void ThreadRegistry::JoinThread(u32 tid, void *arg) { do { { BlockingMutexLock l(&mtx_); - CHECK_LT(tid, n_contexts_); ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); if (tctx->status == ThreadStatusInvalid) { @@ -280,7 +278,6 @@ ThreadStatus ThreadRegistry::FinishThread(u32 tid) { BlockingMutexLock l(&mtx_); CHECK_GT(alive_threads_, 0); alive_threads_--; - CHECK_LT(tid, n_contexts_); ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); bool dead = tctx->detached; @@ -306,7 +303,6 @@ void ThreadRegistry::StartThread(u32 tid, tid_t os_id, ThreadType thread_type, void *arg) { BlockingMutexLock l(&mtx_); running_threads_++; - CHECK_LT(tid, n_contexts_); ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); CHECK_EQ(ThreadStatusCreated, tctx->status); @@ -339,7 +335,6 @@ ThreadContextBase *ThreadRegistry::QuarantinePop() { void ThreadRegistry::SetThreadUserId(u32 tid, uptr user_id) { BlockingMutexLock l(&mtx_); - CHECK_LT(tid, n_contexts_); ThreadContextBase *tctx = threads_[tid]; CHECK_NE(tctx, 0); CHECK_NE(tctx->status, ThreadStatusInvalid); diff --git a/libsanitizer/sanitizer_common/sanitizer_thread_registry.h b/libsanitizer/sanitizer_common/sanitizer_thread_registry.h index dcd445c..0b28bbe 100644 --- a/libsanitizer/sanitizer_common/sanitizer_thread_registry.h +++ b/libsanitizer/sanitizer_common/sanitizer_thread_registry.h @@ -85,22 +85,22 @@ class ThreadContextBase { typedef ThreadContextBase* (*ThreadContextFactory)(u32 tid); -class ThreadRegistry { +class MUTEX ThreadRegistry { public: + ThreadRegistry(ThreadContextFactory factory); ThreadRegistry(ThreadContextFactory factory, u32 max_threads, - u32 thread_quarantine_size, u32 max_reuse = 0); + u32 thread_quarantine_size, u32 max_reuse); void GetNumberOfThreads(uptr *total = nullptr, uptr *running = nullptr, uptr *alive = nullptr); uptr GetMaxAliveThreads(); - void Lock() { mtx_.Lock(); } - void CheckLocked() { mtx_.CheckLocked(); } - void Unlock() { mtx_.Unlock(); } + void Lock() ACQUIRE() { mtx_.Lock(); } + void CheckLocked() const CHECK_LOCKED() { mtx_.CheckLocked(); } + void Unlock() RELEASE() { mtx_.Unlock(); } // Should be guarded by ThreadRegistryLock. ThreadContextBase *GetThreadLocked(u32 tid) { - DCHECK_LT(tid, n_contexts_); - return threads_[tid]; + return threads_.empty() ? nullptr : threads_[tid]; } u32 CreateThread(uptr user_id, bool detached, u32 parent_tid, void *arg); @@ -137,15 +137,13 @@ class ThreadRegistry { BlockingMutex mtx_; - u32 n_contexts_; // Number of created thread contexts, - // at most max_threads_. u64 total_threads_; // Total number of created threads. May be greater than // max_threads_ if contexts were reused. uptr alive_threads_; // Created or running. uptr max_alive_threads_; uptr running_threads_; - ThreadContextBase **threads_; // Array of thread contexts is leaked. + InternalMmapVector<ThreadContextBase *> threads_; IntrusiveList<ThreadContextBase> dead_threads_; IntrusiveList<ThreadContextBase> invalid_threads_; diff --git a/libsanitizer/sanitizer_common/sanitizer_thread_safety.h b/libsanitizer/sanitizer_common/sanitizer_thread_safety.h new file mode 100644 index 0000000..52b25ed --- /dev/null +++ b/libsanitizer/sanitizer_common/sanitizer_thread_safety.h @@ -0,0 +1,42 @@ +//===-- sanitizer_thread_safety.h -------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is shared between sanitizer tools. +// +// Wrappers around thread safety annotations. +// https://clang.llvm.org/docs/ThreadSafetyAnalysis.html +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_THREAD_SAFETY_H +#define SANITIZER_THREAD_SAFETY_H + +#if defined(__clang__) +# define THREAD_ANNOTATION(x) __attribute__((x)) +#else +# define THREAD_ANNOTATION(x) +#endif + +#define MUTEX THREAD_ANNOTATION(capability("mutex")) +#define SCOPED_LOCK THREAD_ANNOTATION(scoped_lockable) +#define GUARDED_BY(x) THREAD_ANNOTATION(guarded_by(x)) +#define PT_GUARDED_BY(x) THREAD_ANNOTATION(pt_guarded_by(x)) +#define REQUIRES(...) THREAD_ANNOTATION(requires_capability(__VA_ARGS__)) +#define REQUIRES_SHARED(...) \ + THREAD_ANNOTATION(requires_shared_capability(__VA_ARGS__)) +#define ACQUIRE(...) THREAD_ANNOTATION(acquire_capability(__VA_ARGS__)) +#define ACQUIRE_SHARED(...) \ + THREAD_ANNOTATION(acquire_shared_capability(__VA_ARGS__)) +#define TRY_ACQUIRE(...) THREAD_ANNOTATION(try_acquire_capability(__VA_ARGS__)) +#define RELEASE(...) THREAD_ANNOTATION(release_capability(__VA_ARGS__)) +#define RELEASE_SHARED(...) \ + THREAD_ANNOTATION(release_shared_capability(__VA_ARGS__)) +#define EXCLUDES(...) THREAD_ANNOTATION(locks_excluded(__VA_ARGS__)) +#define CHECK_LOCKED(...) THREAD_ANNOTATION(assert_capability(__VA_ARGS__)) +#define NO_THREAD_SAFETY_ANALYSIS THREAD_ANNOTATION(no_thread_safety_analysis) + +#endif diff --git a/libsanitizer/sanitizer_common/sanitizer_win.cpp b/libsanitizer/sanitizer_common/sanitizer_win.cpp index f383e13..dddd885 100644 --- a/libsanitizer/sanitizer_common/sanitizer_win.cpp +++ b/libsanitizer/sanitizer_common/sanitizer_win.cpp @@ -44,6 +44,9 @@ TRACELOGGING_DEFINE_PROVIDER(g_asan_provider, "AddressSanitizerLoggingProvider", #define TraceLoggingUnregister(x) #endif +// For WaitOnAddress +# pragma comment(lib, "synchronization.lib") + // A macro to tell the compiler that this part of the code cannot be reached, // if the compiler supports this feature. Since we're using this in // code that is called when terminating the process, the expansion of the @@ -541,13 +544,7 @@ bool IsAbsolutePath(const char *path) { IsPathSeparator(path[2]); } -void SleepForSeconds(int seconds) { - Sleep(seconds * 1000); -} - -void SleepForMillis(int millis) { - Sleep(millis); -} +void internal_usleep(u64 useconds) { Sleep(useconds / 1000); } u64 NanoTime() { static LARGE_INTEGER frequency = {}; @@ -819,6 +816,17 @@ uptr GetRSS() { void *internal_start_thread(void *(*func)(void *arg), void *arg) { return 0; } void internal_join_thread(void *th) { } +void FutexWait(atomic_uint32_t *p, u32 cmp) { + WaitOnAddress(p, &cmp, sizeof(cmp), INFINITE); +} + +void FutexWake(atomic_uint32_t *p, u32 count) { + if (count == 1) + WakeByAddressSingle(p); + else + WakeByAddressAll(p); +} + // ---------------------- BlockingMutex ---------------- {{{1 BlockingMutex::BlockingMutex() { @@ -838,9 +846,7 @@ void BlockingMutex::Unlock() { ReleaseSRWLockExclusive((PSRWLOCK)opaque_storage_); } -void BlockingMutex::CheckLocked() { - CHECK_EQ(owner_, GetThreadSelf()); -} +void BlockingMutex::CheckLocked() const { CHECK_EQ(owner_, GetThreadSelf()); } uptr GetTlsSize() { return 0; diff --git a/libsanitizer/tsan/Makefile.am b/libsanitizer/tsan/Makefile.am index fb20ff7..dcb247e 100644 --- a/libsanitizer/tsan/Makefile.am +++ b/libsanitizer/tsan/Makefile.am @@ -43,13 +43,12 @@ tsan_files = \ tsan_rtl_report.cpp \ tsan_rtl_thread.cpp \ tsan_stack_trace.cpp \ - tsan_stat.cpp \ tsan_suppressions.cpp \ tsan_symbolize.cpp \ tsan_sync.cpp libtsan_la_SOURCES = $(tsan_files) -EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S +EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S tsan_rtl_s390x.S libtsan_la_LIBADD = $(top_builddir)/sanitizer_common/libsanitizer_common.la $(top_builddir)/interception/libinterception.la $(TSAN_TARGET_DEPENDENT_OBJECTS) libtsan_la_DEPENDENCIES = $(top_builddir)/sanitizer_common/libsanitizer_common.la $(top_builddir)/interception/libinterception.la $(TSAN_TARGET_DEPENDENT_OBJECTS) if LIBBACKTRACE_SUPPORTED diff --git a/libsanitizer/tsan/Makefile.in b/libsanitizer/tsan/Makefile.in index ce11d24..83617cf 100644 --- a/libsanitizer/tsan/Makefile.in +++ b/libsanitizer/tsan/Makefile.in @@ -155,8 +155,7 @@ am__objects_1 = tsan_clock.lo tsan_debugging.lo tsan_external.lo \ tsan_platform_posix.lo tsan_platform_windows.lo tsan_report.lo \ tsan_rtl.lo tsan_rtl_mutex.lo tsan_rtl_proc.lo \ tsan_rtl_report.lo tsan_rtl_thread.lo tsan_stack_trace.lo \ - tsan_stat.lo tsan_suppressions.lo tsan_symbolize.lo \ - tsan_sync.lo + tsan_suppressions.lo tsan_symbolize.lo tsan_sync.lo am_libtsan_la_OBJECTS = $(am__objects_1) libtsan_la_OBJECTS = $(am_libtsan_la_OBJECTS) AM_V_lt = $(am__v_lt_@AM_V@) @@ -446,13 +445,12 @@ tsan_files = \ tsan_rtl_report.cpp \ tsan_rtl_thread.cpp \ tsan_stack_trace.cpp \ - tsan_stat.cpp \ tsan_suppressions.cpp \ tsan_symbolize.cpp \ tsan_sync.cpp libtsan_la_SOURCES = $(tsan_files) -EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S +EXTRA_libtsan_la_SOURCES = tsan_rtl_amd64.S tsan_rtl_aarch64.S tsan_rtl_mips64.S tsan_rtl_ppc64.S tsan_rtl_s390x.S libtsan_la_LIBADD = \ $(top_builddir)/sanitizer_common/libsanitizer_common.la \ $(top_builddir)/interception/libinterception.la \ @@ -612,9 +610,9 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_ppc64.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_proc.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_report.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_s390x.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_thread.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_stack_trace.Plo@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_stat.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_suppressions.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_symbolize.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_sync.Plo@am__quote@ diff --git a/libsanitizer/tsan/tsan_clock.cpp b/libsanitizer/tsan/tsan_clock.cpp index 8e51883..61848c2 100644 --- a/libsanitizer/tsan/tsan_clock.cpp +++ b/libsanitizer/tsan/tsan_clock.cpp @@ -80,14 +80,6 @@ // release-store operation by the thread with release_store_tid_ index. // release_store_reused_ - reuse count of release_store_tid_. -// We don't have ThreadState in these methods, so this is an ugly hack that -// works only in C++. -#if !SANITIZER_GO -# define CPP_STAT_INC(typ) StatInc(cur_thread(), typ) -#else -# define CPP_STAT_INC(typ) (void)0 -#endif - namespace __tsan { static atomic_uint32_t *ref_ptr(ClockBlock *cb) { @@ -138,14 +130,11 @@ void ThreadClock::ResetCached(ClockCache *c) { void ThreadClock::acquire(ClockCache *c, SyncClock *src) { DCHECK_LE(nclk_, kMaxTid); DCHECK_LE(src->size_, kMaxTid); - CPP_STAT_INC(StatClockAcquire); // Check if it's empty -> no need to do anything. const uptr nclk = src->size_; - if (nclk == 0) { - CPP_STAT_INC(StatClockAcquireEmpty); + if (nclk == 0) return; - } bool acquired = false; for (unsigned i = 0; i < kDirtyTids; i++) { @@ -162,7 +151,6 @@ void ThreadClock::acquire(ClockCache *c, SyncClock *src) { // Check if we've already acquired src after the last release operation on src if (tid_ >= nclk || src->elem(tid_).reused != reused_) { // O(N) acquire. - CPP_STAT_INC(StatClockAcquireFull); nclk_ = max(nclk_, nclk); u64 *dst_pos = &clk_[0]; for (ClockElem &src_elem : *src) { @@ -180,7 +168,6 @@ void ThreadClock::acquire(ClockCache *c, SyncClock *src) { } if (acquired) { - CPP_STAT_INC(StatClockAcquiredSomething); last_acquire_ = clk_[tid_]; ResetCached(c); } @@ -223,7 +210,6 @@ void ThreadClock::releaseStoreAcquire(ClockCache *c, SyncClock *sc) { sc->release_store_reused_ = 0; if (acquired) { - CPP_STAT_INC(StatClockAcquiredSomething); last_acquire_ = clk_[tid_]; ResetCached(c); } @@ -240,7 +226,6 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) { return; } - CPP_STAT_INC(StatClockRelease); // Check if we need to resize dst. if (dst->size_ < nclk_) dst->Resize(c, nclk_); @@ -257,12 +242,9 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) { } // O(N) release. - CPP_STAT_INC(StatClockReleaseFull); dst->Unshare(c); // First, remember whether we've acquired dst. bool acquired = IsAlreadyAcquired(dst); - if (acquired) - CPP_STAT_INC(StatClockReleaseAcquired); // Update dst->clk_. dst->FlushDirty(); uptr i = 0; @@ -272,8 +254,6 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) { i++; } // Clear 'acquired' flag in the remaining elements. - if (nclk_ < dst->size_) - CPP_STAT_INC(StatClockReleaseClearTail); dst->release_store_tid_ = kInvalidTid; dst->release_store_reused_ = 0; // If we've acquired dst, remember this fact, @@ -285,7 +265,6 @@ void ThreadClock::release(ClockCache *c, SyncClock *dst) { void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) { DCHECK_LE(nclk_, kMaxTid); DCHECK_LE(dst->size_, kMaxTid); - CPP_STAT_INC(StatClockStore); if (dst->size_ == 0 && cached_idx_ != 0) { // Reuse the cached clock. @@ -320,13 +299,11 @@ void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) { if (dst->release_store_tid_ == tid_ && dst->release_store_reused_ == reused_ && !HasAcquiredAfterRelease(dst)) { - CPP_STAT_INC(StatClockStoreFast); UpdateCurrentThread(c, dst); return; } // O(N) release-store. - CPP_STAT_INC(StatClockStoreFull); dst->Unshare(c); // Note: dst can be larger than this ThreadClock. // This is fine since clk_ beyond size is all zeros. @@ -358,7 +335,6 @@ void ThreadClock::ReleaseStore(ClockCache *c, SyncClock *dst) { } void ThreadClock::acq_rel(ClockCache *c, SyncClock *dst) { - CPP_STAT_INC(StatClockAcquireRelease); acquire(c, dst); ReleaseStore(c, dst); } @@ -370,7 +346,6 @@ void ThreadClock::UpdateCurrentThread(ClockCache *c, SyncClock *dst) const { SyncClock::Dirty *dirty = &dst->dirty_[i]; const unsigned tid = dirty->tid(); if (tid == tid_ || tid == kInvalidTid) { - CPP_STAT_INC(StatClockReleaseFast); dirty->set_tid(tid_); dirty->epoch = clk_[tid_]; return; @@ -379,7 +354,6 @@ void ThreadClock::UpdateCurrentThread(ClockCache *c, SyncClock *dst) const { // Reset all 'acquired' flags, O(N). // We are going to touch dst elements, so we need to unshare it. dst->Unshare(c); - CPP_STAT_INC(StatClockReleaseSlow); dst->elem(tid_).epoch = clk_[tid_]; for (uptr i = 0; i < dst->size_; i++) dst->elem(i).reused = 0; @@ -456,7 +430,6 @@ void SyncClock::ResetImpl() { } void SyncClock::Resize(ClockCache *c, uptr nclk) { - CPP_STAT_INC(StatClockReleaseResize); Unshare(c); if (nclk <= capacity()) { // Memory is already allocated, just increase the size. diff --git a/libsanitizer/tsan/tsan_defs.h b/libsanitizer/tsan/tsan_defs.h index f53787a..5c8f280 100644 --- a/libsanitizer/tsan/tsan_defs.h +++ b/libsanitizer/tsan/tsan_defs.h @@ -15,7 +15,6 @@ #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_libc.h" -#include "tsan_stat.h" #include "ubsan/ubsan_platform.h" // Setup defaults for compile definitions. @@ -23,10 +22,6 @@ # define TSAN_NO_HISTORY 0 #endif -#ifndef TSAN_COLLECT_STATS -# define TSAN_COLLECT_STATS 0 -#endif - #ifndef TSAN_CONTAINS_UBSAN # if CAN_SANITIZE_UB && !SANITIZER_GO # define TSAN_CONTAINS_UBSAN 1 @@ -107,23 +102,12 @@ void build_consistency_debug(); void build_consistency_release(); #endif -#if TSAN_COLLECT_STATS -void build_consistency_stats(); -#else -void build_consistency_nostats(); -#endif - static inline void USED build_consistency() { #if SANITIZER_DEBUG build_consistency_debug(); #else build_consistency_release(); #endif -#if TSAN_COLLECT_STATS - build_consistency_stats(); -#else - build_consistency_nostats(); -#endif } template<typename T> diff --git a/libsanitizer/tsan/tsan_interceptors.h b/libsanitizer/tsan/tsan_interceptors.h index 29576ea..c5716f5 100644 --- a/libsanitizer/tsan/tsan_interceptors.h +++ b/libsanitizer/tsan/tsan_interceptors.h @@ -30,14 +30,14 @@ inline bool in_symbolizer() { } // namespace __tsan -#define SCOPED_INTERCEPTOR_RAW(func, ...) \ - cur_thread_init(); \ - ThreadState *thr = cur_thread(); \ - const uptr caller_pc = GET_CALLER_PC(); \ - ScopedInterceptor si(thr, #func, caller_pc); \ - const uptr pc = StackTrace::GetCurrentPc(); \ - (void)pc; \ -/**/ +#define SCOPED_INTERCEPTOR_RAW(func, ...) \ + cur_thread_init(); \ + ThreadState *thr = cur_thread(); \ + const uptr caller_pc = GET_CALLER_PC(); \ + ScopedInterceptor si(thr, #func, caller_pc); \ + const uptr pc = GET_CURRENT_PC(); \ + (void)pc; \ + /**/ #define SCOPED_TSAN_INTERCEPTOR(func, ...) \ SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ diff --git a/libsanitizer/tsan/tsan_interceptors_mac.cpp b/libsanitizer/tsan/tsan_interceptors_mac.cpp index ed10fcc..2d400c7 100644 --- a/libsanitizer/tsan/tsan_interceptors_mac.cpp +++ b/libsanitizer/tsan/tsan_interceptors_mac.cpp @@ -44,8 +44,9 @@ namespace __tsan { // actually aliases of each other, and we cannot have different interceptors for // them, because they're actually the same function. Thus, we have to stay // conservative and treat the non-barrier versions as mo_acq_rel. -static const morder kMacOrderBarrier = mo_acq_rel; -static const morder kMacOrderNonBarrier = mo_acq_rel; +static constexpr morder kMacOrderBarrier = mo_acq_rel; +static constexpr morder kMacOrderNonBarrier = mo_acq_rel; +static constexpr morder kMacFailureOrder = mo_relaxed; #define OSATOMIC_INTERCEPTOR(return_t, t, tsan_t, f, tsan_atomic_f, mo) \ TSAN_INTERCEPTOR(return_t, f, t x, volatile t *ptr) { \ @@ -110,7 +111,7 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, SCOPED_TSAN_INTERCEPTOR(f, old_value, new_value, ptr); \ return tsan_atomic_f##_compare_exchange_strong( \ (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ - kMacOrderNonBarrier, kMacOrderNonBarrier); \ + kMacOrderNonBarrier, kMacFailureOrder); \ } \ \ TSAN_INTERCEPTOR(bool, f##Barrier, t old_value, t new_value, \ @@ -118,7 +119,7 @@ OSATOMIC_INTERCEPTORS_BITWISE(OSAtomicXor, fetch_xor, SCOPED_TSAN_INTERCEPTOR(f##Barrier, old_value, new_value, ptr); \ return tsan_atomic_f##_compare_exchange_strong( \ (volatile tsan_t *)ptr, (tsan_t *)&old_value, (tsan_t)new_value, \ - kMacOrderBarrier, kMacOrderNonBarrier); \ + kMacOrderBarrier, kMacFailureOrder); \ } OSATOMIC_INTERCEPTORS_CAS(OSAtomicCompareAndSwapInt, __tsan_atomic32, a32, int) diff --git a/libsanitizer/tsan/tsan_interceptors_posix.cpp b/libsanitizer/tsan/tsan_interceptors_posix.cpp index 2651e22..6808f2e 100644 --- a/libsanitizer/tsan/tsan_interceptors_posix.cpp +++ b/libsanitizer/tsan/tsan_interceptors_posix.cpp @@ -71,7 +71,8 @@ struct ucontext_t { }; #endif -#if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 +#if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \ + defined(__s390x__) #define PTHREAD_ABI_BASE "GLIBC_2.3.2" #elif defined(__aarch64__) || SANITIZER_PPC64V2 #define PTHREAD_ABI_BASE "GLIBC_2.17" @@ -2270,6 +2271,7 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #define NEED_TLS_GET_ADDR #endif #undef SANITIZER_INTERCEPT_TLS_GET_ADDR +#define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1 #undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) @@ -2585,6 +2587,20 @@ static void syscall_post_fork(uptr pc, int pid) { #include "sanitizer_common/sanitizer_syscalls_netbsd.inc" #ifdef NEED_TLS_GET_ADDR + +static void handle_tls_addr(void *arg, void *res) { + ThreadState *thr = cur_thread(); + if (!thr) + return; + DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr, + thr->tls_addr + thr->tls_size); + if (!dtv) + return; + // New DTLS block has been allocated. + MemoryResetRange(thr, 0, dtv->beg, dtv->size); +} + +#if !SANITIZER_S390 // Define own interceptor instead of sanitizer_common's for three reasons: // 1. It must not process pending signals. // Signal handlers may contain MOVDQA instruction (see below). @@ -2597,18 +2613,18 @@ static void syscall_post_fork(uptr pc, int pid) { // execute MOVDQA with stack addresses. TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) { void *res = REAL(__tls_get_addr)(arg); - ThreadState *thr = cur_thread(); - if (!thr) - return res; - DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr, - thr->tls_addr + thr->tls_size); - if (!dtv) - return res; - // New DTLS block has been allocated. - MemoryResetRange(thr, 0, dtv->beg, dtv->size); + handle_tls_addr(arg, res); + return res; +} +#else // SANITIZER_S390 +TSAN_INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) { + uptr res = __tls_get_offset_wrapper(arg, REAL(__tls_get_offset)); + char *tp = static_cast<char *>(__builtin_thread_pointer()); + handle_tls_addr(arg, res + tp); return res; } #endif +#endif #if SANITIZER_NETBSD TSAN_INTERCEPTOR(void, _lwp_exit) { @@ -2830,7 +2846,12 @@ void InitializeInterceptors() { TSAN_INTERCEPT(_exit); #ifdef NEED_TLS_GET_ADDR +#if !SANITIZER_S390 TSAN_INTERCEPT(__tls_get_addr); +#else + TSAN_INTERCEPT(__tls_get_addr_internal); + TSAN_INTERCEPT(__tls_get_offset); +#endif #endif TSAN_MAYBE_INTERCEPT__LWP_EXIT; diff --git a/libsanitizer/tsan/tsan_interface.h b/libsanitizer/tsan/tsan_interface.h index 6e022b5..124aa2f 100644 --- a/libsanitizer/tsan/tsan_interface.h +++ b/libsanitizer/tsan/tsan_interface.h @@ -196,7 +196,8 @@ typedef unsigned short a16; typedef unsigned int a32; typedef unsigned long long a64; #if !SANITIZER_GO && (defined(__SIZEOF_INT128__) \ - || (__clang_major__ * 100 + __clang_minor__ >= 302)) && !defined(__mips64) + || (__clang_major__ * 100 + __clang_minor__ >= 302)) && \ + !defined(__mips64) && !defined(__s390x__) __extension__ typedef __int128 a128; # define __TSAN_HAS_INT128 1 #else diff --git a/libsanitizer/tsan/tsan_interface_ann.cpp b/libsanitizer/tsan/tsan_interface_ann.cpp index 99516d9..175855f 100644 --- a/libsanitizer/tsan/tsan_interface_ann.cpp +++ b/libsanitizer/tsan/tsan_interface_ann.cpp @@ -49,8 +49,6 @@ class ScopedAnnotation { return ret; \ ThreadState *thr = cur_thread(); \ const uptr caller_pc = (uptr)__builtin_return_address(0); \ - StatInc(thr, StatAnnotation); \ - StatInc(thr, Stat##typ); \ ScopedAnnotation sa(thr, __func__, caller_pc); \ const uptr pc = StackTrace::GetCurrentPc(); \ (void)pc; \ @@ -77,9 +75,7 @@ struct DynamicAnnContext { ExpectRace expect; ExpectRace benign; - DynamicAnnContext() - : mtx(MutexTypeAnnotations, StatMtxAnnotations) { - } + DynamicAnnContext() : mtx(MutexTypeAnnotations) {} }; static DynamicAnnContext *dyn_ann_ctx; diff --git a/libsanitizer/tsan/tsan_interface_atomic.cpp b/libsanitizer/tsan/tsan_interface_atomic.cpp index 3f459af..21fe4a1 100644 --- a/libsanitizer/tsan/tsan_interface_atomic.cpp +++ b/libsanitizer/tsan/tsan_interface_atomic.cpp @@ -402,34 +402,45 @@ static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { template<typename T> static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, morder mo, morder fmo) { - (void)fmo; // Unused because llvm does not pass it yet. + // 31.7.2.18: "The failure argument shall not be memory_order_release + // nor memory_order_acq_rel". LLVM (2021-05) fallbacks to Monotonic + // (mo_relaxed) when those are used. + CHECK(IsLoadOrder(fmo)); + MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); SyncVar *s = 0; - bool write_lock = mo != mo_acquire && mo != mo_consume; - if (mo != mo_relaxed) { + bool write_lock = IsReleaseOrder(mo); + + if (mo != mo_relaxed || fmo != mo_relaxed) s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock); + + T cc = *c; + T pr = func_cas(a, cc, v); + bool success = pr == cc; + if (!success) { + *c = pr; + mo = fmo; + } + + if (s) { thr->fast_state.IncrementEpoch(); // Can't increment epoch w/o writing to the trace as well. TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); - if (IsAcqRelOrder(mo)) + + if (success && IsAcqRelOrder(mo)) AcquireReleaseImpl(thr, pc, &s->clock); - else if (IsReleaseOrder(mo)) + else if (success && IsReleaseOrder(mo)) ReleaseImpl(thr, pc, &s->clock); else if (IsAcquireOrder(mo)) AcquireImpl(thr, pc, &s->clock); - } - T cc = *c; - T pr = func_cas(a, cc, v); - if (s) { + if (write_lock) s->mtx.Unlock(); else s->mtx.ReadUnlock(); } - if (pr == cc) - return true; - *c = pr; - return false; + + return success; } template<typename T> @@ -481,7 +492,6 @@ static morder convert_morder(morder mo) { const uptr callpc = (uptr)__builtin_return_address(0); \ uptr pc = StackTrace::GetCurrentPc(); \ mo = convert_morder(mo); \ - AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \ ScopedAtomic sa(thr, callpc, a, mo, __func__); \ return Atomic##func(thr, pc, __VA_ARGS__); \ /**/ @@ -502,22 +512,6 @@ class ScopedAtomic { ThreadState *thr_; }; -static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) { - StatInc(thr, StatAtomic); - StatInc(thr, t); - StatInc(thr, size == 1 ? StatAtomic1 - : size == 2 ? StatAtomic2 - : size == 4 ? StatAtomic4 - : size == 8 ? StatAtomic8 - : StatAtomic16); - StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed - : mo == mo_consume ? StatAtomicConsume - : mo == mo_acquire ? StatAtomicAcquire - : mo == mo_release ? StatAtomicRelease - : mo == mo_acq_rel ? StatAtomicAcq_Rel - : StatAtomicSeq_Cst); -} - extern "C" { SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { diff --git a/libsanitizer/tsan/tsan_mman.cpp b/libsanitizer/tsan/tsan_mman.cpp index 45a39f0..7765bc0 100644 --- a/libsanitizer/tsan/tsan_mman.cpp +++ b/libsanitizer/tsan/tsan_mman.cpp @@ -70,10 +70,7 @@ struct GlobalProc { Mutex mtx; Processor *proc; - GlobalProc() - : mtx(MutexTypeGlobalProc, StatMtxGlobalProc) - , proc(ProcCreate()) { - } + GlobalProc() : mtx(MutexTypeGlobalProc), proc(ProcCreate()) {} }; static char global_proc_placeholder[sizeof(GlobalProc)] ALIGNED(64); diff --git a/libsanitizer/tsan/tsan_mutex.cpp b/libsanitizer/tsan/tsan_mutex.cpp index 7a0918f..d8b1826 100644 --- a/libsanitizer/tsan/tsan_mutex.cpp +++ b/libsanitizer/tsan/tsan_mutex.cpp @@ -207,15 +207,12 @@ class Backoff { static const int kActiveSpinCnt = 20; }; -Mutex::Mutex(MutexType type, StatType stat_type) { +Mutex::Mutex(MutexType type) { CHECK_GT(type, MutexTypeInvalid); CHECK_LT(type, MutexTypeCount); #if SANITIZER_DEBUG type_ = type; #endif -#if TSAN_COLLECT_STATS - stat_type_ = stat_type; -#endif atomic_store(&state_, kUnlocked, memory_order_relaxed); } @@ -236,9 +233,6 @@ void Mutex::Lock() { cmp = kUnlocked; if (atomic_compare_exchange_weak(&state_, &cmp, kWriteLock, memory_order_acquire)) { -#if TSAN_COLLECT_STATS && !SANITIZER_GO - StatInc(cur_thread(), stat_type_, backoff.Contention()); -#endif return; } } @@ -264,9 +258,6 @@ void Mutex::ReadLock() { for (Backoff backoff; backoff.Do();) { prev = atomic_load(&state_, memory_order_acquire); if ((prev & kWriteLock) == 0) { -#if TSAN_COLLECT_STATS && !SANITIZER_GO - StatInc(cur_thread(), stat_type_, backoff.Contention()); -#endif return; } } diff --git a/libsanitizer/tsan/tsan_mutex.h b/libsanitizer/tsan/tsan_mutex.h index 80fdc6e..9a579ea 100644 --- a/libsanitizer/tsan/tsan_mutex.h +++ b/libsanitizer/tsan/tsan_mutex.h @@ -41,7 +41,7 @@ enum MutexType { class Mutex { public: - explicit Mutex(MutexType type, StatType stat_type); + explicit Mutex(MutexType type); ~Mutex(); void Lock(); @@ -57,9 +57,6 @@ class Mutex { #if SANITIZER_DEBUG MutexType type_; #endif -#if TSAN_COLLECT_STATS - StatType stat_type_; -#endif Mutex(const Mutex&); void operator = (const Mutex&); diff --git a/libsanitizer/tsan/tsan_platform.h b/libsanitizer/tsan/tsan_platform.h index 101522d..8bd218e 100644 --- a/libsanitizer/tsan/tsan_platform.h +++ b/libsanitizer/tsan/tsan_platform.h @@ -365,6 +365,38 @@ struct Mapping47 { // Indicates the runtime will define the memory regions at runtime. #define TSAN_RUNTIME_VMA 1 +#elif defined(__s390x__) +/* +C/C++ on linux/s390x +While the kernel provides a 64-bit address space, we have to restrict ourselves +to 48 bits due to how e.g. SyncVar::GetId() works. +0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB +0e00 0000 0000 - 4000 0000 0000: - +4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app) +8000 0000 0000 - 9000 0000 0000: - +9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) +9800 0000 0000 - a000 0000 0000: - +a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads) +b000 0000 0000 - be00 0000 0000: - +be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator) +*/ +struct Mapping { + static const uptr kMetaShadowBeg = 0x900000000000ull; + static const uptr kMetaShadowEnd = 0x980000000000ull; + static const uptr kTraceMemBeg = 0xa00000000000ull; + static const uptr kTraceMemEnd = 0xb00000000000ull; + static const uptr kShadowBeg = 0x400000000000ull; + static const uptr kShadowEnd = 0x800000000000ull; + static const uptr kHeapMemBeg = 0xbe0000000000ull; + static const uptr kHeapMemEnd = 0xc00000000000ull; + static const uptr kLoAppMemBeg = 0x000000001000ull; + static const uptr kLoAppMemEnd = 0x0e0000000000ull; + static const uptr kHiAppMemBeg = 0xc00000004000ull; + static const uptr kHiAppMemEnd = 0xc00000004000ull; + static const uptr kAppMemMsk = 0xb00000000000ull; + static const uptr kAppMemXor = 0x100000000000ull; + static const uptr kVdsoBeg = 0xfffffffff000ull; +}; #endif #elif SANITIZER_GO && !SANITIZER_WINDOWS && HAS_48_BIT_ADDRESS_SPACE @@ -528,6 +560,28 @@ struct Mapping47 { #define TSAN_RUNTIME_VMA 1 +#elif SANITIZER_GO && defined(__s390x__) +/* +Go on linux/s390x +0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB +1000 0000 0000 - 4000 0000 0000: - +4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app) +8000 0000 0000 - 9000 0000 0000: - +9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) +9800 0000 0000 - a000 0000 0000: - +a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads) +*/ +struct Mapping { + static const uptr kMetaShadowBeg = 0x900000000000ull; + static const uptr kMetaShadowEnd = 0x980000000000ull; + static const uptr kTraceMemBeg = 0xa00000000000ull; + static const uptr kTraceMemEnd = 0xb00000000000ull; + static const uptr kShadowBeg = 0x400000000000ull; + static const uptr kShadowEnd = 0x800000000000ull; + static const uptr kAppMemBeg = 0x000000001000ull; + static const uptr kAppMemEnd = 0x100000000000ull; +}; + #else # error "Unknown platform" #endif diff --git a/libsanitizer/tsan/tsan_platform_linux.cpp b/libsanitizer/tsan/tsan_platform_linux.cpp index e5b6690..cfe597e 100644 --- a/libsanitizer/tsan/tsan_platform_linux.cpp +++ b/libsanitizer/tsan/tsan_platform_linux.cpp @@ -391,6 +391,10 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) { return mangled_sp ^ xor_key; #elif defined(__mips__) return mangled_sp; +#elif defined(__s390x__) + // tcbhead_t.stack_guard + uptr xor_key = ((uptr *)__builtin_thread_pointer())[5]; + return mangled_sp ^ xor_key; #else #error "Unknown platform" #endif @@ -411,6 +415,8 @@ static uptr UnmangleLongJmpSp(uptr mangled_sp) { # define LONG_JMP_SP_ENV_SLOT 13 # elif defined(__mips64) # define LONG_JMP_SP_ENV_SLOT 1 +# elif defined(__s390x__) +# define LONG_JMP_SP_ENV_SLOT 9 # else # define LONG_JMP_SP_ENV_SLOT 6 # endif diff --git a/libsanitizer/tsan/tsan_platform_posix.cpp b/libsanitizer/tsan/tsan_platform_posix.cpp index 73e1d45..1c6198c 100644 --- a/libsanitizer/tsan/tsan_platform_posix.cpp +++ b/libsanitizer/tsan/tsan_platform_posix.cpp @@ -72,11 +72,15 @@ void InitializeShadowMemory() { InitializeShadowMemoryPlatform(); } -static void ProtectRange(uptr beg, uptr end) { +static bool TryProtectRange(uptr beg, uptr end) { CHECK_LE(beg, end); if (beg == end) - return; - if (beg != (uptr)MmapFixedNoAccess(beg, end - beg)) { + return true; + return beg == (uptr)MmapFixedNoAccess(beg, end - beg); +} + +static void ProtectRange(uptr beg, uptr end) { + if (!TryProtectRange(beg, end)) { Printf("FATAL: ThreadSanitizer can not protect [%zx,%zx]\n", beg, end); Printf("FATAL: Make sure you are not using unlimited stack\n"); Die(); @@ -118,6 +122,16 @@ void CheckAndProtect() { ProtectRange(TraceMemEnd(), HeapMemBeg()); ProtectRange(HeapEnd(), HiAppMemBeg()); #endif + +#if defined(__s390x__) + // Protect the rest of the address space. + const uptr user_addr_max_l4 = 0x0020000000000000ull; + const uptr user_addr_max_l5 = 0xfffffffffffff000ull; + // All the maintained s390x kernels support at least 4-level page tables. + ProtectRange(HiAppMemEnd(), user_addr_max_l4); + // Older s390x kernels may not support 5-level page tables. + TryProtectRange(user_addr_max_l4, user_addr_max_l5); +#endif } #endif diff --git a/libsanitizer/tsan/tsan_rtl.cpp b/libsanitizer/tsan/tsan_rtl.cpp index 0efa997..bcf489a 100644 --- a/libsanitizer/tsan/tsan_rtl.cpp +++ b/libsanitizer/tsan/tsan_rtl.cpp @@ -77,7 +77,7 @@ void OnInitialize() { } #endif -static char thread_registry_placeholder[sizeof(ThreadRegistry)]; +static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)]; static ThreadContextBase *CreateThreadContext(u32 tid) { // Map thread trace when context is created. @@ -115,15 +115,15 @@ static const u32 kThreadQuarantineSize = 64; Context::Context() : initialized(), - report_mtx(MutexTypeReport, StatMtxReport), + report_mtx(MutexTypeReport), nreported(), nmissed_expected(), thread_registry(new (thread_registry_placeholder) ThreadRegistry( CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse)), - racy_mtx(MutexTypeRacy, StatMtxRacy), + racy_mtx(MutexTypeRacy), racy_stacks(), racy_addresses(), - fired_suppressions_mtx(MutexTypeFired, StatMtxFired), + fired_suppressions_mtx(MutexTypeFired), clock_alloc(LINKER_INITIALIZED, "clock allocator") { fired_suppressions.reserve(8); } @@ -522,18 +522,14 @@ int Finalize(ThreadState *thr) { failed = OnFinalize(failed); -#if TSAN_COLLECT_STATS - StatAggregate(ctx->stat, thr->stat); - StatOutput(ctx->stat); -#endif - return failed ? common_flags()->exitcode : 0; } #if !SANITIZER_GO -void ForkBefore(ThreadState *thr, uptr pc) { +void ForkBefore(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS { ctx->thread_registry->Lock(); ctx->report_mtx.Lock(); + ScopedErrorReportLock::Lock(); // Suppress all reports in the pthread_atfork callbacks. // Reports will deadlock on the report_mtx. // We could ignore sync operations as well, @@ -545,16 +541,18 @@ void ForkBefore(ThreadState *thr, uptr pc) { thr->ignore_interceptors++; } -void ForkParentAfter(ThreadState *thr, uptr pc) { +void ForkParentAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS { thr->suppress_reports--; // Enabled in ForkBefore. thr->ignore_interceptors--; + ScopedErrorReportLock::Unlock(); ctx->report_mtx.Unlock(); ctx->thread_registry->Unlock(); } -void ForkChildAfter(ThreadState *thr, uptr pc) { +void ForkChildAfter(ThreadState *thr, uptr pc) NO_THREAD_SAFETY_ANALYSIS { thr->suppress_reports--; // Enabled in ForkBefore. thr->ignore_interceptors--; + ScopedErrorReportLock::Unlock(); ctx->report_mtx.Unlock(); ctx->thread_registry->Unlock(); @@ -693,9 +691,6 @@ ALWAYS_INLINE void MemoryAccessImpl1(ThreadState *thr, uptr addr, int kAccessSizeLog, bool kAccessIsWrite, bool kIsAtomic, u64 *shadow_mem, Shadow cur) { - StatInc(thr, StatMop); - StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); - StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); // This potentially can live in an MMX/SSE scratch register. // The required intrinsics are: @@ -752,7 +747,6 @@ void MemoryAccessImpl1(ThreadState *thr, uptr addr, return; // choose a random candidate slot and replace it StoreShadow(shadow_mem + (cur.epoch() % kShadowCnt), store_word); - StatInc(thr, StatShadowReplace); return; RACE: HandleRace(thr, shadow_mem, cur, old); @@ -891,19 +885,11 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, if (!SANITIZER_GO && !kAccessIsWrite && *shadow_mem == kShadowRodata) { // Access to .rodata section, no races here. // Measurements show that it can be 10-20% of all memory accesses. - StatInc(thr, StatMop); - StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); - StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); - StatInc(thr, StatMopRodata); return; } FastState fast_state = thr->fast_state; if (UNLIKELY(fast_state.GetIgnoreBit())) { - StatInc(thr, StatMop); - StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); - StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); - StatInc(thr, StatMopIgnored); return; } @@ -914,10 +900,6 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch, kAccessIsWrite))) { - StatInc(thr, StatMop); - StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); - StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); - StatInc(thr, StatMopSame); return; } @@ -939,10 +921,6 @@ void MemoryAccessImpl(ThreadState *thr, uptr addr, u64 *shadow_mem, Shadow cur) { if (LIKELY(ContainsSameAccess(shadow_mem, cur.raw(), thr->fast_synch_epoch, kAccessIsWrite))) { - StatInc(thr, StatMop); - StatInc(thr, kAccessIsWrite ? StatMopWrite : StatMopRead); - StatInc(thr, (StatType)(StatMop1 + kAccessSizeLog)); - StatInc(thr, StatMopSame); return; } @@ -999,7 +977,6 @@ static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, // Reset middle part. u64 *p1 = p; p = RoundDown(end, kPageSize); - UnmapOrDie((void*)p1, (uptr)p - (uptr)p1); if (!MmapFixedSuperNoReserve((uptr)p1, (uptr)p - (uptr)p1)) Die(); // Set the ending. @@ -1059,7 +1036,6 @@ void MemoryRangeImitateWriteOrResetRange(ThreadState *thr, uptr pc, uptr addr, ALWAYS_INLINE USED void FuncEntry(ThreadState *thr, uptr pc) { - StatInc(thr, StatFuncEnter); DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc); if (kCollectHistory) { thr->fast_state.IncrementEpoch(); @@ -1081,7 +1057,6 @@ void FuncEntry(ThreadState *thr, uptr pc) { ALWAYS_INLINE USED void FuncExit(ThreadState *thr) { - StatInc(thr, StatFuncExit); DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid()); if (kCollectHistory) { thr->fast_state.IncrementEpoch(); @@ -1156,12 +1131,6 @@ void build_consistency_debug() {} void build_consistency_release() {} #endif -#if TSAN_COLLECT_STATS -void build_consistency_stats() {} -#else -void build_consistency_nostats() {} -#endif - } // namespace __tsan #if !SANITIZER_GO diff --git a/libsanitizer/tsan/tsan_rtl.h b/libsanitizer/tsan/tsan_rtl.h index 3ae519d..6576d40 100644 --- a/libsanitizer/tsan/tsan_rtl.h +++ b/libsanitizer/tsan/tsan_rtl.h @@ -400,9 +400,6 @@ struct ThreadState { Vector<JmpBuf> jmp_bufs; int ignore_interceptors; #endif -#if TSAN_COLLECT_STATS - u64 stat[StatCnt]; -#endif const u32 tid; const int unique_id; bool in_symbolizer; @@ -550,7 +547,6 @@ struct Context { Flags flags; - u64 stat[StatCnt]; u64 int_alloc_cnt[MBlockTypeCount]; u64 int_alloc_siz[MBlockTypeCount]; }; @@ -658,22 +654,6 @@ void ObtainCurrentStack(ThreadState *thr, uptr toppc, StackTraceTy *stack, ObtainCurrentStack(thr, pc, &stack); \ stack.ReverseOrder(); -#if TSAN_COLLECT_STATS -void StatAggregate(u64 *dst, u64 *src); -void StatOutput(u64 *stat); -#endif - -void ALWAYS_INLINE StatInc(ThreadState *thr, StatType typ, u64 n = 1) { -#if TSAN_COLLECT_STATS - thr->stat[typ] += n; -#endif -} -void ALWAYS_INLINE StatSet(ThreadState *thr, StatType typ, u64 n) { -#if TSAN_COLLECT_STATS - thr->stat[typ] = n; -#endif -} - void MapShadow(uptr addr, uptr size); void MapThreadTrace(uptr addr, uptr size, const char *name); void DontNeedShadowFor(uptr addr, uptr size); @@ -854,7 +834,6 @@ void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, DCHECK_GE((int)typ, 0); DCHECK_LE((int)typ, 7); DCHECK_EQ(GetLsb(addr, kEventPCBits), addr); - StatInc(thr, StatEvents); u64 pos = fs.GetTracePos(); if (UNLIKELY((pos % kTracePartSize) == 0)) { #if !SANITIZER_GO diff --git a/libsanitizer/tsan/tsan_rtl_mutex.cpp b/libsanitizer/tsan/tsan_rtl_mutex.cpp index 0a8f3aa..a214a33 100644 --- a/libsanitizer/tsan/tsan_rtl_mutex.cpp +++ b/libsanitizer/tsan/tsan_rtl_mutex.cpp @@ -65,7 +65,6 @@ static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ, void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { DPrintf("#%d: MutexCreate %zx flagz=0x%x\n", thr->tid, addr, flagz); - StatInc(thr, StatMutexCreate); if (!(flagz & MutexFlagLinkerInit) && IsAppMem(addr)) { CHECK(!thr->is_freeing); thr->is_freeing = true; @@ -81,7 +80,6 @@ void MutexCreate(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { void MutexDestroy(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr); - StatInc(thr, StatMutexDestroy); SyncVar *s = ctx->metamap.GetIfExistsAndLock(addr, true); if (s == 0) return; @@ -183,11 +181,9 @@ void MutexPostLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz, int rec) { const bool first = s->recursion == 0; s->recursion += rec; if (first) { - StatInc(thr, StatMutexLock); AcquireImpl(thr, pc, &s->clock); AcquireImpl(thr, pc, &s->read_clock); } else if (!s->IsFlagSet(MutexFlagWriteReentrant)) { - StatInc(thr, StatMutexRecLock); } thr->mset.Add(s->GetId(), true, thr->fast_state.epoch()); bool pre_lock = false; @@ -229,11 +225,9 @@ int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { rec = (flagz & MutexFlagRecursiveUnlock) ? s->recursion : 1; s->recursion -= rec; if (s->recursion == 0) { - StatInc(thr, StatMutexUnlock); s->owner_tid = kInvalidTid; ReleaseStoreImpl(thr, pc, &s->clock); } else { - StatInc(thr, StatMutexRecUnlock); } } thr->mset.Del(s->GetId(), true); @@ -268,7 +262,6 @@ void MutexPreReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { DPrintf("#%d: MutexPostReadLock %zx flagz=0x%x\n", thr->tid, addr, flagz); - StatInc(thr, StatMutexReadLock); if (IsAppMem(addr)) MemoryReadAtomic(thr, pc, addr, kSizeLog1); SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, false); @@ -308,7 +301,6 @@ void MutexPostReadLock(ThreadState *thr, uptr pc, uptr addr, u32 flagz) { void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) { DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr); - StatInc(thr, StatMutexReadUnlock); if (IsAppMem(addr)) MemoryReadAtomic(thr, pc, addr, kSizeLog1); SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, addr, true); @@ -348,7 +340,6 @@ void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) { if (s->owner_tid == kInvalidTid) { // Seems to be read unlock. write = false; - StatInc(thr, StatMutexReadUnlock); thr->fast_state.IncrementEpoch(); TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId()); ReleaseImpl(thr, pc, &s->read_clock); @@ -359,11 +350,9 @@ void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) { CHECK_GT(s->recursion, 0); s->recursion--; if (s->recursion == 0) { - StatInc(thr, StatMutexUnlock); s->owner_tid = kInvalidTid; ReleaseStoreImpl(thr, pc, &s->clock); } else { - StatInc(thr, StatMutexRecUnlock); } } else if (!s->IsFlagSet(MutexFlagBroken)) { s->SetFlags(MutexFlagBroken); @@ -494,7 +483,6 @@ void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) { return; thr->clock.set(thr->fast_state.epoch()); thr->clock.acquire(&thr->proc()->clock_cache, c); - StatInc(thr, StatSyncAcquire); } void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) { @@ -503,7 +491,6 @@ void ReleaseStoreAcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) { thr->clock.set(thr->fast_state.epoch()); thr->fast_synch_epoch = thr->fast_state.epoch(); thr->clock.releaseStoreAcquire(&thr->proc()->clock_cache, c); - StatInc(thr, StatSyncReleaseStoreAcquire); } void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { @@ -512,7 +499,6 @@ void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { thr->clock.set(thr->fast_state.epoch()); thr->fast_synch_epoch = thr->fast_state.epoch(); thr->clock.release(&thr->proc()->clock_cache, c); - StatInc(thr, StatSyncRelease); } void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) { @@ -521,7 +507,6 @@ void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) { thr->clock.set(thr->fast_state.epoch()); thr->fast_synch_epoch = thr->fast_state.epoch(); thr->clock.ReleaseStore(&thr->proc()->clock_cache, c); - StatInc(thr, StatSyncRelease); } void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { @@ -530,8 +515,6 @@ void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { thr->clock.set(thr->fast_state.epoch()); thr->fast_synch_epoch = thr->fast_state.epoch(); thr->clock.acq_rel(&thr->proc()->clock_cache, c); - StatInc(thr, StatSyncAcquire); - StatInc(thr, StatSyncRelease); } void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) { diff --git a/libsanitizer/tsan/tsan_rtl_ppc64.S b/libsanitizer/tsan/tsan_rtl_ppc64.S index 9e533a7..8285e21 100644 --- a/libsanitizer/tsan/tsan_rtl_ppc64.S +++ b/libsanitizer/tsan/tsan_rtl_ppc64.S @@ -1,6 +1,5 @@ #include "tsan_ppc_regs.h" - .machine altivec .section .text .hidden __tsan_setjmp .globl _setjmp diff --git a/libsanitizer/tsan/tsan_rtl_s390x.S b/libsanitizer/tsan/tsan_rtl_s390x.S new file mode 100644 index 0000000..fcff35f --- /dev/null +++ b/libsanitizer/tsan/tsan_rtl_s390x.S @@ -0,0 +1,47 @@ +#include "sanitizer_common/sanitizer_asm.h" + +#define CFA_OFFSET 160 +#define R2_REL_OFFSET 16 +#define R3_REL_OFFSET 24 +#define R14_REL_OFFSET 112 +#define R15_REL_OFFSET 120 +#define FRAME_SIZE 160 + +.text + +ASM_HIDDEN(__tsan_setjmp) + +.macro intercept symbol, real +.comm \real, 8, 8 +.globl ASM_SYMBOL_INTERCEPTOR(\symbol) +ASM_TYPE_FUNCTION(ASM_SYMBOL_INTERCEPTOR(\symbol)) +ASM_SYMBOL_INTERCEPTOR(\symbol): + CFI_STARTPROC + stmg %r2, %r3, R2_REL_OFFSET(%r15) + CFI_REL_OFFSET(%r2, R2_REL_OFFSET) + CFI_REL_OFFSET(%r3, R3_REL_OFFSET) + stmg %r14, %r15, R14_REL_OFFSET(%r15) + CFI_REL_OFFSET(%r14, R14_REL_OFFSET) + CFI_REL_OFFSET(%r15, R15_REL_OFFSET) + aghi %r15, -FRAME_SIZE + CFI_ADJUST_CFA_OFFSET(FRAME_SIZE) + la %r2, FRAME_SIZE(%r15) + brasl %r14, ASM_SYMBOL(__tsan_setjmp) + lmg %r14, %r15, FRAME_SIZE + R14_REL_OFFSET(%r15) + CFI_RESTORE(%r14) + CFI_RESTORE(%r15) + CFI_DEF_CFA_OFFSET(CFA_OFFSET) + lmg %r2, %r3, R2_REL_OFFSET(%r15) + CFI_RESTORE(%r2) + CFI_RESTORE(%r3) + larl %r1, \real + lg %r1, 0(%r1) + br %r1 + CFI_ENDPROC + ASM_SIZE(ASM_SYMBOL_INTERCEPTOR(\symbol)) +.endm + +intercept setjmp, _ZN14__interception11real_setjmpE +intercept _setjmp, _ZN14__interception12real__setjmpE +intercept sigsetjmp, _ZN14__interception14real_sigsetjmpE +intercept __sigsetjmp, _ZN14__interception16real___sigsetjmpE diff --git a/libsanitizer/tsan/tsan_rtl_thread.cpp b/libsanitizer/tsan/tsan_rtl_thread.cpp index 6d1ccd8..cdb6e60 100644 --- a/libsanitizer/tsan/tsan_rtl_thread.cpp +++ b/libsanitizer/tsan/tsan_rtl_thread.cpp @@ -61,8 +61,6 @@ void ThreadContext::OnCreated(void *arg) { TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0); ReleaseImpl(args->thr, 0, &sync); creation_stack_id = CurrentStackId(args->thr, args->pc); - if (reuse_count == 0) - StatInc(args->thr, StatThreadMaxTid); } void ThreadContext::OnReset() { @@ -115,7 +113,6 @@ void ThreadContext::OnStarted(void *arg) { thr->fast_synch_epoch = epoch0; AcquireImpl(thr, 0, &sync); - StatInc(thr, StatSyncAcquire); sync.Reset(&thr->proc()->clock_cache); thr->is_inited = true; DPrintf("#%d: ThreadStart epoch=%zu stk_addr=%zx stk_size=%zx " @@ -149,9 +146,6 @@ void ThreadContext::OnFinished() { PlatformCleanUpThreadState(thr); #endif thr->~ThreadState(); -#if TSAN_COLLECT_STATS - StatAggregate(ctx->stat, thr->stat); -#endif thr = 0; } @@ -232,13 +226,11 @@ int ThreadCount(ThreadState *thr) { } int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) { - StatInc(thr, StatThreadCreate); OnCreatedArgs args = { thr, pc }; u32 parent_tid = thr ? thr->tid : kInvalidTid; // No parent for GCD workers. int tid = ctx->thread_registry->CreateThread(uid, detached, parent_tid, &args); DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", parent_tid, tid, uid); - StatSet(thr, StatThreadMaxAlive, ctx->thread_registry->GetMaxAliveThreads()); return tid; } @@ -280,7 +272,6 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id, void ThreadFinish(ThreadState *thr) { ThreadCheckIgnore(thr); - StatInc(thr, StatThreadFinish); if (thr->stk_addr && thr->stk_size) DontNeedShadowFor(thr->stk_addr, thr->stk_size); if (thr->tls_addr && thr->tls_size) @@ -372,13 +363,10 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, } #endif - StatInc(thr, StatMopRange); - if (*shadow_mem == kShadowRodata) { DCHECK(!is_write); // Access to .rodata section, no races here. // Measurements show that it can be 10-20% of all memory accesses. - StatInc(thr, StatMopRangeRodata); return; } diff --git a/libsanitizer/tsan/tsan_stack_trace.cpp b/libsanitizer/tsan/tsan_stack_trace.cpp index 403a21a..6c703d7 100644 --- a/libsanitizer/tsan/tsan_stack_trace.cpp +++ b/libsanitizer/tsan/tsan_stack_trace.cpp @@ -54,10 +54,8 @@ void __sanitizer::BufferedStackTrace::UnwindImpl( uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { uptr top = 0; uptr bottom = 0; - if (StackTrace::WillUseFastUnwind(request_fast)) { - GetThreadStackTopAndBottom(false, &top, &bottom); - Unwind(max_depth, pc, bp, nullptr, top, bottom, true); - } else - Unwind(max_depth, pc, 0, context, 0, 0, false); + GetThreadStackTopAndBottom(false, &top, &bottom); + bool fast = StackTrace::WillUseFastUnwind(request_fast); + Unwind(max_depth, pc, bp, context, top, bottom, fast); } #endif // SANITIZER_GO diff --git a/libsanitizer/tsan/tsan_stat.cpp b/libsanitizer/tsan/tsan_stat.cpp deleted file mode 100644 index 78f3cce..0000000 --- a/libsanitizer/tsan/tsan_stat.cpp +++ /dev/null @@ -1,186 +0,0 @@ -//===-- tsan_stat.cpp -----------------------------------------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is a part of ThreadSanitizer (TSan), a race detector. -// -//===----------------------------------------------------------------------===// -#include "tsan_stat.h" -#include "tsan_rtl.h" - -namespace __tsan { - -#if TSAN_COLLECT_STATS - -void StatAggregate(u64 *dst, u64 *src) { - for (int i = 0; i < StatCnt; i++) - dst[i] += src[i]; -} - -void StatOutput(u64 *stat) { - stat[StatShadowNonZero] = stat[StatShadowProcessed] - stat[StatShadowZero]; - - static const char *name[StatCnt] = {}; - name[StatMop] = "Memory accesses "; - name[StatMopRead] = " Including reads "; - name[StatMopWrite] = " writes "; - name[StatMop1] = " Including size 1 "; - name[StatMop2] = " size 2 "; - name[StatMop4] = " size 4 "; - name[StatMop8] = " size 8 "; - name[StatMopSame] = " Including same "; - name[StatMopIgnored] = " Including ignored "; - name[StatMopRange] = " Including range "; - name[StatMopRodata] = " Including .rodata "; - name[StatMopRangeRodata] = " Including .rodata range "; - name[StatShadowProcessed] = "Shadow processed "; - name[StatShadowZero] = " Including empty "; - name[StatShadowNonZero] = " Including non empty "; - name[StatShadowSameSize] = " Including same size "; - name[StatShadowIntersect] = " intersect "; - name[StatShadowNotIntersect] = " not intersect "; - name[StatShadowSameThread] = " Including same thread "; - name[StatShadowAnotherThread] = " another thread "; - name[StatShadowReplace] = " Including evicted "; - - name[StatFuncEnter] = "Function entries "; - name[StatFuncExit] = "Function exits "; - name[StatEvents] = "Events collected "; - - name[StatThreadCreate] = "Total threads created "; - name[StatThreadFinish] = " threads finished "; - name[StatThreadReuse] = " threads reused "; - name[StatThreadMaxTid] = " max tid "; - name[StatThreadMaxAlive] = " max alive threads "; - - name[StatMutexCreate] = "Mutexes created "; - name[StatMutexDestroy] = " destroyed "; - name[StatMutexLock] = " lock "; - name[StatMutexUnlock] = " unlock "; - name[StatMutexRecLock] = " recursive lock "; - name[StatMutexRecUnlock] = " recursive unlock "; - name[StatMutexReadLock] = " read lock "; - name[StatMutexReadUnlock] = " read unlock "; - - name[StatSyncCreated] = "Sync objects created "; - name[StatSyncDestroyed] = " destroyed "; - name[StatSyncAcquire] = " acquired "; - name[StatSyncRelease] = " released "; - - name[StatClockAcquire] = "Clock acquire "; - name[StatClockAcquireEmpty] = " empty clock "; - name[StatClockAcquireFastRelease] = " fast from release-store "; - name[StatClockAcquireFull] = " full (slow) "; - name[StatClockAcquiredSomething] = " acquired something "; - name[StatClockRelease] = "Clock release "; - name[StatClockReleaseResize] = " resize "; - name[StatClockReleaseFast] = " fast "; - name[StatClockReleaseSlow] = " dirty overflow (slow) "; - name[StatClockReleaseFull] = " full (slow) "; - name[StatClockReleaseAcquired] = " was acquired "; - name[StatClockReleaseClearTail] = " clear tail "; - name[StatClockStore] = "Clock release store "; - name[StatClockStoreResize] = " resize "; - name[StatClockStoreFast] = " fast "; - name[StatClockStoreFull] = " slow "; - name[StatClockStoreTail] = " clear tail "; - name[StatClockAcquireRelease] = "Clock acquire-release "; - - name[StatAtomic] = "Atomic operations "; - name[StatAtomicLoad] = " Including load "; - name[StatAtomicStore] = " store "; - name[StatAtomicExchange] = " exchange "; - name[StatAtomicFetchAdd] = " fetch_add "; - name[StatAtomicFetchSub] = " fetch_sub "; - name[StatAtomicFetchAnd] = " fetch_and "; - name[StatAtomicFetchOr] = " fetch_or "; - name[StatAtomicFetchXor] = " fetch_xor "; - name[StatAtomicFetchNand] = " fetch_nand "; - name[StatAtomicCAS] = " compare_exchange "; - name[StatAtomicFence] = " fence "; - name[StatAtomicRelaxed] = " Including relaxed "; - name[StatAtomicConsume] = " consume "; - name[StatAtomicAcquire] = " acquire "; - name[StatAtomicRelease] = " release "; - name[StatAtomicAcq_Rel] = " acq_rel "; - name[StatAtomicSeq_Cst] = " seq_cst "; - name[StatAtomic1] = " Including size 1 "; - name[StatAtomic2] = " size 2 "; - name[StatAtomic4] = " size 4 "; - name[StatAtomic8] = " size 8 "; - name[StatAtomic16] = " size 16 "; - - name[StatAnnotation] = "Dynamic annotations "; - name[StatAnnotateHappensBefore] = " HappensBefore "; - name[StatAnnotateHappensAfter] = " HappensAfter "; - name[StatAnnotateCondVarSignal] = " CondVarSignal "; - name[StatAnnotateCondVarSignalAll] = " CondVarSignalAll "; - name[StatAnnotateMutexIsNotPHB] = " MutexIsNotPHB "; - name[StatAnnotateCondVarWait] = " CondVarWait "; - name[StatAnnotateRWLockCreate] = " RWLockCreate "; - name[StatAnnotateRWLockCreateStatic] = " StatAnnotateRWLockCreateStatic "; - name[StatAnnotateRWLockDestroy] = " RWLockDestroy "; - name[StatAnnotateRWLockAcquired] = " RWLockAcquired "; - name[StatAnnotateRWLockReleased] = " RWLockReleased "; - name[StatAnnotateTraceMemory] = " TraceMemory "; - name[StatAnnotateFlushState] = " FlushState "; - name[StatAnnotateNewMemory] = " NewMemory "; - name[StatAnnotateNoOp] = " NoOp "; - name[StatAnnotateFlushExpectedRaces] = " FlushExpectedRaces "; - name[StatAnnotateEnableRaceDetection] = " EnableRaceDetection "; - name[StatAnnotateMutexIsUsedAsCondVar] = " MutexIsUsedAsCondVar "; - name[StatAnnotatePCQGet] = " PCQGet "; - name[StatAnnotatePCQPut] = " PCQPut "; - name[StatAnnotatePCQDestroy] = " PCQDestroy "; - name[StatAnnotatePCQCreate] = " PCQCreate "; - name[StatAnnotateExpectRace] = " ExpectRace "; - name[StatAnnotateBenignRaceSized] = " BenignRaceSized "; - name[StatAnnotateBenignRace] = " BenignRace "; - name[StatAnnotateIgnoreReadsBegin] = " IgnoreReadsBegin "; - name[StatAnnotateIgnoreReadsEnd] = " IgnoreReadsEnd "; - name[StatAnnotateIgnoreWritesBegin] = " IgnoreWritesBegin "; - name[StatAnnotateIgnoreWritesEnd] = " IgnoreWritesEnd "; - name[StatAnnotateIgnoreSyncBegin] = " IgnoreSyncBegin "; - name[StatAnnotateIgnoreSyncEnd] = " IgnoreSyncEnd "; - name[StatAnnotatePublishMemoryRange] = " PublishMemoryRange "; - name[StatAnnotateUnpublishMemoryRange] = " UnpublishMemoryRange "; - name[StatAnnotateThreadName] = " ThreadName "; - name[Stat__tsan_mutex_create] = " __tsan_mutex_create "; - name[Stat__tsan_mutex_destroy] = " __tsan_mutex_destroy "; - name[Stat__tsan_mutex_pre_lock] = " __tsan_mutex_pre_lock "; - name[Stat__tsan_mutex_post_lock] = " __tsan_mutex_post_lock "; - name[Stat__tsan_mutex_pre_unlock] = " __tsan_mutex_pre_unlock "; - name[Stat__tsan_mutex_post_unlock] = " __tsan_mutex_post_unlock "; - name[Stat__tsan_mutex_pre_signal] = " __tsan_mutex_pre_signal "; - name[Stat__tsan_mutex_post_signal] = " __tsan_mutex_post_signal "; - name[Stat__tsan_mutex_pre_divert] = " __tsan_mutex_pre_divert "; - name[Stat__tsan_mutex_post_divert] = " __tsan_mutex_post_divert "; - - name[StatMtxTotal] = "Contentionz "; - name[StatMtxTrace] = " Trace "; - name[StatMtxThreads] = " Threads "; - name[StatMtxReport] = " Report "; - name[StatMtxSyncVar] = " SyncVar "; - name[StatMtxSyncTab] = " SyncTab "; - name[StatMtxSlab] = " Slab "; - name[StatMtxAtExit] = " Atexit "; - name[StatMtxAnnotations] = " Annotations "; - name[StatMtxMBlock] = " MBlock "; - name[StatMtxDeadlockDetector] = " DeadlockDetector "; - name[StatMtxFired] = " FiredSuppressions "; - name[StatMtxRacy] = " RacyStacks "; - name[StatMtxFD] = " FD "; - name[StatMtxGlobalProc] = " GlobalProc "; - - Printf("Statistics:\n"); - for (int i = 0; i < StatCnt; i++) - Printf("%s: %16zu\n", name[i], (uptr)stat[i]); -} - -#endif - -} // namespace __tsan diff --git a/libsanitizer/tsan/tsan_stat.h b/libsanitizer/tsan/tsan_stat.h deleted file mode 100644 index 8b26a59..0000000 --- a/libsanitizer/tsan/tsan_stat.h +++ /dev/null @@ -1,191 +0,0 @@ -//===-- tsan_stat.h ---------------------------------------------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// This file is a part of ThreadSanitizer (TSan), a race detector. -// -//===----------------------------------------------------------------------===// - -#ifndef TSAN_STAT_H -#define TSAN_STAT_H - -namespace __tsan { - -enum StatType { - // Memory access processing related stuff. - StatMop, - StatMopRead, - StatMopWrite, - StatMop1, // These must be consequtive. - StatMop2, - StatMop4, - StatMop8, - StatMopSame, - StatMopIgnored, - StatMopRange, - StatMopRodata, - StatMopRangeRodata, - StatShadowProcessed, - StatShadowZero, - StatShadowNonZero, // Derived. - StatShadowSameSize, - StatShadowIntersect, - StatShadowNotIntersect, - StatShadowSameThread, - StatShadowAnotherThread, - StatShadowReplace, - - // Func processing. - StatFuncEnter, - StatFuncExit, - - // Trace processing. - StatEvents, - - // Threads. - StatThreadCreate, - StatThreadFinish, - StatThreadReuse, - StatThreadMaxTid, - StatThreadMaxAlive, - - // Mutexes. - StatMutexCreate, - StatMutexDestroy, - StatMutexLock, - StatMutexUnlock, - StatMutexRecLock, - StatMutexRecUnlock, - StatMutexReadLock, - StatMutexReadUnlock, - - // Synchronization. - StatSyncCreated, - StatSyncDestroyed, - StatSyncAcquire, - StatSyncRelease, - StatSyncReleaseStoreAcquire, - - // Clocks - acquire. - StatClockAcquire, - StatClockAcquireEmpty, - StatClockAcquireFastRelease, - StatClockAcquireFull, - StatClockAcquiredSomething, - // Clocks - release. - StatClockRelease, - StatClockReleaseResize, - StatClockReleaseFast, - StatClockReleaseSlow, - StatClockReleaseFull, - StatClockReleaseAcquired, - StatClockReleaseClearTail, - // Clocks - release store. - StatClockStore, - StatClockStoreResize, - StatClockStoreFast, - StatClockStoreFull, - StatClockStoreTail, - // Clocks - acquire-release. - StatClockAcquireRelease, - - // Atomics. - StatAtomic, - StatAtomicLoad, - StatAtomicStore, - StatAtomicExchange, - StatAtomicFetchAdd, - StatAtomicFetchSub, - StatAtomicFetchAnd, - StatAtomicFetchOr, - StatAtomicFetchXor, - StatAtomicFetchNand, - StatAtomicCAS, - StatAtomicFence, - StatAtomicRelaxed, - StatAtomicConsume, - StatAtomicAcquire, - StatAtomicRelease, - StatAtomicAcq_Rel, - StatAtomicSeq_Cst, - StatAtomic1, - StatAtomic2, - StatAtomic4, - StatAtomic8, - StatAtomic16, - - // Dynamic annotations. - StatAnnotation, - StatAnnotateHappensBefore, - StatAnnotateHappensAfter, - StatAnnotateCondVarSignal, - StatAnnotateCondVarSignalAll, - StatAnnotateMutexIsNotPHB, - StatAnnotateCondVarWait, - StatAnnotateRWLockCreate, - StatAnnotateRWLockCreateStatic, - StatAnnotateRWLockDestroy, - StatAnnotateRWLockAcquired, - StatAnnotateRWLockReleased, - StatAnnotateTraceMemory, - StatAnnotateFlushState, - StatAnnotateNewMemory, - StatAnnotateNoOp, - StatAnnotateFlushExpectedRaces, - StatAnnotateEnableRaceDetection, - StatAnnotateMutexIsUsedAsCondVar, - StatAnnotatePCQGet, - StatAnnotatePCQPut, - StatAnnotatePCQDestroy, - StatAnnotatePCQCreate, - StatAnnotateExpectRace, - StatAnnotateBenignRaceSized, - StatAnnotateBenignRace, - StatAnnotateIgnoreReadsBegin, - StatAnnotateIgnoreReadsEnd, - StatAnnotateIgnoreWritesBegin, - StatAnnotateIgnoreWritesEnd, - StatAnnotateIgnoreSyncBegin, - StatAnnotateIgnoreSyncEnd, - StatAnnotatePublishMemoryRange, - StatAnnotateUnpublishMemoryRange, - StatAnnotateThreadName, - Stat__tsan_mutex_create, - Stat__tsan_mutex_destroy, - Stat__tsan_mutex_pre_lock, - Stat__tsan_mutex_post_lock, - Stat__tsan_mutex_pre_unlock, - Stat__tsan_mutex_post_unlock, - Stat__tsan_mutex_pre_signal, - Stat__tsan_mutex_post_signal, - Stat__tsan_mutex_pre_divert, - Stat__tsan_mutex_post_divert, - - // Internal mutex contentionz. - StatMtxTotal, - StatMtxTrace, - StatMtxThreads, - StatMtxReport, - StatMtxSyncVar, - StatMtxSyncTab, - StatMtxSlab, - StatMtxAnnotations, - StatMtxAtExit, - StatMtxMBlock, - StatMtxDeadlockDetector, - StatMtxFired, - StatMtxRacy, - StatMtxFD, - StatMtxGlobalProc, - - // This must be the last. - StatCnt -}; - -} // namespace __tsan - -#endif // TSAN_STAT_H diff --git a/libsanitizer/tsan/tsan_sync.cpp b/libsanitizer/tsan/tsan_sync.cpp index ba24f98..d25434a 100644 --- a/libsanitizer/tsan/tsan_sync.cpp +++ b/libsanitizer/tsan/tsan_sync.cpp @@ -18,10 +18,7 @@ namespace __tsan { void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s); -SyncVar::SyncVar() - : mtx(MutexTypeSyncVar, StatMtxSyncVar) { - Reset(0); -} +SyncVar::SyncVar() : mtx(MutexTypeSyncVar) { Reset(0); } void SyncVar::Init(ThreadState *thr, uptr pc, uptr addr, u64 uid) { this->addr = addr; diff --git a/libsanitizer/tsan/tsan_trace.h b/libsanitizer/tsan/tsan_trace.h index fbd0f72..9f2677b 100644 --- a/libsanitizer/tsan/tsan_trace.h +++ b/libsanitizer/tsan/tsan_trace.h @@ -65,9 +65,7 @@ struct Trace { // CreateThreadContext. TraceHeader headers[kTraceParts]; - Trace() - : mtx(MutexTypeTrace, StatMtxTrace) { - } + Trace() : mtx(MutexTypeTrace) {} }; } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_update_shadow_word_inl.h b/libsanitizer/tsan/tsan_update_shadow_word_inl.h index 056c3aa..d23dfb0 100644 --- a/libsanitizer/tsan/tsan_update_shadow_word_inl.h +++ b/libsanitizer/tsan/tsan_update_shadow_word_inl.h @@ -13,12 +13,10 @@ // produce sligtly less efficient code. //===----------------------------------------------------------------------===// do { - StatInc(thr, StatShadowProcessed); const unsigned kAccessSize = 1 << kAccessSizeLog; u64 *sp = &shadow_mem[idx]; old = LoadShadow(sp); if (LIKELY(old.IsZero())) { - StatInc(thr, StatShadowZero); if (!stored) { StoreIfNotYetStored(sp, &store_word); stored = true; @@ -27,17 +25,14 @@ do { } // is the memory access equal to the previous? if (LIKELY(Shadow::Addr0AndSizeAreEqual(cur, old))) { - StatInc(thr, StatShadowSameSize); // same thread? if (LIKELY(Shadow::TidsAreEqual(old, cur))) { - StatInc(thr, StatShadowSameThread); if (LIKELY(old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))) { StoreIfNotYetStored(sp, &store_word); stored = true; } break; } - StatInc(thr, StatShadowAnotherThread); if (HappensBefore(old, thr)) { if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic)) { StoreIfNotYetStored(sp, &store_word); @@ -51,12 +46,8 @@ do { } // Do the memory access intersect? if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) { - StatInc(thr, StatShadowIntersect); - if (Shadow::TidsAreEqual(old, cur)) { - StatInc(thr, StatShadowSameThread); + if (Shadow::TidsAreEqual(old, cur)) break; - } - StatInc(thr, StatShadowAnotherThread); if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic)) break; if (LIKELY(HappensBefore(old, thr))) @@ -64,6 +55,5 @@ do { goto RACE; } // The accesses do not intersect. - StatInc(thr, StatShadowNotIntersect); break; } while (0); diff --git a/libsanitizer/ubsan/ubsan_diag_standalone.cpp b/libsanitizer/ubsan/ubsan_diag_standalone.cpp index 300179a..5526ae0 100644 --- a/libsanitizer/ubsan/ubsan_diag_standalone.cpp +++ b/libsanitizer/ubsan/ubsan_diag_standalone.cpp @@ -20,11 +20,9 @@ void __sanitizer::BufferedStackTrace::UnwindImpl( uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { uptr top = 0; uptr bottom = 0; - if (StackTrace::WillUseFastUnwind(request_fast)) { - GetThreadStackTopAndBottom(false, &top, &bottom); - Unwind(max_depth, pc, bp, nullptr, top, bottom, true); - } else - Unwind(max_depth, pc, bp, context, 0, 0, false); + GetThreadStackTopAndBottom(false, &top, &bottom); + bool fast = StackTrace::WillUseFastUnwind(request_fast); + Unwind(max_depth, pc, bp, context, top, bottom, fast); } extern "C" { diff --git a/libsanitizer/ubsan/ubsan_flags.cpp b/libsanitizer/ubsan/ubsan_flags.cpp index 9a66bd3..25cefd4 100644 --- a/libsanitizer/ubsan/ubsan_flags.cpp +++ b/libsanitizer/ubsan/ubsan_flags.cpp @@ -50,7 +50,6 @@ void InitializeFlags() { { CommonFlags cf; cf.CopyFrom(*common_flags()); - cf.print_summary = false; cf.external_symbolizer_path = GetFlag("UBSAN_SYMBOLIZER_PATH"); OverrideCommonFlags(cf); } diff --git a/libsanitizer/ubsan/ubsan_handlers.cpp b/libsanitizer/ubsan/ubsan_handlers.cpp index 2184625..e201e6b 100644 --- a/libsanitizer/ubsan/ubsan_handlers.cpp +++ b/libsanitizer/ubsan/ubsan_handlers.cpp @@ -894,21 +894,6 @@ void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, } // namespace __ubsan -void __ubsan::__ubsan_handle_cfi_bad_icall(CFIBadIcallData *CallData, - ValueHandle Function) { - GET_REPORT_OPTIONS(false); - CFICheckFailData Data = {CFITCK_ICall, CallData->Loc, CallData->Type}; - handleCFIBadIcall(&Data, Function, Opts); -} - -void __ubsan::__ubsan_handle_cfi_bad_icall_abort(CFIBadIcallData *CallData, - ValueHandle Function) { - GET_REPORT_OPTIONS(true); - CFICheckFailData Data = {CFITCK_ICall, CallData->Loc, CallData->Type}; - handleCFIBadIcall(&Data, Function, Opts); - Die(); -} - void __ubsan::__ubsan_handle_cfi_check_fail(CFICheckFailData *Data, ValueHandle Value, uptr ValidVtable) { diff --git a/libsanitizer/ubsan/ubsan_handlers.h b/libsanitizer/ubsan/ubsan_handlers.h index 9f41235..219fb15 100644 --- a/libsanitizer/ubsan/ubsan_handlers.h +++ b/libsanitizer/ubsan/ubsan_handlers.h @@ -215,20 +215,12 @@ enum CFITypeCheckKind : unsigned char { CFITCK_VMFCall, }; -struct CFIBadIcallData { - SourceLocation Loc; - const TypeDescriptor &Type; -}; - struct CFICheckFailData { CFITypeCheckKind CheckKind; SourceLocation Loc; const TypeDescriptor &Type; }; -/// \brief Handle control flow integrity failure for indirect function calls. -RECOVERABLE(cfi_bad_icall, CFIBadIcallData *Data, ValueHandle Function) - /// \brief Handle control flow integrity failures. RECOVERABLE(cfi_check_fail, CFICheckFailData *Data, ValueHandle Function, uptr VtableIsValid) diff --git a/libsanitizer/ubsan/ubsan_platform.h b/libsanitizer/ubsan/ubsan_platform.h index 42944a0..d2cc2e1 100644 --- a/libsanitizer/ubsan/ubsan_platform.h +++ b/libsanitizer/ubsan/ubsan_platform.h @@ -12,16 +12,14 @@ #ifndef UBSAN_PLATFORM_H #define UBSAN_PLATFORM_H -#ifndef CAN_SANITIZE_UB // Other platforms should be easy to add, and probably work as-is. #if defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) || \ - defined(__NetBSD__) || defined(__DragonFly__) || \ - (defined(__sun__) && defined(__svr4__)) || \ - defined(_WIN32) || defined(__Fuchsia__) || defined(__rtems__) -# define CAN_SANITIZE_UB 1 + defined(__NetBSD__) || defined(__DragonFly__) || \ + (defined(__sun__) && defined(__svr4__)) || defined(_WIN32) || \ + defined(__Fuchsia__) +#define CAN_SANITIZE_UB 1 #else # define CAN_SANITIZE_UB 0 #endif -#endif //CAN_SANITIZE_UB #endif |