libsanitizer: merge from upstream (87e6e490e79384a5)

6 files changed, 109 insertions, 58 deletions

diff --git a/libsanitizer/hwasan/hwasan.cpp b/libsanitizer/hwasan/hwasan.cpp
index 097136f..26aae9b 100644
--- a/libsanitizer/hwasan/hwasan.cpp
+++ b/libsanitizer/hwasan/hwasan.cpp
@@ -445,16 +445,32 @@ void __hwasan_print_shadow(const void *p, uptr sz) {
 sptr __hwasan_test_shadow(const void *p, uptr sz) {
   if (sz == 0)
     return -1;
-  tag_t ptr_tag = GetTagFromPointer((uptr)p);
-  uptr ptr_raw = UntagAddr(reinterpret_cast<uptr>(p));
+  uptr ptr = reinterpret_cast<uptr>(p);
+  tag_t ptr_tag = GetTagFromPointer(ptr);
+  uptr ptr_raw = UntagAddr(ptr);
   uptr shadow_first = MemToShadow(ptr_raw);
-  uptr shadow_last = MemToShadow(ptr_raw + sz - 1);
-  for (uptr s = shadow_first; s <= shadow_last; ++s)
-    if (*(tag_t *)s != ptr_tag) {
-      sptr offset = ShadowToMem(s) - ptr_raw;
+  uptr shadow_last = MemToShadow(ptr_raw + sz);
+  for (uptr s = shadow_first; s < shadow_last; ++s) {
+    if (UNLIKELY(*(tag_t *)s != ptr_tag)) {
+      uptr short_size =
+          ShortTagSize(*(tag_t *)s, AddTagToPointer(ShadowToMem(s), ptr_tag));
+      sptr offset = ShadowToMem(s) - ptr_raw + short_size;
       return offset < 0 ? 0 : offset;
     }
-  return -1;
+  }
+
+  uptr end = ptr + sz;
+  uptr tail_sz = end & (kShadowAlignment - 1);
+  if (!tail_sz)
+    return -1;
+
+  uptr short_size =
+      ShortTagSize(*(tag_t *)shadow_last, end & ~(kShadowAlignment - 1));
+  if (LIKELY(tail_sz <= short_size))
+    return -1;
+
+  sptr offset = sz - tail_sz + short_size;
+  return offset < 0 ? 0 : offset;
 }
 
 u16 __sanitizer_unaligned_load16(const uu16 *p) {
diff --git a/libsanitizer/hwasan/hwasan.h b/libsanitizer/hwasan/hwasan.h
index c3d71a2..37ef482 100644
--- a/libsanitizer/hwasan/hwasan.h
+++ b/libsanitizer/hwasan/hwasan.h
@@ -16,6 +16,7 @@
 
 #include "hwasan_flags.h"
 #include "hwasan_interface_internal.h"
+#include "hwasan_mapping.h"
 #include "sanitizer_common/sanitizer_common.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
@@ -78,12 +79,23 @@ const unsigned kRecordFPShift = 48;
 const unsigned kRecordFPLShift = 4;
 const unsigned kRecordFPModulus = 1 << (64 - kRecordFPShift + kRecordFPLShift);
 
+static inline bool InTaggableRegion(uptr addr) {
+#if defined(HWASAN_ALIASING_MODE)
+  // Aliases are mapped next to shadow so that the upper bits match the shadow
+  // base.
+  return (addr >> kTaggableRegionCheckShift) ==
+         (__hwasan::GetShadowOffset() >> kTaggableRegionCheckShift);
+#endif
+  return true;
+}
+
 static inline tag_t GetTagFromPointer(uptr p) {
-  return (p >> kAddressTagShift) & kTagMask;
+  return InTaggableRegion(p) ? ((p >> kAddressTagShift) & kTagMask) : 0;
 }
 
 static inline uptr UntagAddr(uptr tagged_addr) {
-  return tagged_addr & ~kAddressTagMask;
+  return InTaggableRegion(tagged_addr) ? (tagged_addr & ~kAddressTagMask)
+                                       : tagged_addr;
 }
 
 static inline void *UntagPtr(const void *tagged_ptr) {
@@ -92,7 +104,9 @@ static inline void *UntagPtr(const void *tagged_ptr) {
 }
 
 static inline uptr AddTagToPointer(uptr p, tag_t tag) {
-  return (p & ~kAddressTagMask) | ((uptr)tag << kAddressTagShift);
+  return InTaggableRegion(p)
+             ? ((p & ~kAddressTagMask) | ((uptr)tag << kAddressTagShift))
+             : p;
 }
 
 namespace __hwasan {
diff --git a/libsanitizer/hwasan/hwasan_allocator.cpp b/libsanitizer/hwasan/hwasan_allocator.cpp
index d3cb5c8..3b59741 100644
--- a/libsanitizer/hwasan/hwasan_allocator.cpp
+++ b/libsanitizer/hwasan/hwasan_allocator.cpp
@@ -213,7 +213,10 @@ static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
     ReportOutOfMemory(size, stack);
   }
   if (zeroise) {
-    internal_memset(allocated, 0, size);
+    // The secondary allocator mmaps memory, which should be zero-inited so we
+    // don't need to explicitly clear it.
+    if (allocator.FromPrimary(allocated))
+      internal_memset(allocated, 0, size);
   } else if (flags()->max_malloc_fill_size > 0) {
     uptr fill_size = Min(size, (uptr)flags()->max_malloc_fill_size);
     internal_memset(allocated, flags()->malloc_fill_byte, fill_size);
@@ -287,9 +290,7 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
   CHECK(tagged_ptr);
   RunFreeHooks(tagged_ptr);
 
-  bool in_taggable_region =
-      InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr));
-  void *untagged_ptr = in_taggable_region ? UntagPtr(tagged_ptr) : tagged_ptr;
+  void *untagged_ptr = UntagPtr(tagged_ptr);
 
   if (CheckInvalidFree(stack, untagged_ptr, tagged_ptr))
     return;
@@ -308,6 +309,9 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
   u32 alloc_context_id = meta->GetAllocStackId();
   u32 alloc_thread_id = meta->GetAllocThreadId();
 
+  bool in_taggable_region =
+      InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr));
+
   // Check tail magic.
   uptr tagged_size = TaggedSize(orig_size);
   if (flags()->free_checks_tail_magic && orig_size &&
@@ -370,10 +374,7 @@ static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
 
 static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old,
                               uptr new_size, uptr alignment) {
-  void *untagged_ptr_old =
-      InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr_old))
-          ? UntagPtr(tagged_ptr_old)
-          : tagged_ptr_old;
+  void *untagged_ptr_old = UntagPtr(tagged_ptr_old);
   if (CheckInvalidFree(stack, untagged_ptr_old, tagged_ptr_old))
     return nullptr;
   void *tagged_ptr_new =
@@ -381,9 +382,9 @@ static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old,
   if (tagged_ptr_old && tagged_ptr_new) {
     Metadata *meta =
         reinterpret_cast<Metadata *>(allocator.GetMetaData(untagged_ptr_old));
-    internal_memcpy(
-        UntagPtr(tagged_ptr_new), untagged_ptr_old,
-        Min(new_size, static_cast<uptr>(meta->GetRequestedSize())));
+    void *untagged_ptr_new = UntagPtr(tagged_ptr_new);
+    internal_memcpy(untagged_ptr_new, untagged_ptr_old,
+                    Min(new_size, static_cast<uptr>(meta->GetRequestedSize())));
     HwasanDeallocate(stack, tagged_ptr_old);
   }
   return tagged_ptr_new;
@@ -426,12 +427,13 @@ static const void *AllocationBegin(const void *p) {
   return (const void *)AddTagToPointer((uptr)beg, tag);
 }
 
-static uptr AllocationSize(const void *tagged_ptr) {
-  const void *untagged_ptr = UntagPtr(tagged_ptr);
+static uptr AllocationSize(const void *p) {
+  const void *untagged_ptr = UntagPtr(p);
   if (!untagged_ptr) return 0;
   const void *beg = allocator.GetBlockBegin(untagged_ptr);
-  Metadata *b = (Metadata *)allocator.GetMetaData(untagged_ptr);
-  if (beg != untagged_ptr) return 0;
+  if (!beg)
+    return 0;
+  Metadata *b = (Metadata *)allocator.GetMetaData(beg);
   return b->GetRequestedSize();
 }
 
@@ -540,7 +542,7 @@ void GetAllocatorGlobalRange(uptr *begin, uptr *end) {
 }
 
 uptr PointsIntoChunk(void *p) {
-  p = __hwasan::InTaggableRegion(reinterpret_cast<uptr>(p)) ? UntagPtr(p) : p;
+  p = UntagPtr(p);
   uptr addr = reinterpret_cast<uptr>(p);
   uptr chunk =
       reinterpret_cast<uptr>(__hwasan::allocator.GetBlockBeginFastLocked(p));
@@ -558,8 +560,7 @@ uptr PointsIntoChunk(void *p) {
 }
 
 uptr GetUserBegin(uptr chunk) {
-  if (__hwasan::InTaggableRegion(chunk))
-    CHECK_EQ(UntagAddr(chunk), chunk);
+  CHECK_EQ(UntagAddr(chunk), chunk);
   void *block = __hwasan::allocator.GetBlockBeginFastLocked(
       reinterpret_cast<void *>(chunk));
   if (!block)
@@ -573,15 +574,14 @@ uptr GetUserBegin(uptr chunk) {
 }
 
 uptr GetUserAddr(uptr chunk) {
-  tag_t mem_tag = *(tag_t *)__hwasan::MemToShadow(chunk);
-  if (!__hwasan::InTaggableRegion(chunk))
+  if (!InTaggableRegion(chunk))
     return chunk;
+  tag_t mem_tag = *(tag_t *)__hwasan::MemToShadow(chunk);
   return AddTagToPointer(chunk, mem_tag);
 }
 
 LsanMetadata::LsanMetadata(uptr chunk) {
-  if (__hwasan::InTaggableRegion(chunk))
-    CHECK_EQ(UntagAddr(chunk), chunk);
+  CHECK_EQ(UntagAddr(chunk), chunk);
   metadata_ =
       chunk ? __hwasan::allocator.GetMetaData(reinterpret_cast<void *>(chunk))
             : nullptr;
@@ -619,7 +619,7 @@ void ForEachChunk(ForEachChunkCallback callback, void *arg) {
 }
 
 IgnoreObjectResult IgnoreObject(const void *p) {
-  p = __hwasan::InTaggableRegion(reinterpret_cast<uptr>(p)) ? UntagPtr(p) : p;
+  p = UntagPtr(p);
   uptr addr = reinterpret_cast<uptr>(p);
   uptr chunk = reinterpret_cast<uptr>(__hwasan::allocator.GetBlockBegin(p));
   if (!chunk)
@@ -674,3 +674,5 @@ const void *__sanitizer_get_allocated_begin(const void *p) {
 }
 
 uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
+
+void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); }
diff --git a/libsanitizer/hwasan/hwasan_allocator.h b/libsanitizer/hwasan/hwasan_allocator.h
index b7a06da..ecf3f68 100644
--- a/libsanitizer/hwasan/hwasan_allocator.h
+++ b/libsanitizer/hwasan/hwasan_allocator.h
@@ -127,16 +127,6 @@ typedef RingBuffer<HeapAllocationRecord> HeapAllocationsRingBuffer;
 
 void GetAllocatorStats(AllocatorStatCounters s);
 
-inline bool InTaggableRegion(uptr addr) {
-#if defined(HWASAN_ALIASING_MODE)
-  // Aliases are mapped next to shadow so that the upper bits match the shadow
-  // base.
-  return (addr >> kTaggableRegionCheckShift) ==
-         (GetShadowOffset() >> kTaggableRegionCheckShift);
-#endif
-  return true;
-}
-
 } // namespace __hwasan
 
 #endif // HWASAN_ALLOCATOR_H
diff --git a/libsanitizer/hwasan/hwasan_checks.h b/libsanitizer/hwasan/hwasan_checks.h
index 514d351..0911af3 100644
--- a/libsanitizer/hwasan/hwasan_checks.h
+++ b/libsanitizer/hwasan/hwasan_checks.h
@@ -125,8 +125,22 @@ __attribute__((always_inline)) static void SigTrap(uptr p, uptr size) {
   // __builtin_unreachable();
 }
 
-__attribute__((always_inline, nodebug)) static bool PossiblyShortTagMatches(
-    tag_t mem_tag, uptr ptr, uptr sz) {
+__attribute__((always_inline, nodebug)) static inline uptr ShortTagSize(
+    tag_t mem_tag, uptr ptr) {
+  DCHECK(IsAligned(ptr, kShadowAlignment));
+  tag_t ptr_tag = GetTagFromPointer(ptr);
+  if (ptr_tag == mem_tag)
+    return kShadowAlignment;
+  if (!mem_tag || mem_tag >= kShadowAlignment)
+    return 0;
+  if (*(u8 *)(ptr | (kShadowAlignment - 1)) != ptr_tag)
+    return 0;
+  return mem_tag;
+}
+
+__attribute__((always_inline, nodebug)) static inline bool
+PossiblyShortTagMatches(tag_t mem_tag, uptr ptr, uptr sz) {
+  DCHECK(IsAligned(ptr, kShadowAlignment));
   tag_t ptr_tag = GetTagFromPointer(ptr);
   if (ptr_tag == mem_tag)
     return true;
@@ -134,9 +148,6 @@ __attribute__((always_inline, nodebug)) static bool PossiblyShortTagMatches(
     return false;
   if ((ptr & (kShadowAlignment - 1)) + sz > mem_tag)
     return false;
-#if !defined(__aarch64__) && !(SANITIZER_RISCV64)
-  ptr = UntagAddr(ptr);
-#endif
   return *(u8 *)(ptr | (kShadowAlignment - 1)) == ptr_tag;
 }
 
@@ -169,7 +180,7 @@ __attribute__((always_inline, nodebug)) static void CheckAddressSized(uptr p,
         __builtin_unreachable();
     }
   uptr end = p + sz;
-  uptr tail_sz = end & 0xf;
+  uptr tail_sz = end & (kShadowAlignment - 1);
   if (UNLIKELY(tail_sz != 0 &&
                !PossiblyShortTagMatches(
                    *shadow_last, end & ~(kShadowAlignment - 1), tail_sz))) {
diff --git a/libsanitizer/hwasan/hwasan_interceptors.cpp b/libsanitizer/hwasan/hwasan_interceptors.cpp
index 16ac85e..67edba4 100644
--- a/libsanitizer/hwasan/hwasan_interceptors.cpp
+++ b/libsanitizer/hwasan/hwasan_interceptors.cpp
@@ -14,9 +14,11 @@
 // sanitizer_common/sanitizer_common_interceptors.h
 //===----------------------------------------------------------------------===//
 
-#include "interception/interception.h"
 #include "hwasan.h"
+#include "hwasan_checks.h"
 #include "hwasan_thread.h"
+#include "interception/interception.h"
+#include "sanitizer_common/sanitizer_linux.h"
 #include "sanitizer_common/sanitizer_stackdepot.h"
 
 #if !SANITIZER_FUCHSIA
@@ -28,31 +30,47 @@ using namespace __hwasan;
 struct ThreadStartArg {
   thread_callback_t callback;
   void *param;
+  __sanitizer_sigset_t starting_sigset_;
 };
 
 static void *HwasanThreadStartFunc(void *arg) {
   __hwasan_thread_enter();
   ThreadStartArg A = *reinterpret_cast<ThreadStartArg*>(arg);
+  SetSigProcMask(&A.starting_sigset_, nullptr);
   UnmapOrDie(arg, GetPageSizeCached());
   return A.callback(A.param);
 }
 
+#    define COMMON_SYSCALL_PRE_READ_RANGE(p, s) __hwasan_loadN((uptr)p, (uptr)s)
+#    define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
+      __hwasan_storeN((uptr)p, (uptr)s)
+#    define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
+      do {                                       \
+        (void)(p);                               \
+        (void)(s);                               \
+      } while (false)
+#    define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
+      do {                                        \
+        (void)(p);                                \
+        (void)(s);                                \
+      } while (false)
+#    include "sanitizer_common/sanitizer_common_syscalls.inc"
+#    include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
+
 INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*),
             void * param) {
   EnsureMainThreadIDIsCorrect();
   ScopedTaggingDisabler tagging_disabler;
   ThreadStartArg *A = reinterpret_cast<ThreadStartArg *> (MmapOrDie(
       GetPageSizeCached(), "pthread_create"));
-  *A = {callback, param};
-  int res;
-  {
-    // ASAN uses the same approach to disable leaks from pthread_create.
+  A->callback = callback;
+  A->param = param;
+  ScopedBlockSignals block(&A->starting_sigset_);
+  // ASAN uses the same approach to disable leaks from pthread_create.
 #    if CAN_SANITIZE_LEAKS
-    __lsan::ScopedInterceptorDisabler lsan_disabler;
+  __lsan::ScopedInterceptorDisabler lsan_disabler;
 #    endif
-    res = REAL(pthread_create)(th, attr, &HwasanThreadStartFunc, A);
-  }
-  return res;
+  return REAL(pthread_create)(th, attr, &HwasanThreadStartFunc, A);
 }
 
 INTERCEPTOR(int, pthread_join, void *t, void **arg) {