diff options
author | Kostya Serebryany <kcc@google.com> | 2014-05-22 07:09:21 +0000 |
---|---|---|
committer | Kostya Serebryany <kcc@gcc.gnu.org> | 2014-05-22 07:09:21 +0000 |
commit | dee5ea7a0bfe95367820a443ef4a7c813e598b55 (patch) | |
tree | f90afdf42b3ae78508a5c6422f458a5bb0216aa2 /libsanitizer/tsan | |
parent | b95591361e32a755231d99c348f8a43e2aed0187 (diff) | |
download | gcc-dee5ea7a0bfe95367820a443ef4a7c813e598b55.zip gcc-dee5ea7a0bfe95367820a443ef4a7c813e598b55.tar.gz gcc-dee5ea7a0bfe95367820a443ef4a7c813e598b55.tar.bz2 |
libsanitizer merge from upstream r209283
From-SVN: r210743
Diffstat (limited to 'libsanitizer/tsan')
38 files changed, 1712 insertions, 1839 deletions
diff --git a/libsanitizer/tsan/Makefile.am b/libsanitizer/tsan/Makefile.am index 39ed252..5d98e21 100644 --- a/libsanitizer/tsan/Makefile.am +++ b/libsanitizer/tsan/Makefile.am @@ -11,32 +11,32 @@ ACLOCAL_AMFLAGS = -I m4 toolexeclib_LTLIBRARIES = libtsan.la tsan_files = \ - tsan_clock.cc \ - tsan_interface_atomic.cc \ - tsan_mutex.cc \ - tsan_report.cc \ - tsan_rtl_thread.cc \ - tsan_symbolize.cc \ - tsan_flags.cc \ - tsan_interface.cc \ - tsan_platform_linux.cc \ - tsan_rtl.cc \ - tsan_stat.cc \ - tsan_sync.cc \ - tsan_ignoreset.cc \ - tsan_interceptors.cc \ - tsan_md5.cc \ - tsan_platform_mac.cc \ - tsan_rtl_mutex.cc \ - tsan_suppressions.cc \ - tsan_interface_ann.cc \ - tsan_mman.cc \ - tsan_rtl_report.cc \ + tsan_clock.cc \ tsan_fd.cc \ - tsan_interface_java.cc \ - tsan_mutexset.cc \ - tsan_symbolize_addr2line_linux.cc \ - tsan_rtl_amd64.S + tsan_flags.cc \ + tsan_ignoreset.cc \ + tsan_interceptors.cc \ + tsan_interface_ann.cc \ + tsan_interface_atomic.cc \ + tsan_interface.cc \ + tsan_interface_java.cc \ + tsan_md5.cc \ + tsan_mman.cc \ + tsan_mutex.cc \ + tsan_mutexset.cc \ + tsan_platform_linux.cc \ + tsan_platform_mac.cc \ + tsan_platform_windows.cc \ + tsan_report.cc \ + tsan_rtl.cc \ + tsan_rtl_mutex.cc \ + tsan_rtl_report.cc \ + tsan_rtl_thread.cc \ + tsan_stat.cc \ + tsan_suppressions.cc \ + tsan_symbolize.cc \ + tsan_sync.cc \ + tsan_rtl_amd64.S libtsan_la_SOURCES = $(tsan_files) libtsan_la_LIBADD = $(top_builddir)/sanitizer_common/libsanitizer_common.la $(top_builddir)/interception/libinterception.la diff --git a/libsanitizer/tsan/Makefile.in b/libsanitizer/tsan/Makefile.in index 01c27b9..068aaa8 100644 --- a/libsanitizer/tsan/Makefile.in +++ b/libsanitizer/tsan/Makefile.in @@ -84,15 +84,15 @@ libtsan_la_DEPENDENCIES = \ $(top_builddir)/sanitizer_common/libsanitizer_common.la \ $(top_builddir)/interception/libinterception.la \ $(am__append_1) $(am__DEPENDENCIES_1) -am__objects_1 = tsan_clock.lo tsan_interface_atomic.lo tsan_mutex.lo \ - tsan_report.lo tsan_rtl_thread.lo tsan_symbolize.lo \ - tsan_flags.lo tsan_interface.lo tsan_platform_linux.lo \ - tsan_rtl.lo tsan_stat.lo tsan_sync.lo tsan_ignoreset.lo \ - tsan_interceptors.lo tsan_md5.lo tsan_platform_mac.lo \ - tsan_rtl_mutex.lo tsan_suppressions.lo tsan_interface_ann.lo \ - tsan_mman.lo tsan_rtl_report.lo tsan_fd.lo \ - tsan_interface_java.lo tsan_mutexset.lo \ - tsan_symbolize_addr2line_linux.lo tsan_rtl_amd64.lo +am__objects_1 = tsan_clock.lo tsan_fd.lo tsan_flags.lo \ + tsan_ignoreset.lo tsan_interceptors.lo tsan_interface_ann.lo \ + tsan_interface_atomic.lo tsan_interface.lo \ + tsan_interface_java.lo tsan_md5.lo tsan_mman.lo tsan_mutex.lo \ + tsan_mutexset.lo tsan_platform_linux.lo tsan_platform_mac.lo \ + tsan_platform_windows.lo tsan_report.lo tsan_rtl.lo \ + tsan_rtl_mutex.lo tsan_rtl_report.lo tsan_rtl_thread.lo \ + tsan_stat.lo tsan_suppressions.lo tsan_symbolize.lo \ + tsan_sync.lo tsan_rtl_amd64.lo am_libtsan_la_OBJECTS = $(am__objects_1) libtsan_la_OBJECTS = $(am_libtsan_la_OBJECTS) libtsan_la_LINK = $(LIBTOOL) --tag=CXX $(AM_LIBTOOLFLAGS) \ @@ -280,32 +280,32 @@ AM_CXXFLAGS = -Wall -W -Wno-unused-parameter -Wwrite-strings -pedantic \ ACLOCAL_AMFLAGS = -I m4 toolexeclib_LTLIBRARIES = libtsan.la tsan_files = \ - tsan_clock.cc \ - tsan_interface_atomic.cc \ - tsan_mutex.cc \ - tsan_report.cc \ - tsan_rtl_thread.cc \ - tsan_symbolize.cc \ - tsan_flags.cc \ - tsan_interface.cc \ - tsan_platform_linux.cc \ - tsan_rtl.cc \ - tsan_stat.cc \ - tsan_sync.cc \ - tsan_ignoreset.cc \ - tsan_interceptors.cc \ - tsan_md5.cc \ - tsan_platform_mac.cc \ - tsan_rtl_mutex.cc \ - tsan_suppressions.cc \ - tsan_interface_ann.cc \ - tsan_mman.cc \ - tsan_rtl_report.cc \ + tsan_clock.cc \ tsan_fd.cc \ - tsan_interface_java.cc \ - tsan_mutexset.cc \ - tsan_symbolize_addr2line_linux.cc \ - tsan_rtl_amd64.S + tsan_flags.cc \ + tsan_ignoreset.cc \ + tsan_interceptors.cc \ + tsan_interface_ann.cc \ + tsan_interface_atomic.cc \ + tsan_interface.cc \ + tsan_interface_java.cc \ + tsan_md5.cc \ + tsan_mman.cc \ + tsan_mutex.cc \ + tsan_mutexset.cc \ + tsan_platform_linux.cc \ + tsan_platform_mac.cc \ + tsan_platform_windows.cc \ + tsan_report.cc \ + tsan_rtl.cc \ + tsan_rtl_mutex.cc \ + tsan_rtl_report.cc \ + tsan_rtl_thread.cc \ + tsan_stat.cc \ + tsan_suppressions.cc \ + tsan_symbolize.cc \ + tsan_sync.cc \ + tsan_rtl_amd64.S libtsan_la_SOURCES = $(tsan_files) libtsan_la_LIBADD = \ @@ -442,6 +442,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_mutexset.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_platform_linux.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_platform_mac.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_platform_windows.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_report.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_rtl_amd64.Plo@am__quote@ @@ -451,7 +452,6 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_stat.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_suppressions.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_symbolize.Plo@am__quote@ -@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_symbolize_addr2line_linux.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/tsan_sync.Plo@am__quote@ .S.o: diff --git a/libsanitizer/tsan/tsan_clock.cc b/libsanitizer/tsan/tsan_clock.cc index 5d45a5d..b944cc5 100644 --- a/libsanitizer/tsan/tsan_clock.cc +++ b/libsanitizer/tsan/tsan_clock.cc @@ -11,99 +11,338 @@ #include "tsan_clock.h" #include "tsan_rtl.h" -// It's possible to optimize clock operations for some important cases -// so that they are O(1). The cases include singletons, once's, local mutexes. -// First, SyncClock must be re-implemented to allow indexing by tid. -// It must not necessarily be a full vector clock, though. For example it may -// be a multi-level table. -// Then, each slot in SyncClock must contain a dirty bit (it's united with -// the clock value, so no space increase). The acquire algorithm looks -// as follows: -// void acquire(thr, tid, thr_clock, sync_clock) { -// if (!sync_clock[tid].dirty) -// return; // No new info to acquire. -// // This handles constant reads of singleton pointers and -// // stop-flags. -// acquire_impl(thr_clock, sync_clock); // As usual, O(N). -// sync_clock[tid].dirty = false; -// sync_clock.dirty_count--; -// } -// The release operation looks as follows: -// void release(thr, tid, thr_clock, sync_clock) { -// // thr->sync_cache is a simple fixed-size hash-based cache that holds -// // several previous sync_clock's. -// if (thr->sync_cache[sync_clock] >= thr->last_acquire_epoch) { -// // The thread did no acquire operations since last release on this clock. -// // So update only the thread's slot (other slots can't possibly change). -// sync_clock[tid].clock = thr->epoch; -// if (sync_clock.dirty_count == sync_clock.cnt -// || (sync_clock.dirty_count == sync_clock.cnt - 1 -// && sync_clock[tid].dirty == false)) -// // All dirty flags are set, bail out. -// return; -// set all dirty bits, but preserve the thread's bit. // O(N) -// update sync_clock.dirty_count; -// return; +// SyncClock and ThreadClock implement vector clocks for sync variables +// (mutexes, atomic variables, file descriptors, etc) and threads, respectively. +// ThreadClock contains fixed-size vector clock for maximum number of threads. +// SyncClock contains growable vector clock for currently necessary number of +// threads. +// Together they implement very simple model of operations, namely: +// +// void ThreadClock::acquire(const SyncClock *src) { +// for (int i = 0; i < kMaxThreads; i++) +// clock[i] = max(clock[i], src->clock[i]); +// } +// +// void ThreadClock::release(SyncClock *dst) const { +// for (int i = 0; i < kMaxThreads; i++) +// dst->clock[i] = max(dst->clock[i], clock[i]); +// } +// +// void ThreadClock::ReleaseStore(SyncClock *dst) const { +// for (int i = 0; i < kMaxThreads; i++) +// dst->clock[i] = clock[i]; +// } +// +// void ThreadClock::acq_rel(SyncClock *dst) { +// acquire(dst); +// release(dst); // } -// release_impl(thr_clock, sync_clock); // As usual, O(N). -// set all dirty bits, but preserve the thread's bit. -// // The previous step is combined with release_impl(), so that -// // we scan the arrays only once. -// update sync_clock.dirty_count; -// } +// +// Conformance to this model is extensively verified in tsan_clock_test.cc. +// However, the implementation is significantly more complex. The complexity +// allows to implement important classes of use cases in O(1) instead of O(N). +// +// The use cases are: +// 1. Singleton/once atomic that has a single release-store operation followed +// by zillions of acquire-loads (the acquire-load is O(1)). +// 2. Thread-local mutex (both lock and unlock can be O(1)). +// 3. Leaf mutex (unlock is O(1)). +// 4. A mutex shared by 2 threads (both lock and unlock can be O(1)). +// 5. An atomic with a single writer (writes can be O(1)). +// The implementation dynamically adopts to workload. So if an atomic is in +// read-only phase, these reads will be O(1); if it later switches to read/write +// phase, the implementation will correctly handle that by switching to O(N). +// +// Thread-safety note: all const operations on SyncClock's are conducted under +// a shared lock; all non-const operations on SyncClock's are conducted under +// an exclusive lock; ThreadClock's are private to respective threads and so +// do not need any protection. +// +// Description of ThreadClock state: +// clk_ - fixed size vector clock. +// nclk_ - effective size of the vector clock (the rest is zeros). +// tid_ - index of the thread associated with he clock ("current thread"). +// last_acquire_ - current thread time when it acquired something from +// other threads. +// +// Description of SyncClock state: +// clk_ - variable size vector clock, low kClkBits hold timestamp, +// the remaining bits hold "acquired" flag (the actual value is thread's +// reused counter); +// if acquried == thr->reused_, then the respective thread has already +// acquired this clock (except possibly dirty_tids_). +// dirty_tids_ - holds up to two indeces in the vector clock that other threads +// need to acquire regardless of "acquired" flag value; +// release_store_tid_ - denotes that the clock state is a result of +// release-store operation by the thread with release_store_tid_ index. +// release_store_reused_ - reuse count of release_store_tid_. + +// We don't have ThreadState in these methods, so this is an ugly hack that +// works only in C++. +#ifndef TSAN_GO +# define CPP_STAT_INC(typ) StatInc(cur_thread(), typ) +#else +# define CPP_STAT_INC(typ) (void)0 +#endif namespace __tsan { -ThreadClock::ThreadClock() { - nclk_ = 0; - for (uptr i = 0; i < (uptr)kMaxTidInClock; i++) - clk_[i] = 0; +const unsigned kInvalidTid = (unsigned)-1; + +ThreadClock::ThreadClock(unsigned tid, unsigned reused) + : tid_(tid) + , reused_(reused + 1) { // 0 has special meaning + CHECK_LT(tid, kMaxTidInClock); + CHECK_EQ(reused_, ((u64)reused_ << kClkBits) >> kClkBits); + nclk_ = tid_ + 1; + last_acquire_ = 0; + internal_memset(clk_, 0, sizeof(clk_)); + clk_[tid_].reused = reused_; } void ThreadClock::acquire(const SyncClock *src) { DCHECK(nclk_ <= kMaxTid); DCHECK(src->clk_.Size() <= kMaxTid); + CPP_STAT_INC(StatClockAcquire); + // Check if it's empty -> no need to do anything. const uptr nclk = src->clk_.Size(); - if (nclk == 0) + if (nclk == 0) { + CPP_STAT_INC(StatClockAcquireEmpty); return; + } + + // Check if we've already acquired src after the last release operation on src + bool acquired = false; + if (nclk > tid_) { + CPP_STAT_INC(StatClockAcquireLarge); + if (src->clk_[tid_].reused == reused_) { + CPP_STAT_INC(StatClockAcquireRepeat); + for (unsigned i = 0; i < kDirtyTids; i++) { + unsigned tid = src->dirty_tids_[i]; + if (tid != kInvalidTid) { + u64 epoch = src->clk_[tid].epoch; + if (clk_[tid].epoch < epoch) { + clk_[tid].epoch = epoch; + acquired = true; + } + } + } + if (acquired) { + CPP_STAT_INC(StatClockAcquiredSomething); + last_acquire_ = clk_[tid_].epoch; + } + return; + } + } + + // O(N) acquire. + CPP_STAT_INC(StatClockAcquireFull); nclk_ = max(nclk_, nclk); for (uptr i = 0; i < nclk; i++) { - if (clk_[i] < src->clk_[i]) - clk_[i] = src->clk_[i]; + u64 epoch = src->clk_[i].epoch; + if (clk_[i].epoch < epoch) { + clk_[i].epoch = epoch; + acquired = true; + } + } + + // Remember that this thread has acquired this clock. + if (nclk > tid_) + src->clk_[tid_].reused = reused_; + + if (acquired) { + CPP_STAT_INC(StatClockAcquiredSomething); + last_acquire_ = clk_[tid_].epoch; } } void ThreadClock::release(SyncClock *dst) const { - DCHECK(nclk_ <= kMaxTid); - DCHECK(dst->clk_.Size() <= kMaxTid); + DCHECK_LE(nclk_, kMaxTid); + DCHECK_LE(dst->clk_.Size(), kMaxTid); - if (dst->clk_.Size() < nclk_) + if (dst->clk_.Size() == 0) { + // ReleaseStore will correctly set release_store_tid_, + // which can be important for future operations. + ReleaseStore(dst); + return; + } + + CPP_STAT_INC(StatClockRelease); + // Check if we need to resize dst. + if (dst->clk_.Size() < nclk_) { + CPP_STAT_INC(StatClockReleaseResize); dst->clk_.Resize(nclk_); + } + + // Check if we had not acquired anything from other threads + // since the last release on dst. If so, we need to update + // only dst->clk_[tid_]. + if (dst->clk_[tid_].epoch > last_acquire_) { + UpdateCurrentThread(dst); + if (dst->release_store_tid_ != tid_ || + dst->release_store_reused_ != reused_) + dst->release_store_tid_ = kInvalidTid; + return; + } + + // O(N) release. + CPP_STAT_INC(StatClockReleaseFull); + // First, remember whether we've acquired dst. + bool acquired = IsAlreadyAcquired(dst); + if (acquired) + CPP_STAT_INC(StatClockReleaseAcquired); + // Update dst->clk_. for (uptr i = 0; i < nclk_; i++) { - if (dst->clk_[i] < clk_[i]) - dst->clk_[i] = clk_[i]; + dst->clk_[i].epoch = max(dst->clk_[i].epoch, clk_[i].epoch); + dst->clk_[i].reused = 0; } + // Clear 'acquired' flag in the remaining elements. + if (nclk_ < dst->clk_.Size()) + CPP_STAT_INC(StatClockReleaseClearTail); + for (uptr i = nclk_; i < dst->clk_.Size(); i++) + dst->clk_[i].reused = 0; + for (unsigned i = 0; i < kDirtyTids; i++) + dst->dirty_tids_[i] = kInvalidTid; + dst->release_store_tid_ = kInvalidTid; + dst->release_store_reused_ = 0; + // If we've acquired dst, remember this fact, + // so that we don't need to acquire it on next acquire. + if (acquired) + dst->clk_[tid_].reused = reused_; } void ThreadClock::ReleaseStore(SyncClock *dst) const { DCHECK(nclk_ <= kMaxTid); DCHECK(dst->clk_.Size() <= kMaxTid); + CPP_STAT_INC(StatClockStore); - if (dst->clk_.Size() < nclk_) + // Check if we need to resize dst. + if (dst->clk_.Size() < nclk_) { + CPP_STAT_INC(StatClockStoreResize); dst->clk_.Resize(nclk_); - for (uptr i = 0; i < nclk_; i++) - dst->clk_[i] = clk_[i]; - for (uptr i = nclk_; i < dst->clk_.Size(); i++) - dst->clk_[i] = 0; + } + + if (dst->release_store_tid_ == tid_ && + dst->release_store_reused_ == reused_ && + dst->clk_[tid_].epoch > last_acquire_) { + CPP_STAT_INC(StatClockStoreFast); + UpdateCurrentThread(dst); + return; + } + + // O(N) release-store. + CPP_STAT_INC(StatClockStoreFull); + for (uptr i = 0; i < nclk_; i++) { + dst->clk_[i].epoch = clk_[i].epoch; + dst->clk_[i].reused = 0; + } + // Clear the tail of dst->clk_. + if (nclk_ < dst->clk_.Size()) { + internal_memset(&dst->clk_[nclk_], 0, + (dst->clk_.Size() - nclk_) * sizeof(dst->clk_[0])); + CPP_STAT_INC(StatClockStoreTail); + } + for (unsigned i = 0; i < kDirtyTids; i++) + dst->dirty_tids_[i] = kInvalidTid; + dst->release_store_tid_ = tid_; + dst->release_store_reused_ = reused_; + // Rememeber that we don't need to acquire it in future. + dst->clk_[tid_].reused = reused_; } void ThreadClock::acq_rel(SyncClock *dst) { + CPP_STAT_INC(StatClockAcquireRelease); acquire(dst); - release(dst); + ReleaseStore(dst); +} + +// Updates only single element related to the current thread in dst->clk_. +void ThreadClock::UpdateCurrentThread(SyncClock *dst) const { + // Update the threads time, but preserve 'acquired' flag. + dst->clk_[tid_].epoch = clk_[tid_].epoch; + + for (unsigned i = 0; i < kDirtyTids; i++) { + if (dst->dirty_tids_[i] == tid_) { + CPP_STAT_INC(StatClockReleaseFast1); + return; + } + if (dst->dirty_tids_[i] == kInvalidTid) { + CPP_STAT_INC(StatClockReleaseFast2); + dst->dirty_tids_[i] = tid_; + return; + } + } + // Reset all 'acquired' flags, O(N). + CPP_STAT_INC(StatClockReleaseSlow); + for (uptr i = 0; i < dst->clk_.Size(); i++) { + dst->clk_[i].reused = 0; + } + for (unsigned i = 0; i < kDirtyTids; i++) + dst->dirty_tids_[i] = kInvalidTid; +} + +// Checks whether the current threads has already acquired src. +bool ThreadClock::IsAlreadyAcquired(const SyncClock *src) const { + if (src->clk_[tid_].reused != reused_) + return false; + for (unsigned i = 0; i < kDirtyTids; i++) { + unsigned tid = src->dirty_tids_[i]; + if (tid != kInvalidTid) { + if (clk_[tid].epoch < src->clk_[tid].epoch) + return false; + } + } + return true; +} + +// Sets a single element in the vector clock. +// This function is called only from weird places like AcquireGlobal. +void ThreadClock::set(unsigned tid, u64 v) { + DCHECK_LT(tid, kMaxTid); + DCHECK_GE(v, clk_[tid].epoch); + clk_[tid].epoch = v; + if (nclk_ <= tid) + nclk_ = tid + 1; + last_acquire_ = clk_[tid_].epoch; +} + +void ThreadClock::DebugDump(int(*printf)(const char *s, ...)) { + printf("clock=["); + for (uptr i = 0; i < nclk_; i++) + printf("%s%llu", i == 0 ? "" : ",", clk_[i].epoch); + printf("] reused=["); + for (uptr i = 0; i < nclk_; i++) + printf("%s%llu", i == 0 ? "" : ",", clk_[i].reused); + printf("] tid=%u/%u last_acq=%llu", + tid_, reused_, last_acquire_); } SyncClock::SyncClock() - : clk_(MBlockClock) { + : clk_(MBlockClock) { + release_store_tid_ = kInvalidTid; + release_store_reused_ = 0; + for (uptr i = 0; i < kDirtyTids; i++) + dirty_tids_[i] = kInvalidTid; +} + +void SyncClock::Reset() { + clk_.Reset(); + release_store_tid_ = kInvalidTid; + release_store_reused_ = 0; + for (uptr i = 0; i < kDirtyTids; i++) + dirty_tids_[i] = kInvalidTid; +} + +void SyncClock::DebugDump(int(*printf)(const char *s, ...)) { + printf("clock=["); + for (uptr i = 0; i < clk_.Size(); i++) + printf("%s%llu", i == 0 ? "" : ",", clk_[i].epoch); + printf("] reused=["); + for (uptr i = 0; i < clk_.Size(); i++) + printf("%s%llu", i == 0 ? "" : ",", clk_[i].reused); + printf("] release_store_tid=%d/%d dirty_tids=%d/%d", + release_store_tid_, release_store_reused_, + dirty_tids_[0], dirty_tids_[1]); } } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_clock.h b/libsanitizer/tsan/tsan_clock.h index 8e4bf99..2ce480b 100644 --- a/libsanitizer/tsan/tsan_clock.h +++ b/libsanitizer/tsan/tsan_clock.h @@ -16,6 +16,11 @@ namespace __tsan { +struct ClockElem { + u64 epoch : kClkBits; + u64 reused : 64 - kClkBits; +}; + // The clock that lives in sync variables (mutexes, atomics, etc). class SyncClock { public: @@ -25,38 +30,43 @@ class SyncClock { return clk_.Size(); } - void Reset() { - clk_.Reset(); + u64 get(unsigned tid) const { + DCHECK_LT(tid, clk_.Size()); + return clk_[tid].epoch; } + void Reset(); + + void DebugDump(int(*printf)(const char *s, ...)); + private: - Vector<u64> clk_; + unsigned release_store_tid_; + unsigned release_store_reused_; + static const uptr kDirtyTids = 2; + unsigned dirty_tids_[kDirtyTids]; + mutable Vector<ClockElem> clk_; friend struct ThreadClock; }; // The clock that lives in threads. struct ThreadClock { public: - ThreadClock(); + explicit ThreadClock(unsigned tid, unsigned reused = 0); u64 get(unsigned tid) const { DCHECK_LT(tid, kMaxTidInClock); - return clk_[tid]; + return clk_[tid].epoch; } - void set(unsigned tid, u64 v) { - DCHECK_LT(tid, kMaxTid); - DCHECK_GE(v, clk_[tid]); - clk_[tid] = v; - if (nclk_ <= tid) - nclk_ = tid + 1; + void set(unsigned tid, u64 v); + + void set(u64 v) { + DCHECK_GE(v, clk_[tid_].epoch); + clk_[tid_].epoch = v; } - void tick(unsigned tid) { - DCHECK_LT(tid, kMaxTid); - clk_[tid]++; - if (nclk_ <= tid) - nclk_ = tid + 1; + void tick() { + clk_[tid_].epoch++; } uptr size() const { @@ -68,9 +78,19 @@ struct ThreadClock { void acq_rel(SyncClock *dst); void ReleaseStore(SyncClock *dst) const; + void DebugReset(); + void DebugDump(int(*printf)(const char *s, ...)); + private: + static const uptr kDirtyTids = SyncClock::kDirtyTids; + const unsigned tid_; + const unsigned reused_; + u64 last_acquire_; uptr nclk_; - u64 clk_[kMaxTidInClock]; + ClockElem clk_[kMaxTidInClock]; + + bool IsAlreadyAcquired(const SyncClock *src) const; + void UpdateCurrentThread(SyncClock *dst) const; }; } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_defs.h b/libsanitizer/tsan/tsan_defs.h index 3f20797..cc65ae8 100644 --- a/libsanitizer/tsan/tsan_defs.h +++ b/libsanitizer/tsan/tsan_defs.h @@ -39,6 +39,7 @@ const int kTidBits = 13; const unsigned kMaxTid = 1 << kTidBits; const unsigned kMaxTidInClock = kMaxTid * 2; // This includes msb 'freed' bit. const int kClkBits = 42; +const unsigned kMaxTidReuse = (1 << (64 - kClkBits)) - 1; const uptr kShadowStackSize = 64 * 1024; const uptr kTraceStackSize = 256; @@ -63,6 +64,12 @@ const uptr kShadowSize = 8; // Shadow memory is kShadowMultiplier times larger than user memory. const uptr kShadowMultiplier = kShadowSize * kShadowCnt / kShadowCell; +#if defined(TSAN_NO_HISTORY) && TSAN_NO_HISTORY +const bool kCollectHistory = false; +#else +const bool kCollectHistory = true; +#endif + #if defined(TSAN_COLLECT_STATS) && TSAN_COLLECT_STATS const bool kCollectStats = true; #else diff --git a/libsanitizer/tsan/tsan_fd.cc b/libsanitizer/tsan/tsan_fd.cc index b7ac311..7d62ae4 100644 --- a/libsanitizer/tsan/tsan_fd.cc +++ b/libsanitizer/tsan/tsan_fd.cc @@ -63,7 +63,7 @@ static void unref(ThreadState *thr, uptr pc, FdSync *s) { CHECK_NE(s, &fdctx.globsync); CHECK_NE(s, &fdctx.filesync); CHECK_NE(s, &fdctx.socksync); - SyncVar *v = CTX()->synctab.GetAndRemove(thr, pc, (uptr)s); + SyncVar *v = ctx->synctab.GetAndRemove(thr, pc, (uptr)s); if (v) DestroyAndFree(v); internal_free(s); @@ -283,13 +283,13 @@ void FdSocketConnect(ThreadState *thr, uptr pc, int fd) { init(thr, pc, fd, &fdctx.socksync); } -uptr File2addr(char *path) { +uptr File2addr(const char *path) { (void)path; static u64 addr; return (uptr)&addr; } -uptr Dir2addr(char *path) { +uptr Dir2addr(const char *path) { (void)path; static u64 addr; return (uptr)&addr; diff --git a/libsanitizer/tsan/tsan_fd.h b/libsanitizer/tsan/tsan_fd.h index 3306873..57ae62c 100644 --- a/libsanitizer/tsan/tsan_fd.h +++ b/libsanitizer/tsan/tsan_fd.h @@ -55,8 +55,8 @@ void FdSocketConnect(ThreadState *thr, uptr pc, int fd); bool FdLocation(uptr addr, int *fd, int *tid, u32 *stack); void FdOnFork(ThreadState *thr, uptr pc); -uptr File2addr(char *path); -uptr Dir2addr(char *path); +uptr File2addr(const char *path); +uptr Dir2addr(const char *path); } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_flags.cc b/libsanitizer/tsan/tsan_flags.cc index 158e24f..e241cb6 100644 --- a/libsanitizer/tsan/tsan_flags.cc +++ b/libsanitizer/tsan/tsan_flags.cc @@ -18,17 +18,13 @@ namespace __tsan { Flags *flags() { - return &CTX()->flags; + return &ctx->flags; } // Can be overriden in frontend. #ifdef TSAN_EXTERNAL_HOOKS -void OverrideFlags(Flags *f); extern "C" const char* __tsan_default_options(); #else -void WEAK OverrideFlags(Flags *f) { - (void)f; -} extern "C" SANITIZER_INTERFACE_ATTRIBUTE const char *WEAK __tsan_default_options() { return ""; @@ -36,30 +32,35 @@ const char *WEAK __tsan_default_options() { #endif static void ParseFlags(Flags *f, const char *env) { - ParseFlag(env, &f->enable_annotations, "enable_annotations"); - ParseFlag(env, &f->suppress_equal_stacks, "suppress_equal_stacks"); - ParseFlag(env, &f->suppress_equal_addresses, "suppress_equal_addresses"); - ParseFlag(env, &f->suppress_java, "suppress_java"); - ParseFlag(env, &f->report_bugs, "report_bugs"); - ParseFlag(env, &f->report_thread_leaks, "report_thread_leaks"); - ParseFlag(env, &f->report_destroy_locked, "report_destroy_locked"); - ParseFlag(env, &f->report_signal_unsafe, "report_signal_unsafe"); - ParseFlag(env, &f->report_atomic_races, "report_atomic_races"); - ParseFlag(env, &f->force_seq_cst_atomics, "force_seq_cst_atomics"); - ParseFlag(env, &f->suppressions, "suppressions"); - ParseFlag(env, &f->print_suppressions, "print_suppressions"); - ParseFlag(env, &f->print_benign, "print_benign"); - ParseFlag(env, &f->exitcode, "exitcode"); - ParseFlag(env, &f->halt_on_error, "halt_on_error"); - ParseFlag(env, &f->atexit_sleep_ms, "atexit_sleep_ms"); - ParseFlag(env, &f->profile_memory, "profile_memory"); - ParseFlag(env, &f->flush_memory_ms, "flush_memory_ms"); - ParseFlag(env, &f->flush_symbolizer_ms, "flush_symbolizer_ms"); - ParseFlag(env, &f->memory_limit_mb, "memory_limit_mb"); - ParseFlag(env, &f->stop_on_start, "stop_on_start"); - ParseFlag(env, &f->running_on_valgrind, "running_on_valgrind"); - ParseFlag(env, &f->history_size, "history_size"); - ParseFlag(env, &f->io_sync, "io_sync"); + ParseFlag(env, &f->enable_annotations, "enable_annotations", ""); + ParseFlag(env, &f->suppress_equal_stacks, "suppress_equal_stacks", ""); + ParseFlag(env, &f->suppress_equal_addresses, "suppress_equal_addresses", ""); + ParseFlag(env, &f->suppress_java, "suppress_java", ""); + ParseFlag(env, &f->report_bugs, "report_bugs", ""); + ParseFlag(env, &f->report_thread_leaks, "report_thread_leaks", ""); + ParseFlag(env, &f->report_destroy_locked, "report_destroy_locked", ""); + ParseFlag(env, &f->report_mutex_bugs, "report_mutex_bugs", ""); + ParseFlag(env, &f->report_signal_unsafe, "report_signal_unsafe", ""); + ParseFlag(env, &f->report_atomic_races, "report_atomic_races", ""); + ParseFlag(env, &f->force_seq_cst_atomics, "force_seq_cst_atomics", ""); + ParseFlag(env, &f->suppressions, "suppressions", ""); + ParseFlag(env, &f->print_suppressions, "print_suppressions", ""); + ParseFlag(env, &f->print_benign, "print_benign", ""); + ParseFlag(env, &f->exitcode, "exitcode", ""); + ParseFlag(env, &f->halt_on_error, "halt_on_error", ""); + ParseFlag(env, &f->atexit_sleep_ms, "atexit_sleep_ms", ""); + ParseFlag(env, &f->profile_memory, "profile_memory", ""); + ParseFlag(env, &f->flush_memory_ms, "flush_memory_ms", ""); + ParseFlag(env, &f->flush_symbolizer_ms, "flush_symbolizer_ms", ""); + ParseFlag(env, &f->memory_limit_mb, "memory_limit_mb", ""); + ParseFlag(env, &f->stop_on_start, "stop_on_start", ""); + ParseFlag(env, &f->running_on_valgrind, "running_on_valgrind", ""); + ParseFlag(env, &f->history_size, "history_size", ""); + ParseFlag(env, &f->io_sync, "io_sync", ""); + ParseFlag(env, &f->die_after_fork, "die_after_fork", ""); + + // DDFlags + ParseFlag(env, &f->second_deadlock_stack, "second_deadlock_stack", ""); } void InitializeFlags(Flags *f, const char *env) { @@ -73,6 +74,7 @@ void InitializeFlags(Flags *f, const char *env) { f->report_bugs = true; f->report_thread_leaks = true; f->report_destroy_locked = true; + f->report_mutex_bugs = true; f->report_signal_unsafe = true; f->report_atomic_races = true; f->force_seq_cst_atomics = false; @@ -90,11 +92,16 @@ void InitializeFlags(Flags *f, const char *env) { f->running_on_valgrind = false; f->history_size = kGoMode ? 1 : 2; // There are a lot of goroutines in Go. f->io_sync = 1; + f->die_after_fork = true; + + // DDFlags + f->second_deadlock_stack = false; SetCommonFlagsDefaults(f); + // Override some common flags defaults. + f->allow_addr2line = true; // Let a frontend override. - OverrideFlags(f); ParseFlags(f, __tsan_default_options()); ParseCommonFlagsFromString(f, __tsan_default_options()); // Override from command line. @@ -111,6 +118,8 @@ void InitializeFlags(Flags *f, const char *env) { f->report_signal_unsafe = false; } + if (f->help) PrintFlagDescriptions(); + if (f->history_size < 0 || f->history_size > 7) { Printf("ThreadSanitizer: incorrect value for history_size" " (must be [0..7])\n"); diff --git a/libsanitizer/tsan/tsan_flags.h b/libsanitizer/tsan/tsan_flags.h index 05d11a4..4bf459d 100644 --- a/libsanitizer/tsan/tsan_flags.h +++ b/libsanitizer/tsan/tsan_flags.h @@ -12,23 +12,18 @@ #ifndef TSAN_FLAGS_H #define TSAN_FLAGS_H -// ----------- ATTENTION ------------- -// ThreadSanitizer user may provide its implementation of weak -// symbol __tsan::OverrideFlags(__tsan::Flags). Therefore, this -// header may be included in the user code, and shouldn't include -// other headers from TSan or common sanitizer runtime. - #include "sanitizer_common/sanitizer_flags.h" +#include "sanitizer_common/sanitizer_deadlock_detector_interface.h" namespace __tsan { -struct Flags : CommonFlags { +struct Flags : CommonFlags, DDFlags { // Enable dynamic annotations, otherwise they are no-ops. bool enable_annotations; - // Supress a race report if we've already output another race report + // Suppress a race report if we've already output another race report // with the same stack. bool suppress_equal_stacks; - // Supress a race report if we've already output another race report + // Suppress a race report if we've already output another race report // on the same address. bool suppress_equal_addresses; // Suppress weird race reports that can be seen if JVM is embed @@ -40,6 +35,8 @@ struct Flags : CommonFlags { bool report_thread_leaks; // Report destruction of a locked mutex? bool report_destroy_locked; + // Report incorrect usages of mutexes and mutex annotations? + bool report_mutex_bugs; // Report violations of async signal-safety // (e.g. malloc() call from a signal handler). bool report_signal_unsafe; @@ -85,6 +82,8 @@ struct Flags : CommonFlags { // 1 - reasonable level of synchronization (write->read) // 2 - global synchronization of all IO operations int io_sync; + // Die after multi-threaded fork if the child creates new threads. + bool die_after_fork; }; Flags *flags(); diff --git a/libsanitizer/tsan/tsan_interceptors.cc b/libsanitizer/tsan/tsan_interceptors.cc index 0574beb..19a3b7b 100644 --- a/libsanitizer/tsan/tsan_interceptors.cc +++ b/libsanitizer/tsan/tsan_interceptors.cc @@ -27,7 +27,7 @@ using namespace __tsan; // NOLINT -const int kSigCount = 64; +const int kSigCount = 65; struct my_siginfo_t { // The size is determined by looking at sizeof of real siginfo_t on linux. @@ -51,6 +51,7 @@ extern "C" int pthread_sigmask(int how, const __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset); // REAL(sigfillset) defined in common interceptors. DECLARE_REAL(int, sigfillset, __sanitizer_sigset_t *set) +DECLARE_REAL(int, fflush, __sanitizer_FILE *fp) extern "C" void *pthread_self(); extern "C" void _exit(int status); extern "C" int *__errno_location(); @@ -60,6 +61,7 @@ extern "C" void *__libc_calloc(uptr size, uptr n); extern "C" void *__libc_realloc(void *ptr, uptr size); extern "C" void __libc_free(void *ptr); extern "C" int mallopt(int param, int value); +extern __sanitizer_FILE *stdout, *stderr; const int PTHREAD_MUTEX_RECURSIVE = 1; const int PTHREAD_MUTEX_RECURSIVE_NP = 1; const int EINVAL = 22; @@ -71,6 +73,7 @@ const int SIGABRT = 6; const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; +const int SIGTERM = 15; const int SIGBUS = 7; const int SIGSYS = 31; void *const MAP_FAILED = (void*)-1; @@ -142,7 +145,6 @@ void InitializeLibIgnore() { static SignalContext *SigCtx(ThreadState *thr) { SignalContext *ctx = (SignalContext*)thr->signal_ctx; if (ctx == 0 && thr->is_alive) { - ScopedInRtl in_rtl; ctx = (SignalContext*)MmapOrDie(sizeof(*ctx), "SignalContext"); MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx)); thr->signal_ctx = ctx; @@ -159,7 +161,6 @@ class ScopedInterceptor { private: ThreadState *const thr_; const uptr pc_; - const int in_rtl_; bool in_ignored_lib_; }; @@ -167,16 +168,12 @@ ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc) : thr_(thr) , pc_(pc) - , in_rtl_(thr->in_rtl) , in_ignored_lib_(false) { - if (thr_->in_rtl == 0) { + if (!thr_->ignore_interceptors) { Initialize(thr); FuncEntry(thr, pc); - thr_->in_rtl++; - DPrintf("#%d: intercept %s()\n", thr_->tid, fname); - } else { - thr_->in_rtl++; } + DPrintf("#%d: intercept %s()\n", thr_->tid, fname); if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) { in_ignored_lib_ = true; thr_->in_ignored_lib = true; @@ -189,18 +186,14 @@ ScopedInterceptor::~ScopedInterceptor() { thr_->in_ignored_lib = false; ThreadIgnoreEnd(thr_, pc_); } - thr_->in_rtl--; - if (thr_->in_rtl == 0) { - FuncExit(thr_); + if (!thr_->ignore_interceptors) { ProcessPendingSignals(thr_); + FuncExit(thr_); } - CHECK_EQ(in_rtl_, thr_->in_rtl); } #define SCOPED_INTERCEPTOR_RAW(func, ...) \ ThreadState *thr = cur_thread(); \ - StatInc(thr, StatInterceptor); \ - StatInc(thr, StatInt_##func); \ const uptr caller_pc = GET_CALLER_PC(); \ ScopedInterceptor si(thr, #func, caller_pc); \ const uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \ @@ -210,15 +203,16 @@ ScopedInterceptor::~ScopedInterceptor() { #define SCOPED_TSAN_INTERCEPTOR(func, ...) \ SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ if (REAL(func) == 0) { \ - Printf("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \ + Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \ Die(); \ } \ - if (thr->in_rtl > 1 || thr->in_ignored_lib) \ + if (thr->ignore_interceptors || thr->in_ignored_lib) \ return REAL(func)(__VA_ARGS__); \ /**/ #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__) #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) +#define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name)) @@ -233,6 +227,13 @@ struct BlockingCall { } SignalContext *ctx; + + // When we are in a "blocking call", we process signals asynchronously + // (right when they arrive). In this context we do not expect to be + // executing any user/runtime code. The known interceptor sequence when + // this is not true is: pthread_join -> munmap(stack). It's fine + // to ignore munmap in this case -- we handle stack shadow separately. + ScopedIgnoreInterceptors ignore_interceptors; }; TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) { @@ -258,22 +259,14 @@ TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { TSAN_INTERCEPTOR(void*, dlopen, const char *filename, int flag) { SCOPED_INTERCEPTOR_RAW(dlopen, filename, flag); - // dlopen will execute global constructors, so it must be not in rtl. - CHECK_EQ(thr->in_rtl, 1); - thr->in_rtl = 0; void *res = REAL(dlopen)(filename, flag); - thr->in_rtl = 1; libignore()->OnLibraryLoaded(filename); return res; } TSAN_INTERCEPTOR(int, dlclose, void *handle) { SCOPED_INTERCEPTOR_RAW(dlclose, handle); - // dlclose will execute global destructors, so it must be not in rtl. - CHECK_EQ(thr->in_rtl, 1); - thr->in_rtl = 0; int res = REAL(dlclose)(handle); - thr->in_rtl = 1; libignore()->OnLibraryUnloaded(); return res; } @@ -301,7 +294,6 @@ class AtExitContext { } void exit(ThreadState *thr, uptr pc) { - CHECK_EQ(thr->in_rtl, 0); for (;;) { atexit_t f = 0; void *arg = 0; @@ -313,14 +305,12 @@ class AtExitContext { f = stack_[pos_]; arg = args_[pos_]; is_on_exit = is_on_exits_[pos_]; - ScopedInRtl in_rtl; Acquire(thr, pc, (uptr)this); } } if (f == 0) break; DPrintf("#%d: executing atexit func %p\n", thr->tid, f); - CHECK_EQ(thr->in_rtl, 0); if (is_on_exit) ((void(*)(int status, void *arg))f)(0, arg); else @@ -342,7 +332,9 @@ static AtExitContext *atexit_ctx; TSAN_INTERCEPTOR(int, atexit, void (*f)()) { if (cur_thread()->in_symbolizer) return 0; - SCOPED_TSAN_INTERCEPTOR(atexit, f); + // We want to setup the atexit callback even if we are in ignored lib + // or after fork. + SCOPED_INTERCEPTOR_RAW(atexit, f); return atexit_ctx->atexit(thr, pc, false, (void(*)())f, 0); } @@ -413,7 +405,6 @@ static void LongJmp(ThreadState *thr, uptr *env) { // FIXME: put everything below into a common extern "C" block? extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) { - ScopedInRtl in_rtl; SetJmp(cur_thread(), sp, mangled_sp); } @@ -587,21 +578,21 @@ void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) { user_free(thr, pc, ptr); SANITIZER_INTERFACE_ATTRIBUTE -void operator delete(void *ptr); -void operator delete(void *ptr) { +void operator delete(void *ptr) throw(); +void operator delete(void *ptr) throw() { OPERATOR_DELETE_BODY(_ZdlPv); } SANITIZER_INTERFACE_ATTRIBUTE -void operator delete[](void *ptr); -void operator delete[](void *ptr) { - OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t); +void operator delete[](void *ptr) throw(); +void operator delete[](void *ptr) throw() { + OPERATOR_DELETE_BODY(_ZdaPv); } SANITIZER_INTERFACE_ATTRIBUTE void operator delete(void *ptr, std::nothrow_t const&); void operator delete(void *ptr, std::nothrow_t const&) { - OPERATOR_DELETE_BODY(_ZdaPv); + OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t); } SANITIZER_INTERFACE_ATTRIBUTE @@ -643,20 +634,6 @@ TSAN_INTERCEPTOR(int, memcmp, const void *s1, const void *s2, uptr n) { return res; } -TSAN_INTERCEPTOR(void*, memchr, void *s, int c, uptr n) { - SCOPED_TSAN_INTERCEPTOR(memchr, s, c, n); - void *res = REAL(memchr)(s, c, n); - uptr len = res ? (char*)res - (char*)s + 1 : n; - MemoryAccessRange(thr, pc, (uptr)s, len, false); - return res; -} - -TSAN_INTERCEPTOR(void*, memrchr, char *s, int c, uptr n) { - SCOPED_TSAN_INTERCEPTOR(memrchr, s, c, n); - MemoryAccessRange(thr, pc, (uptr)s, n, false); - return REAL(memrchr)(s, c, n); -} - TSAN_INTERCEPTOR(void*, memmove, void *dst, void *src, uptr n) { SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n); MemoryAccessRange(thr, pc, (uptr)dst, n, true); @@ -827,7 +804,6 @@ static void thread_finalize(void *v) { return; } { - ScopedInRtl in_rtl; ThreadState *thr = cur_thread(); ThreadFinish(thr); SignalContext *sctx = thr->signal_ctx; @@ -852,7 +828,8 @@ extern "C" void *__tsan_thread_start_func(void *arg) { int tid = 0; { ThreadState *thr = cur_thread(); - ScopedInRtl in_rtl; + // Thread-local state is not initialized yet. + ScopedIgnoreInterceptors ignore; if (pthread_setspecific(g_thread_finalize_key, (void *)kPthreadDestructorIterations)) { Printf("ThreadSanitizer: failed to set thread key\n"); @@ -862,7 +839,6 @@ extern "C" void *__tsan_thread_start_func(void *arg) { pthread_yield(); atomic_store(&p->tid, 0, memory_order_release); ThreadStart(thr, tid, GetTid()); - CHECK_EQ(thr->in_rtl, 1); } void *res = callback(param); // Prevent the callback from being tail called, @@ -875,6 +851,17 @@ extern "C" void *__tsan_thread_start_func(void *arg) { TSAN_INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), void * param) { SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param); + if (ctx->after_multithreaded_fork) { + if (flags()->die_after_fork) { + Report("ThreadSanitizer: starting new threads after multi-threaded " + "fork is not supported. Dying (set die_after_fork=0 to override)\n"); + Die(); + } else { + VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded " + "fork is not supported (pid %d). Continuing because of " + "die_after_fork=0, but you are on your own\n", internal_getpid()); + } + } __sanitizer_pthread_attr_t myattr; if (attr == 0) { pthread_attr_init(&myattr); @@ -882,13 +869,20 @@ TSAN_INTERCEPTOR(int, pthread_create, } int detached = 0; REAL(pthread_attr_getdetachstate)(attr, &detached); - AdjustStackSizeLinux(attr); + AdjustStackSize(attr); ThreadParam p; p.callback = callback; p.param = param; atomic_store(&p.tid, 0, memory_order_relaxed); - int res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); + int res = -1; + { + // Otherwise we see false positives in pthread stack manipulation. + ScopedIgnoreInterceptors ignore; + ThreadIgnoreBegin(thr, pc); + res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); + ThreadIgnoreEnd(thr, pc); + } if (res == 0) { int tid = ThreadCreate(thr, pc, *(uptr*)th, detached); CHECK_NE(tid, 0); @@ -904,7 +898,9 @@ TSAN_INTERCEPTOR(int, pthread_create, TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret); int tid = ThreadTid(thr, pc, (uptr)th); + ThreadIgnoreBegin(thr, pc); int res = BLOCK_REAL(pthread_join)(th, ret); + ThreadIgnoreEnd(thr, pc); if (res == 0) { ThreadJoin(thr, pc, tid); } @@ -921,6 +917,122 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) { return res; } +// Problem: +// NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2). +// pthread_cond_t has different size in the different versions. +// If call new REAL functions for old pthread_cond_t, they will corrupt memory +// after pthread_cond_t (old cond is smaller). +// If we call old REAL functions for new pthread_cond_t, we will lose some +// functionality (e.g. old functions do not support waiting against +// CLOCK_REALTIME). +// Proper handling would require to have 2 versions of interceptors as well. +// But this is messy, in particular requires linker scripts when sanitizer +// runtime is linked into a shared library. +// Instead we assume we don't have dynamic libraries built against old +// pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag +// that allows to work with old libraries (but this mode does not support +// some features, e.g. pthread_condattr_getpshared). +static void *init_cond(void *c, bool force = false) { + // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions. + // So we allocate additional memory on the side large enough to hold + // any pthread_cond_t object. Always call new REAL functions, but pass + // the aux object to them. + // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes + // first word of pthread_cond_t to zero. + // It's all relevant only for linux. + if (!common_flags()->legacy_pthread_cond) + return c; + atomic_uintptr_t *p = (atomic_uintptr_t*)c; + uptr cond = atomic_load(p, memory_order_acquire); + if (!force && cond != 0) + return (void*)cond; + void *newcond = WRAP(malloc)(pthread_cond_t_sz); + internal_memset(newcond, 0, pthread_cond_t_sz); + if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond, + memory_order_acq_rel)) + return newcond; + WRAP(free)(newcond); + return (void*)cond; +} + +struct CondMutexUnlockCtx { + ThreadState *thr; + uptr pc; + void *m; +}; + +static void cond_mutex_unlock(CondMutexUnlockCtx *arg) { + MutexLock(arg->thr, arg->pc, (uptr)arg->m); +} + +INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { + void *cond = init_cond(c, true); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a); + MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); + return REAL(pthread_cond_init)(cond, a); +} + +INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m); + MutexUnlock(thr, pc, (uptr)m); + MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); + CondMutexUnlockCtx arg = {thr, pc, m}; + // This ensures that we handle mutex lock even in case of pthread_cancel. + // See test/tsan/cond_cancel.cc. + int res = call_pthread_cancel_with_cleanup( + (int(*)(void *c, void *m, void *abstime))REAL(pthread_cond_wait), + cond, m, 0, (void(*)(void *arg))cond_mutex_unlock, &arg); + if (res == errno_EOWNERDEAD) + MutexRepair(thr, pc, (uptr)m); + MutexLock(thr, pc, (uptr)m); + return res; +} + +INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime); + MutexUnlock(thr, pc, (uptr)m); + MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); + CondMutexUnlockCtx arg = {thr, pc, m}; + // This ensures that we handle mutex lock even in case of pthread_cancel. + // See test/tsan/cond_cancel.cc. + int res = call_pthread_cancel_with_cleanup( + REAL(pthread_cond_timedwait), cond, m, abstime, + (void(*)(void *arg))cond_mutex_unlock, &arg); + if (res == errno_EOWNERDEAD) + MutexRepair(thr, pc, (uptr)m); + MutexLock(thr, pc, (uptr)m); + return res; +} + +INTERCEPTOR(int, pthread_cond_signal, void *c) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond); + MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); + return REAL(pthread_cond_signal)(cond); +} + +INTERCEPTOR(int, pthread_cond_broadcast, void *c) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond); + MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); + return REAL(pthread_cond_broadcast)(cond); +} + +INTERCEPTOR(int, pthread_cond_destroy, void *c) { + void *cond = init_cond(c); + SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond); + MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); + int res = REAL(pthread_cond_destroy)(cond); + if (common_flags()->legacy_pthread_cond) { + // Free our aux cond and zero the pointer to not leave dangling pointers. + WRAP(free)(cond); + atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed); + } + return res; +} + TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a); int res = REAL(pthread_mutex_init)(m, a); @@ -952,7 +1064,7 @@ TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { if (res == EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); if (res == 0 || res == EOWNERDEAD) - MutexLock(thr, pc, (uptr)m); + MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); return res; } @@ -996,7 +1108,7 @@ TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); int res = REAL(pthread_spin_trylock)(m); if (res == 0) { - MutexLock(thr, pc, (uptr)m); + MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); } return res; } @@ -1039,7 +1151,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); int res = REAL(pthread_rwlock_tryrdlock)(m); if (res == 0) { - MutexReadLock(thr, pc, (uptr)m); + MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); } return res; } @@ -1066,7 +1178,7 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); int res = REAL(pthread_rwlock_trywrlock)(m); if (res == 0) { - MutexLock(thr, pc, (uptr)m); + MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true); } return res; } @@ -1087,23 +1199,6 @@ TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { return res; } -TSAN_INTERCEPTOR(int, pthread_cond_destroy, void *c) { - SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, c); - MemoryWrite(thr, pc, (uptr)c, kSizeLog1); - int res = REAL(pthread_cond_destroy)(c); - return res; -} - -TSAN_INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, - void *abstime) { - SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, c, m, abstime); - MutexUnlock(thr, pc, (uptr)m); - MemoryRead(thr, pc, (uptr)c, kSizeLog1); - int res = REAL(pthread_cond_timedwait)(c, m, abstime); - MutexLock(thr, pc, (uptr)m); - return res; -} - TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); MemoryWrite(thr, pc, (uptr)b, kSizeLog1); @@ -1132,19 +1227,13 @@ TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { SCOPED_INTERCEPTOR_RAW(pthread_once, o, f); - // Using SCOPED_INTERCEPTOR_RAW, because if we are called from an ignored lib, - // the user callback must be executed with thr->in_rtl == 0. if (o == 0 || f == 0) return EINVAL; atomic_uint32_t *a = static_cast<atomic_uint32_t*>(o); u32 v = atomic_load(a, memory_order_acquire); if (v == 0 && atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { - const int old_in_rtl = thr->in_rtl; - thr->in_rtl = 0; (*f)(); - CHECK_EQ(thr->in_rtl, 0); - thr->in_rtl = old_in_rtl; if (!thr->in_ignored_lib) Release(thr, pc, (uptr)o); atomic_store(a, 2, memory_order_release); @@ -1509,10 +1598,9 @@ TSAN_INTERCEPTOR(int, unlink, char *path) { return res; } -TSAN_INTERCEPTOR(void*, fopen, char *path, char *mode) { - SCOPED_TSAN_INTERCEPTOR(fopen, path, mode); - void *res = REAL(fopen)(path, mode); - Acquire(thr, pc, File2addr(path)); +TSAN_INTERCEPTOR(void*, tmpfile, int fake) { + SCOPED_TSAN_INTERCEPTOR(tmpfile, fake); + void *res = REAL(tmpfile)(fake); if (res) { int fd = fileno_unlocked(res); if (fd >= 0) @@ -1521,15 +1609,9 @@ TSAN_INTERCEPTOR(void*, fopen, char *path, char *mode) { return res; } -TSAN_INTERCEPTOR(void*, freopen, char *path, char *mode, void *stream) { - SCOPED_TSAN_INTERCEPTOR(freopen, path, mode, stream); - if (stream) { - int fd = fileno_unlocked(stream); - if (fd >= 0) - FdClose(thr, pc, fd); - } - void *res = REAL(freopen)(path, mode, stream); - Acquire(thr, pc, File2addr(path)); +TSAN_INTERCEPTOR(void*, tmpfile64, int fake) { + SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake); + void *res = REAL(tmpfile64)(fake); if (res) { int fd = fileno_unlocked(res); if (fd >= 0) @@ -1538,19 +1620,6 @@ TSAN_INTERCEPTOR(void*, freopen, char *path, char *mode, void *stream) { return res; } -TSAN_INTERCEPTOR(int, fclose, void *stream) { - // libc file streams can call user-supplied functions, see fopencookie. - { - SCOPED_TSAN_INTERCEPTOR(fclose, stream); - if (stream) { - int fd = fileno_unlocked(stream); - if (fd >= 0) - FdClose(thr, pc, fd); - } - } - return REAL(fclose)(stream); -} - TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) { // libc file streams can call user-supplied functions, see fopencookie. { @@ -1569,14 +1638,6 @@ TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) { return REAL(fwrite)(p, size, nmemb, f); } -TSAN_INTERCEPTOR(int, fflush, void *stream) { - // libc file streams can call user-supplied functions, see fopencookie. - { - SCOPED_TSAN_INTERCEPTOR(fflush, stream); - } - return REAL(fflush)(stream); -} - TSAN_INTERCEPTOR(void, abort, int fake) { SCOPED_TSAN_INTERCEPTOR(abort, fake); REAL(fflush)(0); @@ -1626,30 +1687,106 @@ TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { return res; } +namespace __tsan { + +static void CallUserSignalHandler(ThreadState *thr, bool sync, bool sigact, + int sig, my_siginfo_t *info, void *uctx) { + // Ensure that the handler does not spoil errno. + const int saved_errno = errno; + errno = 99; + // Need to remember pc before the call, because the handler can reset it. + uptr pc = sigact ? + (uptr)sigactions[sig].sa_sigaction : + (uptr)sigactions[sig].sa_handler; + pc += 1; // return address is expected, OutputReport() will undo this + if (sigact) + sigactions[sig].sa_sigaction(sig, info, uctx); + else + sigactions[sig].sa_handler(sig); + // We do not detect errno spoiling for SIGTERM, + // because some SIGTERM handlers do spoil errno but reraise SIGTERM, + // tsan reports false positive in such case. + // It's difficult to properly detect this situation (reraise), + // because in async signal processing case (when handler is called directly + // from rtl_generic_sighandler) we have not yet received the reraised + // signal; and it looks too fragile to intercept all ways to reraise a signal. + if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) { + __tsan::StackTrace stack; + stack.ObtainCurrent(thr, pc); + ThreadRegistryLock l(ctx->thread_registry); + ScopedReport rep(ReportTypeErrnoInSignal); + if (!IsFiredSuppression(ctx, rep, stack)) { + rep.AddStack(&stack); + OutputReport(ctx, rep, rep.GetReport()->stacks[0]); + } + } + errno = saved_errno; +} + +void ProcessPendingSignals(ThreadState *thr) { + SignalContext *sctx = SigCtx(thr); + if (sctx == 0 || sctx->pending_signal_count == 0 || thr->in_signal_handler) + return; + thr->in_signal_handler = true; + sctx->pending_signal_count = 0; + // These are too big for stack. + static THREADLOCAL __sanitizer_sigset_t emptyset, oldset; + REAL(sigfillset)(&emptyset); + pthread_sigmask(SIG_SETMASK, &emptyset, &oldset); + for (int sig = 0; sig < kSigCount; sig++) { + SignalDesc *signal = &sctx->pending_signals[sig]; + if (signal->armed) { + signal->armed = false; + if (sigactions[sig].sa_handler != SIG_DFL + && sigactions[sig].sa_handler != SIG_IGN) { + CallUserSignalHandler(thr, false, signal->sigaction, + sig, &signal->siginfo, &signal->ctx); + } + } + } + pthread_sigmask(SIG_SETMASK, &oldset, 0); + CHECK_EQ(thr->in_signal_handler, true); + thr->in_signal_handler = false; +} + +} // namespace __tsan + +static bool is_sync_signal(SignalContext *sctx, int sig) { + return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || + sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || + // If we are sending signal to ourselves, we must process it now. + (sctx && sig == sctx->int_signal_send); +} + void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, my_siginfo_t *info, void *ctx) { ThreadState *thr = cur_thread(); SignalContext *sctx = SigCtx(thr); + if (sig < 0 || sig >= kSigCount) { + VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig); + return; + } // Don't mess with synchronous signals. - if (sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || - sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || - // If we are sending signal to ourselves, we must process it now. - (sctx && sig == sctx->int_signal_send) || + const bool sync = is_sync_signal(sctx, sig); + if (sync || // If we are in blocking function, we can safely process it now // (but check if we are in a recursive interceptor, // i.e. pthread_join()->munmap()). - (sctx && sctx->in_blocking_func == 1 && thr->in_rtl == 1)) { - int in_rtl = thr->in_rtl; - thr->in_rtl = 0; + (sctx && sctx->in_blocking_func == 1)) { CHECK_EQ(thr->in_signal_handler, false); thr->in_signal_handler = true; - if (sigact) - sigactions[sig].sa_sigaction(sig, info, ctx); - else - sigactions[sig].sa_handler(sig); + if (sctx && sctx->in_blocking_func == 1) { + // We ignore interceptors in blocking functions, + // temporary enbled them again while we are calling user function. + int const i = thr->ignore_interceptors; + thr->ignore_interceptors = 0; + CallUserSignalHandler(thr, sync, sigact, sig, info, ctx); + thr->ignore_interceptors = i; + } else { + CallUserSignalHandler(thr, sync, sigact, sig, info, ctx); + } CHECK_EQ(thr->in_signal_handler, true); thr->in_signal_handler = false; - thr->in_rtl = in_rtl; return; } @@ -1769,11 +1906,7 @@ TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, // and can report false race between malloc and free // inside of getaddrinfo. So ignore memory accesses. ThreadIgnoreBegin(thr, pc); - // getaddrinfo calls fopen, which can be intercepted by user. - thr->in_rtl--; - CHECK_EQ(thr->in_rtl, 0); int res = REAL(getaddrinfo)(node, service, hints, rv); - thr->in_rtl++; ThreadIgnoreEnd(thr, pc); return res; } @@ -1784,8 +1917,7 @@ static void MlockIsUnsupported() { static atomic_uint8_t printed; if (atomic_exchange(&printed, 1, memory_order_relaxed)) return; - if (flags()->verbosity > 0) - Printf("INFO: ThreadSanitizer ignores mlock/mlockall/munlock/munlockall\n"); + VPrintf(1, "INFO: ThreadSanitizer ignores mlock/munlock[all]\n"); } TSAN_INTERCEPTOR(int, mlock, const void *addr, uptr len) { @@ -1809,17 +1941,42 @@ TSAN_INTERCEPTOR(int, munlockall, void) { } TSAN_INTERCEPTOR(int, fork, int fake) { + if (cur_thread()->in_symbolizer) + return REAL(fork)(fake); SCOPED_INTERCEPTOR_RAW(fork, fake); + ForkBefore(thr, pc); int pid = REAL(fork)(fake); if (pid == 0) { // child + ForkChildAfter(thr, pc); FdOnFork(thr, pc); } else if (pid > 0) { // parent + ForkParentAfter(thr, pc); + } else { + // error + ForkParentAfter(thr, pc); } return pid; } +TSAN_INTERCEPTOR(int, vfork, int fake) { + // Some programs (e.g. openjdk) call close for all file descriptors + // in the child process. Under tsan it leads to false positives, because + // address space is shared, so the parent process also thinks that + // the descriptors are closed (while they are actually not). + // This leads to false positives due to missed synchronization. + // Strictly saying this is undefined behavior, because vfork child is not + // allowed to call any functions other than exec/exit. But this is what + // openjdk does, so we want to handle it. + // We could disable interceptors in the child process. But it's not possible + // to simply intercept and wrap vfork, because vfork child is not allowed + // to return from the function that calls vfork, and that's exactly what + // we would do. So this would require some assembly trickery as well. + // Instead we simply turn vfork into fork. + return WRAP(fork)(fake); +} + static int OnExit(ThreadState *thr) { int status = Finalize(thr); REAL(fflush)(0); @@ -1841,19 +1998,10 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, } #include "sanitizer_common/sanitizer_platform_interceptors.h" -// Causes interceptor recursion (getpwuid_r() calls fopen()) -#undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS -#undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS // Causes interceptor recursion (getaddrinfo() and fopen()) #undef SANITIZER_INTERCEPT_GETADDRINFO -#undef SANITIZER_INTERCEPT_GETNAMEINFO -// Causes interceptor recursion (glob64() calls lstat64()) -#undef SANITIZER_INTERCEPT_GLOB #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) -#define COMMON_INTERCEPTOR_UNPOISON_PARAM(ctx, count) \ - do { \ - } while (false) #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ @@ -1871,6 +2019,19 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, ctx = (void *)&_ctx; \ (void) ctx; +#define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \ + Acquire(thr, pc, File2addr(path)); \ + if (file) { \ + int fd = fileno_unlocked(file); \ + if (fd >= 0) FdFileCreate(thr, pc, fd); \ + } + +#define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \ + if (file) { \ + int fd = fileno_unlocked(file); \ + if (fd >= 0) FdClose(thr, pc, fd); \ + } + #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd) @@ -1887,7 +2048,7 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name) #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ - CTX()->thread_registry->SetThreadNameByUserId(thread, name) + __tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name) #define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name) @@ -1914,6 +2075,8 @@ static void HandleRecvmsg(ThreadState *thr, uptr pc, #define TSAN_SYSCALL() \ ThreadState *thr = cur_thread(); \ + if (thr->ignore_interceptors) \ + return; \ ScopedSyscall scoped_syscall(thr) \ /**/ @@ -1922,15 +2085,11 @@ struct ScopedSyscall { explicit ScopedSyscall(ThreadState *thr) : thr(thr) { - if (thr->in_rtl == 0) - Initialize(thr); - thr->in_rtl++; + Initialize(thr); } ~ScopedSyscall() { - thr->in_rtl--; - if (thr->in_rtl == 0) - ProcessPendingSignals(thr); + ProcessPendingSignals(thr); } }; @@ -1942,12 +2101,12 @@ static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { static void syscall_acquire(uptr pc, uptr addr) { TSAN_SYSCALL(); Acquire(thr, pc, addr); - Printf("syscall_acquire(%p)\n", addr); + DPrintf("syscall_acquire(%p)\n", addr); } static void syscall_release(uptr pc, uptr addr) { TSAN_SYSCALL(); - Printf("syscall_release(%p)\n", addr); + DPrintf("syscall_release(%p)\n", addr); Release(thr, pc, addr); } @@ -1959,26 +2118,32 @@ static void syscall_fd_close(uptr pc, int fd) { static USED void syscall_fd_acquire(uptr pc, int fd) { TSAN_SYSCALL(); FdAcquire(thr, pc, fd); - Printf("syscall_fd_acquire(%p)\n", fd); + DPrintf("syscall_fd_acquire(%p)\n", fd); } static USED void syscall_fd_release(uptr pc, int fd) { TSAN_SYSCALL(); - Printf("syscall_fd_release(%p)\n", fd); + DPrintf("syscall_fd_release(%p)\n", fd); FdRelease(thr, pc, fd); } static void syscall_pre_fork(uptr pc) { TSAN_SYSCALL(); + ForkBefore(thr, pc); } -static void syscall_post_fork(uptr pc, int res) { +static void syscall_post_fork(uptr pc, int pid) { TSAN_SYSCALL(); - if (res == 0) { + if (pid == 0) { // child + ForkChildAfter(thr, pc); FdOnFork(thr, pc); - } else if (res > 0) { + } else if (pid > 0) { // parent + ForkParentAfter(thr, pc); + } else { + // error + ForkParentAfter(thr, pc); } } @@ -2027,68 +2192,21 @@ static void finalize(void *arg) { uptr pc = 0; atexit_ctx->exit(thr, pc); int status = Finalize(thr); - REAL(fflush)(0); + // Make sure the output is not lost. + // Flushing all the streams here may freeze the process if a child thread is + // performing file stream operations at the same time. + REAL(fflush)(stdout); + REAL(fflush)(stderr); if (status) REAL(_exit)(status); } -void ProcessPendingSignals(ThreadState *thr) { - CHECK_EQ(thr->in_rtl, 0); - SignalContext *sctx = SigCtx(thr); - if (sctx == 0 || sctx->pending_signal_count == 0 || thr->in_signal_handler) - return; - Context *ctx = CTX(); - thr->in_signal_handler = true; - sctx->pending_signal_count = 0; - // These are too big for stack. - static THREADLOCAL __sanitizer_sigset_t emptyset, oldset; - REAL(sigfillset)(&emptyset); - pthread_sigmask(SIG_SETMASK, &emptyset, &oldset); - for (int sig = 0; sig < kSigCount; sig++) { - SignalDesc *signal = &sctx->pending_signals[sig]; - if (signal->armed) { - signal->armed = false; - if (sigactions[sig].sa_handler != SIG_DFL - && sigactions[sig].sa_handler != SIG_IGN) { - // Insure that the handler does not spoil errno. - const int saved_errno = errno; - errno = 0; - if (signal->sigaction) - sigactions[sig].sa_sigaction(sig, &signal->siginfo, &signal->ctx); - else - sigactions[sig].sa_handler(sig); - if (flags()->report_bugs && errno != 0) { - ScopedInRtl in_rtl; - __tsan::StackTrace stack; - uptr pc = signal->sigaction ? - (uptr)sigactions[sig].sa_sigaction : - (uptr)sigactions[sig].sa_handler; - pc += 1; // return address is expected, OutputReport() will undo this - stack.Init(&pc, 1); - ThreadRegistryLock l(ctx->thread_registry); - ScopedReport rep(ReportTypeErrnoInSignal); - if (!IsFiredSuppression(ctx, rep, stack)) { - rep.AddStack(&stack); - OutputReport(ctx, rep, rep.GetReport()->stacks[0]); - } - } - errno = saved_errno; - } - } - } - pthread_sigmask(SIG_SETMASK, &oldset, 0); - CHECK_EQ(thr->in_signal_handler, true); - thr->in_signal_handler = false; -} - static void unreachable() { - Printf("FATAL: ThreadSanitizer: unreachable called\n"); + Report("FATAL: ThreadSanitizer: unreachable called\n"); Die(); } void InitializeInterceptors() { - CHECK_GT(cur_thread()->in_rtl, 0); - // We need to setup it early, because functions like dlsym() can call it. REAL(memset) = internal_memset; REAL(memcpy) = internal_memcpy; @@ -2098,7 +2216,7 @@ void InitializeInterceptors() { mallopt(1, 0); // M_MXFAST mallopt(-3, 32*1024); // M_MMAP_THRESHOLD - SANITIZER_COMMON_INTERCEPTORS_INIT; + InitializeCommonInterceptors(); // We can not use TSAN_INTERCEPT to get setjmp addr, // because it does &setjmp and setjmp is not present in some versions of libc. @@ -2128,8 +2246,6 @@ void InitializeInterceptors() { TSAN_INTERCEPT(strlen); TSAN_INTERCEPT(memset); TSAN_INTERCEPT(memcpy); - TSAN_INTERCEPT(memchr); - TSAN_INTERCEPT(memrchr); TSAN_INTERCEPT(memmove); TSAN_INTERCEPT(memcmp); TSAN_INTERCEPT(strchr); @@ -2144,6 +2260,13 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_join); TSAN_INTERCEPT(pthread_detach); + TSAN_INTERCEPT_VER(pthread_cond_init, "GLIBC_2.3.2"); + TSAN_INTERCEPT_VER(pthread_cond_signal, "GLIBC_2.3.2"); + TSAN_INTERCEPT_VER(pthread_cond_broadcast, "GLIBC_2.3.2"); + TSAN_INTERCEPT_VER(pthread_cond_wait, "GLIBC_2.3.2"); + TSAN_INTERCEPT_VER(pthread_cond_timedwait, "GLIBC_2.3.2"); + TSAN_INTERCEPT_VER(pthread_cond_destroy, "GLIBC_2.3.2"); + TSAN_INTERCEPT(pthread_mutex_init); TSAN_INTERCEPT(pthread_mutex_destroy); TSAN_INTERCEPT(pthread_mutex_trylock); @@ -2165,9 +2288,6 @@ void InitializeInterceptors() { TSAN_INTERCEPT(pthread_rwlock_timedwrlock); TSAN_INTERCEPT(pthread_rwlock_unlock); - INTERCEPT_FUNCTION_VER(pthread_cond_destroy, "GLIBC_2.3.2"); - INTERCEPT_FUNCTION_VER(pthread_cond_timedwait, "GLIBC_2.3.2"); - TSAN_INTERCEPT(pthread_barrier_init); TSAN_INTERCEPT(pthread_barrier_destroy); TSAN_INTERCEPT(pthread_barrier_wait); @@ -2223,12 +2343,10 @@ void InitializeInterceptors() { TSAN_INTERCEPT(recv); TSAN_INTERCEPT(unlink); - TSAN_INTERCEPT(fopen); - TSAN_INTERCEPT(freopen); - TSAN_INTERCEPT(fclose); + TSAN_INTERCEPT(tmpfile); + TSAN_INTERCEPT(tmpfile64); TSAN_INTERCEPT(fread); TSAN_INTERCEPT(fwrite); - TSAN_INTERCEPT(fflush); TSAN_INTERCEPT(abort); TSAN_INTERCEPT(puts); TSAN_INTERCEPT(rmdir); @@ -2255,6 +2373,7 @@ void InitializeInterceptors() { TSAN_INTERCEPT(munlockall); TSAN_INTERCEPT(fork); + TSAN_INTERCEPT(vfork); TSAN_INTERCEPT(dlopen); TSAN_INTERCEPT(dlclose); TSAN_INTERCEPT(on_exit); @@ -2280,16 +2399,19 @@ void InitializeInterceptors() { FdInit(); } -void internal_start_thread(void(*func)(void *arg), void *arg) { - // Start the thread with signals blocked, otherwise it can steal users - // signals. - __sanitizer_kernel_sigset_t set, old; +void *internal_start_thread(void(*func)(void *arg), void *arg) { + // Start the thread with signals blocked, otherwise it can steal user signals. + __sanitizer_sigset_t set, old; internal_sigfillset(&set); internal_sigprocmask(SIG_SETMASK, &set, &old); void *th; REAL(pthread_create)(&th, 0, (void*(*)(void *arg))func, arg); - REAL(pthread_detach)(th); internal_sigprocmask(SIG_SETMASK, &old, 0); + return th; +} + +void internal_join_thread(void *th) { + REAL(pthread_join)(th, 0); } } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_interface_ann.cc b/libsanitizer/tsan/tsan_interface_ann.cc index 38224f4..20e3d9a 100644 --- a/libsanitizer/tsan/tsan_interface_ann.cc +++ b/libsanitizer/tsan/tsan_interface_ann.cc @@ -31,22 +31,16 @@ class ScopedAnnotation { public: ScopedAnnotation(ThreadState *thr, const char *aname, const char *f, int l, uptr pc) - : thr_(thr) - , in_rtl_(thr->in_rtl) { - CHECK_EQ(thr_->in_rtl, 0); + : thr_(thr) { FuncEntry(thr_, pc); - thr_->in_rtl++; DPrintf("#%d: annotation %s() %s:%d\n", thr_->tid, aname, f, l); } ~ScopedAnnotation() { - thr_->in_rtl--; - CHECK_EQ(in_rtl_, thr_->in_rtl); FuncExit(thr_); } private: ThreadState *const thr_; - const int in_rtl_; }; #define SCOPED_ANNOTATION(typ) \ @@ -56,7 +50,7 @@ class ScopedAnnotation { const uptr caller_pc = (uptr)__builtin_return_address(0); \ StatInc(thr, StatAnnotation); \ StatInc(thr, Stat##typ); \ - ScopedAnnotation sa(thr, __FUNCTION__, f, l, caller_pc); \ + ScopedAnnotation sa(thr, __func__, f, l, caller_pc); \ const uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \ (void)pc; \ /**/ @@ -309,7 +303,7 @@ void INTERFACE_ATTRIBUTE AnnotateFlushExpectedRaces(char *f, int l) { while (dyn_ann_ctx->expect.next != &dyn_ann_ctx->expect) { ExpectRace *race = dyn_ann_ctx->expect.next; if (race->hitcount == 0) { - CTX()->nmissed_expected++; + ctx->nmissed_expected++; ReportMissedExpectedRace(race); } race->prev->next = race->next; diff --git a/libsanitizer/tsan/tsan_interface_atomic.cc b/libsanitizer/tsan/tsan_interface_atomic.cc index 180d87b..3f5a4cc 100644 --- a/libsanitizer/tsan/tsan_interface_atomic.cc +++ b/libsanitizer/tsan/tsan_interface_atomic.cc @@ -10,7 +10,7 @@ //===----------------------------------------------------------------------===// // ThreadSanitizer atomic operations are based on C++11/C1x standards. -// For background see C++11 standard. A slightly older, publically +// For background see C++11 standard. A slightly older, publicly // available draft of the standard (not entirely up-to-date, but close enough // for casual browsing) is available here: // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf @@ -19,7 +19,7 @@ #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_stacktrace.h" -#include "tsan_interface_atomic.h" +#include "sanitizer_common/sanitizer_mutex.h" #include "tsan_flags.h" #include "tsan_rtl.h" @@ -28,42 +28,52 @@ using namespace __tsan; // NOLINT #define SCOPED_ATOMIC(func, ...) \ const uptr callpc = (uptr)__builtin_return_address(0); \ uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \ - mo = ConvertOrder(mo); \ mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \ ThreadState *const thr = cur_thread(); \ + if (thr->ignore_interceptors) \ + return NoTsanAtomic##func(__VA_ARGS__); \ AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \ - ScopedAtomic sa(thr, callpc, a, mo, __FUNCTION__); \ + ScopedAtomic sa(thr, callpc, a, mo, __func__); \ return Atomic##func(thr, pc, __VA_ARGS__); \ /**/ -// Some shortcuts. -typedef __tsan_memory_order morder; -typedef __tsan_atomic8 a8; -typedef __tsan_atomic16 a16; -typedef __tsan_atomic32 a32; -typedef __tsan_atomic64 a64; -typedef __tsan_atomic128 a128; -const morder mo_relaxed = __tsan_memory_order_relaxed; -const morder mo_consume = __tsan_memory_order_consume; -const morder mo_acquire = __tsan_memory_order_acquire; -const morder mo_release = __tsan_memory_order_release; -const morder mo_acq_rel = __tsan_memory_order_acq_rel; -const morder mo_seq_cst = __tsan_memory_order_seq_cst; +// These should match declarations from public tsan_interface_atomic.h header. +typedef unsigned char a8; +typedef unsigned short a16; // NOLINT +typedef unsigned int a32; +typedef unsigned long long a64; // NOLINT +#if defined(__SIZEOF_INT128__) \ + || (__clang_major__ * 100 + __clang_minor__ >= 302) +__extension__ typedef __int128 a128; +# define __TSAN_HAS_INT128 1 +#else +# define __TSAN_HAS_INT128 0 +#endif + +// Protects emulation of 128-bit atomic operations. +static StaticSpinMutex mutex128; + +// Part of ABI, do not change. +// http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/atomic?view=markup +typedef enum { + mo_relaxed, + mo_consume, + mo_acquire, + mo_release, + mo_acq_rel, + mo_seq_cst +} morder; class ScopedAtomic { public: ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a, morder mo, const char *func) : thr_(thr) { - CHECK_EQ(thr_->in_rtl, 0); - ProcessPendingSignals(thr); FuncEntry(thr_, pc); DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo); - thr_->in_rtl++; } ~ScopedAtomic() { - thr_->in_rtl--; - CHECK_EQ(thr_->in_rtl, 0); + ProcessPendingSignals(thr_); FuncExit(thr_); } private: @@ -108,27 +118,6 @@ static bool IsAcqRelOrder(morder mo) { return mo == mo_acq_rel || mo == mo_seq_cst; } -static morder ConvertOrder(morder mo) { - if (mo > (morder)100500) { - mo = morder(mo - 100500); - if (mo == morder(1 << 0)) - mo = mo_relaxed; - else if (mo == morder(1 << 1)) - mo = mo_consume; - else if (mo == morder(1 << 2)) - mo = mo_acquire; - else if (mo == morder(1 << 3)) - mo = mo_release; - else if (mo == morder(1 << 4)) - mo = mo_acq_rel; - else if (mo == morder(1 << 5)) - mo = mo_seq_cst; - } - CHECK_GE(mo, mo_relaxed); - CHECK_LE(mo, mo_seq_cst); - return mo; -} - template<typename T> T func_xchg(volatile T *v, T op) { T res = __sync_lock_test_and_set(v, op); // __sync_lock_test_and_set does not contain full barrier. @@ -178,48 +167,56 @@ template<typename T> T func_cas(volatile T *v, T cmp, T xch) { // from non-instrumented code. #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16 a128 func_xchg(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); a128 cmp = *v; *v = op; return cmp; } a128 func_add(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); a128 cmp = *v; *v = cmp + op; return cmp; } a128 func_sub(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); a128 cmp = *v; *v = cmp - op; return cmp; } a128 func_and(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); a128 cmp = *v; *v = cmp & op; return cmp; } a128 func_or(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); a128 cmp = *v; *v = cmp | op; return cmp; } a128 func_xor(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); a128 cmp = *v; *v = cmp ^ op; return cmp; } a128 func_nand(volatile a128 *v, a128 op) { + SpinMutexLock lock(&mutex128); a128 cmp = *v; *v = ~(cmp & op); return cmp; } a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) { + SpinMutexLock lock(&mutex128); a128 cur = *v; if (cur == cmp) *v = xch; @@ -241,26 +238,74 @@ static int SizeLog() { // this leads to false negatives only in very obscure cases. } +static atomic_uint8_t *to_atomic(const volatile a8 *a) { + return (atomic_uint8_t*)a; +} + +static atomic_uint16_t *to_atomic(const volatile a16 *a) { + return (atomic_uint16_t*)a; +} + +static atomic_uint32_t *to_atomic(const volatile a32 *a) { + return (atomic_uint32_t*)a; +} + +static atomic_uint64_t *to_atomic(const volatile a64 *a) { + return (atomic_uint64_t*)a; +} + +static memory_order to_mo(morder mo) { + switch (mo) { + case mo_relaxed: return memory_order_relaxed; + case mo_consume: return memory_order_consume; + case mo_acquire: return memory_order_acquire; + case mo_release: return memory_order_release; + case mo_acq_rel: return memory_order_acq_rel; + case mo_seq_cst: return memory_order_seq_cst; + } + CHECK(0); + return memory_order_seq_cst; +} + +template<typename T> +static T NoTsanAtomicLoad(const volatile T *a, morder mo) { + return atomic_load(to_atomic(a), to_mo(mo)); +} + +static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) { + SpinMutexLock lock(&mutex128); + return *a; +} + template<typename T> static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) { CHECK(IsLoadOrder(mo)); // This fast-path is critical for performance. // Assume the access is atomic. - if (!IsAcquireOrder(mo) && sizeof(T) <= sizeof(a)) { + if (!IsAcquireOrder(mo)) { MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>()); - return *a; // as if atomic + return NoTsanAtomicLoad(a, mo); } - SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, false); + SyncVar *s = ctx->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, false); AcquireImpl(thr, pc, &s->clock); - T v = *a; + T v = NoTsanAtomicLoad(a, mo); s->mtx.ReadUnlock(); - __sync_synchronize(); MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>()); return v; } template<typename T> +static void NoTsanAtomicStore(volatile T *a, T v, morder mo) { + atomic_store(to_atomic(a), v, to_mo(mo)); +} + +static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) { + SpinMutexLock lock(&mutex128); + *a = v; +} + +template<typename T> static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { CHECK(IsStoreOrder(mo)); @@ -269,21 +314,18 @@ static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v, // Assume the access is atomic. // Strictly saying even relaxed store cuts off release sequence, // so must reset the clock. - if (!IsReleaseOrder(mo) && sizeof(T) <= sizeof(a)) { - *a = v; // as if atomic + if (!IsReleaseOrder(mo)) { + NoTsanAtomicStore(a, v, mo); return; } __sync_synchronize(); - SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); + SyncVar *s = ctx->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); thr->fast_state.IncrementEpoch(); // Can't increment epoch w/o writing to the trace as well. TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); ReleaseImpl(thr, pc, &s->clock); - *a = v; + NoTsanAtomicStore(a, v, mo); s->mtx.Unlock(); - // Trainling memory barrier to provide sequential consistency - // for Dekker-like store-load synchronization. - __sync_synchronize(); } template<typename T, T (*F)(volatile T *v, T op)> @@ -291,7 +333,7 @@ static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); SyncVar *s = 0; if (mo != mo_relaxed) { - s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); + s = ctx->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); thr->fast_state.IncrementEpoch(); // Can't increment epoch w/o writing to the trace as well. TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); @@ -309,6 +351,41 @@ static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { } template<typename T> +static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) { + return func_xchg(a, v); +} + +template<typename T> +static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) { + return func_add(a, v); +} + +template<typename T> +static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) { + return func_sub(a, v); +} + +template<typename T> +static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) { + return func_and(a, v); +} + +template<typename T> +static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) { + return func_or(a, v); +} + +template<typename T> +static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) { + return func_xor(a, v); +} + +template<typename T> +static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) { + return func_nand(a, v); +} + +template<typename T> static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) { return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo); @@ -351,13 +428,34 @@ static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v, } template<typename T> +static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) { + return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo)); +} + +static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v, + morder mo, morder fmo) { + a128 old = *c; + a128 cur = func_cas(a, old, v); + if (cur == old) + return true; + *c = cur; + return false; +} + +template<typename T> +static bool NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) { + return NoTsanAtomicCAS(a, &c, v, mo, fmo); +} + +template<typename T> static bool AtomicCAS(ThreadState *thr, uptr pc, volatile T *a, T *c, T v, morder mo, morder fmo) { (void)fmo; // Unused because llvm does not pass it yet. MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>()); SyncVar *s = 0; + bool write_lock = mo != mo_acquire && mo != mo_consume; if (mo != mo_relaxed) { - s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true); + s = ctx->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock); thr->fast_state.IncrementEpoch(); // Can't increment epoch w/o writing to the trace as well. TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); @@ -370,8 +468,12 @@ static bool AtomicCAS(ThreadState *thr, uptr pc, } T cc = *c; T pr = func_cas(a, cc, v); - if (s) - s->mtx.Unlock(); + if (s) { + if (write_lock) + s->mtx.Unlock(); + else + s->mtx.ReadUnlock(); + } if (pr == cc) return true; *c = pr; @@ -385,293 +487,362 @@ static T AtomicCAS(ThreadState *thr, uptr pc, return c; } +static void NoTsanAtomicFence(morder mo) { + __sync_synchronize(); +} + static void AtomicFence(ThreadState *thr, uptr pc, morder mo) { // FIXME(dvyukov): not implemented. __sync_synchronize(); } +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) { SCOPED_ATOMIC(Load, a, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) { SCOPED_ATOMIC(Load, a, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) { SCOPED_ATOMIC(Load, a, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) { SCOPED_ATOMIC(Load, a, mo); } #if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) { SCOPED_ATOMIC(Load, a, mo); } #endif +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) { SCOPED_ATOMIC(Store, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) { SCOPED_ATOMIC(Store, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) { SCOPED_ATOMIC(Store, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) { SCOPED_ATOMIC(Store, a, v, mo); } #if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) { SCOPED_ATOMIC(Store, a, v, mo); } #endif +SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) { SCOPED_ATOMIC(Exchange, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) { SCOPED_ATOMIC(Exchange, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) { SCOPED_ATOMIC(Exchange, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) { SCOPED_ATOMIC(Exchange, a, v, mo); } #if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) { SCOPED_ATOMIC(Exchange, a, v, mo); } #endif +SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) { SCOPED_ATOMIC(FetchAdd, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) { SCOPED_ATOMIC(FetchAdd, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) { SCOPED_ATOMIC(FetchAdd, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) { SCOPED_ATOMIC(FetchAdd, a, v, mo); } #if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) { SCOPED_ATOMIC(FetchAdd, a, v, mo); } #endif +SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) { SCOPED_ATOMIC(FetchSub, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) { SCOPED_ATOMIC(FetchSub, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) { SCOPED_ATOMIC(FetchSub, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) { SCOPED_ATOMIC(FetchSub, a, v, mo); } #if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) { SCOPED_ATOMIC(FetchSub, a, v, mo); } #endif +SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) { SCOPED_ATOMIC(FetchAnd, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) { SCOPED_ATOMIC(FetchAnd, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) { SCOPED_ATOMIC(FetchAnd, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) { SCOPED_ATOMIC(FetchAnd, a, v, mo); } #if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) { SCOPED_ATOMIC(FetchAnd, a, v, mo); } #endif +SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) { SCOPED_ATOMIC(FetchOr, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) { SCOPED_ATOMIC(FetchOr, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) { SCOPED_ATOMIC(FetchOr, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) { SCOPED_ATOMIC(FetchOr, a, v, mo); } #if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) { SCOPED_ATOMIC(FetchOr, a, v, mo); } #endif +SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) { SCOPED_ATOMIC(FetchXor, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) { SCOPED_ATOMIC(FetchXor, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) { SCOPED_ATOMIC(FetchXor, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) { SCOPED_ATOMIC(FetchXor, a, v, mo); } #if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) { SCOPED_ATOMIC(FetchXor, a, v, mo); } #endif +SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) { SCOPED_ATOMIC(FetchNand, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) { SCOPED_ATOMIC(FetchNand, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) { SCOPED_ATOMIC(FetchNand, a, v, mo); } +SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) { SCOPED_ATOMIC(FetchNand, a, v, mo); } #if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) { SCOPED_ATOMIC(FetchNand, a, v, mo); } #endif +SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } +SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } +SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } +SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } #if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } #endif +SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } +SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } +SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } +SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } #if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } #endif +SANITIZER_INTERFACE_ATTRIBUTE a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } + +SANITIZER_INTERFACE_ATTRIBUTE a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } +SANITIZER_INTERFACE_ATTRIBUTE a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } +SANITIZER_INTERFACE_ATTRIBUTE a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } #if __TSAN_HAS_INT128 +SANITIZER_INTERFACE_ATTRIBUTE a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v, morder mo, morder fmo) { SCOPED_ATOMIC(CAS, a, c, v, mo, fmo); } #endif +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic_thread_fence(morder mo) { char* a = 0; SCOPED_ATOMIC(Fence, mo); } +SANITIZER_INTERFACE_ATTRIBUTE void __tsan_atomic_signal_fence(morder mo) { } +} // extern "C" diff --git a/libsanitizer/tsan/tsan_interface_atomic.h b/libsanitizer/tsan/tsan_interface_atomic.h deleted file mode 100644 index c500614..0000000 --- a/libsanitizer/tsan/tsan_interface_atomic.h +++ /dev/null @@ -1,203 +0,0 @@ -//===-- tsan_interface_atomic.h ---------------------------------*- C++ -*-===// -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of ThreadSanitizer (TSan), a race detector. -// -//===----------------------------------------------------------------------===// -#ifndef TSAN_INTERFACE_ATOMIC_H -#define TSAN_INTERFACE_ATOMIC_H - -#ifndef INTERFACE_ATTRIBUTE -# define INTERFACE_ATTRIBUTE __attribute__((visibility("default"))) -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -typedef char __tsan_atomic8; -typedef short __tsan_atomic16; // NOLINT -typedef int __tsan_atomic32; -typedef long __tsan_atomic64; // NOLINT - -#if defined(__SIZEOF_INT128__) \ - || (__clang_major__ * 100 + __clang_minor__ >= 302) -__extension__ typedef __int128 __tsan_atomic128; -#define __TSAN_HAS_INT128 1 -#else -typedef char __tsan_atomic128; -#define __TSAN_HAS_INT128 0 -#endif - -// Part of ABI, do not change. -// http://llvm.org/viewvc/llvm-project/libcxx/trunk/include/atomic?view=markup -typedef enum { - __tsan_memory_order_relaxed, - __tsan_memory_order_consume, - __tsan_memory_order_acquire, - __tsan_memory_order_release, - __tsan_memory_order_acq_rel, - __tsan_memory_order_seq_cst -} __tsan_memory_order; - -__tsan_atomic8 __tsan_atomic8_load(const volatile __tsan_atomic8 *a, - __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic16 __tsan_atomic16_load(const volatile __tsan_atomic16 *a, - __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic32 __tsan_atomic32_load(const volatile __tsan_atomic32 *a, - __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic64 __tsan_atomic64_load(const volatile __tsan_atomic64 *a, - __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic128 __tsan_atomic128_load(const volatile __tsan_atomic128 *a, - __tsan_memory_order mo) INTERFACE_ATTRIBUTE; - -void __tsan_atomic8_store(volatile __tsan_atomic8 *a, __tsan_atomic8 v, - __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -void __tsan_atomic16_store(volatile __tsan_atomic16 *a, __tsan_atomic16 v, - __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -void __tsan_atomic32_store(volatile __tsan_atomic32 *a, __tsan_atomic32 v, - __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -void __tsan_atomic64_store(volatile __tsan_atomic64 *a, __tsan_atomic64 v, - __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -void __tsan_atomic128_store(volatile __tsan_atomic128 *a, __tsan_atomic128 v, - __tsan_memory_order mo) INTERFACE_ATTRIBUTE; - -__tsan_atomic8 __tsan_atomic8_exchange(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic16 __tsan_atomic16_exchange(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic32 __tsan_atomic32_exchange(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic64 __tsan_atomic64_exchange(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic128 __tsan_atomic128_exchange(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; - -__tsan_atomic8 __tsan_atomic8_fetch_add(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic16 __tsan_atomic16_fetch_add(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic32 __tsan_atomic32_fetch_add(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic64 __tsan_atomic64_fetch_add(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic128 __tsan_atomic128_fetch_add(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; - -__tsan_atomic8 __tsan_atomic8_fetch_sub(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic16 __tsan_atomic16_fetch_sub(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic32 __tsan_atomic32_fetch_sub(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic64 __tsan_atomic64_fetch_sub(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic128 __tsan_atomic128_fetch_sub(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; - -__tsan_atomic8 __tsan_atomic8_fetch_and(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic16 __tsan_atomic16_fetch_and(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic32 __tsan_atomic32_fetch_and(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic64 __tsan_atomic64_fetch_and(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic128 __tsan_atomic128_fetch_and(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; - -__tsan_atomic8 __tsan_atomic8_fetch_or(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic16 __tsan_atomic16_fetch_or(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic32 __tsan_atomic32_fetch_or(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic64 __tsan_atomic64_fetch_or(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic128 __tsan_atomic128_fetch_or(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; - -__tsan_atomic8 __tsan_atomic8_fetch_xor(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic16 __tsan_atomic16_fetch_xor(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic32 __tsan_atomic32_fetch_xor(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic64 __tsan_atomic64_fetch_xor(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic128 __tsan_atomic128_fetch_xor(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; - -__tsan_atomic8 __tsan_atomic8_fetch_nand(volatile __tsan_atomic8 *a, - __tsan_atomic8 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic16 __tsan_atomic16_fetch_nand(volatile __tsan_atomic16 *a, - __tsan_atomic16 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic32 __tsan_atomic32_fetch_nand(volatile __tsan_atomic32 *a, - __tsan_atomic32 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic64 __tsan_atomic64_fetch_nand(volatile __tsan_atomic64 *a, - __tsan_atomic64 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; -__tsan_atomic128 __tsan_atomic128_fetch_nand(volatile __tsan_atomic128 *a, - __tsan_atomic128 v, __tsan_memory_order mo) INTERFACE_ATTRIBUTE; - -int __tsan_atomic8_compare_exchange_weak(volatile __tsan_atomic8 *a, - __tsan_atomic8 *c, __tsan_atomic8 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; -int __tsan_atomic16_compare_exchange_weak(volatile __tsan_atomic16 *a, - __tsan_atomic16 *c, __tsan_atomic16 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; -int __tsan_atomic32_compare_exchange_weak(volatile __tsan_atomic32 *a, - __tsan_atomic32 *c, __tsan_atomic32 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; -int __tsan_atomic64_compare_exchange_weak(volatile __tsan_atomic64 *a, - __tsan_atomic64 *c, __tsan_atomic64 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; -int __tsan_atomic128_compare_exchange_weak(volatile __tsan_atomic128 *a, - __tsan_atomic128 *c, __tsan_atomic128 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; - -int __tsan_atomic8_compare_exchange_strong(volatile __tsan_atomic8 *a, - __tsan_atomic8 *c, __tsan_atomic8 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; -int __tsan_atomic16_compare_exchange_strong(volatile __tsan_atomic16 *a, - __tsan_atomic16 *c, __tsan_atomic16 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; -int __tsan_atomic32_compare_exchange_strong(volatile __tsan_atomic32 *a, - __tsan_atomic32 *c, __tsan_atomic32 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; -int __tsan_atomic64_compare_exchange_strong(volatile __tsan_atomic64 *a, - __tsan_atomic64 *c, __tsan_atomic64 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; -int __tsan_atomic128_compare_exchange_strong(volatile __tsan_atomic128 *a, - __tsan_atomic128 *c, __tsan_atomic128 v, __tsan_memory_order mo, - __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; - -__tsan_atomic8 __tsan_atomic8_compare_exchange_val( - volatile __tsan_atomic8 *a, __tsan_atomic8 c, __tsan_atomic8 v, - __tsan_memory_order mo, __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; -__tsan_atomic16 __tsan_atomic16_compare_exchange_val( - volatile __tsan_atomic16 *a, __tsan_atomic16 c, __tsan_atomic16 v, - __tsan_memory_order mo, __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; -__tsan_atomic32 __tsan_atomic32_compare_exchange_val( - volatile __tsan_atomic32 *a, __tsan_atomic32 c, __tsan_atomic32 v, - __tsan_memory_order mo, __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; -__tsan_atomic64 __tsan_atomic64_compare_exchange_val( - volatile __tsan_atomic64 *a, __tsan_atomic64 c, __tsan_atomic64 v, - __tsan_memory_order mo, __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; -__tsan_atomic128 __tsan_atomic128_compare_exchange_val( - volatile __tsan_atomic128 *a, __tsan_atomic128 c, __tsan_atomic128 v, - __tsan_memory_order mo, __tsan_memory_order fail_mo) INTERFACE_ATTRIBUTE; - -void __tsan_atomic_thread_fence(__tsan_memory_order mo) INTERFACE_ATTRIBUTE; -void __tsan_atomic_signal_fence(__tsan_memory_order mo) INTERFACE_ATTRIBUTE; - -#ifdef __cplusplus -} // extern "C" -#endif - -#undef INTERFACE_ATTRIBUTE - -#endif // #ifndef TSAN_INTERFACE_ATOMIC_H diff --git a/libsanitizer/tsan/tsan_interface_java.cc b/libsanitizer/tsan/tsan_interface_java.cc index 70b5e5f..7f45169 100644 --- a/libsanitizer/tsan/tsan_interface_java.cc +++ b/libsanitizer/tsan/tsan_interface_java.cc @@ -77,13 +77,9 @@ class ScopedJavaFunc { : thr_(thr) { Initialize(thr_); FuncEntry(thr, pc); - CHECK_EQ(thr_->in_rtl, 0); - thr_->in_rtl++; } ~ScopedJavaFunc() { - thr_->in_rtl--; - CHECK_EQ(thr_->in_rtl, 0); FuncExit(thr_); // FIXME(dvyukov): process pending signals. } @@ -134,7 +130,7 @@ SyncVar* GetJavaSync(ThreadState *thr, uptr pc, uptr addr, } if (s == 0 && create) { DPrintf("#%d: creating new sync for %p\n", thr->tid, addr); - s = CTX()->synctab.Create(thr, pc, addr); + s = ctx->synctab.Create(thr, pc, addr); s->next = b->head; b->head = s; } diff --git a/libsanitizer/tsan/tsan_mman.cc b/libsanitizer/tsan/tsan_mman.cc index 832374b..bd30cd5 100644 --- a/libsanitizer/tsan/tsan_mman.cc +++ b/libsanitizer/tsan/tsan_mman.cc @@ -88,7 +88,6 @@ void AllocatorPrintStats() { static void SignalUnsafeCall(ThreadState *thr, uptr pc) { if (!thr->in_signal_handler || !flags()->report_signal_unsafe) return; - Context *ctx = CTX(); StackTrace stack; stack.ObtainCurrent(thr, pc); ThreadRegistryLock l(ctx->thread_registry); @@ -100,7 +99,6 @@ static void SignalUnsafeCall(ThreadState *thr, uptr pc) { } void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align) { - CHECK_GT(thr->in_rtl, 0); if ((sz >= (1ull << 40)) || (align >= (1ull << 40))) return AllocatorReturnNull(); void *p = allocator()->Allocate(&thr->alloc_cache, sz, align); @@ -108,7 +106,7 @@ void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align) { return 0; MBlock *b = new(allocator()->GetMetaData(p)) MBlock; b->Init(sz, thr->tid, CurrentStackId(thr, pc)); - if (CTX() && CTX()->initialized) { + if (ctx && ctx->initialized) { if (thr->ignore_reads_and_writes == 0) MemoryRangeImitateWrite(thr, pc, (uptr)p, sz); else @@ -120,7 +118,6 @@ void *user_alloc(ThreadState *thr, uptr pc, uptr sz, uptr align) { } void user_free(ThreadState *thr, uptr pc, void *p) { - CHECK_GT(thr->in_rtl, 0); CHECK_NE(p, (void*)0); DPrintf("#%d: free(%p)\n", thr->tid, p); MBlock *b = (MBlock*)allocator()->GetMetaData(p); @@ -136,7 +133,7 @@ void user_free(ThreadState *thr, uptr pc, void *p) { } b->ListReset(); } - if (CTX() && CTX()->initialized && thr->in_rtl == 1) { + if (ctx && ctx->initialized) { if (thr->ignore_reads_and_writes == 0) MemoryRangeFreed(thr, pc, (uptr)p, b->Size()); } @@ -145,7 +142,6 @@ void user_free(ThreadState *thr, uptr pc, void *p) { } void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) { - CHECK_GT(thr->in_rtl, 0); void *p2 = 0; // FIXME: Handle "shrinking" more efficiently, // it seems that some software actually does this. @@ -165,7 +161,6 @@ void *user_realloc(ThreadState *thr, uptr pc, void *p, uptr sz) { } uptr user_alloc_usable_size(ThreadState *thr, uptr pc, void *p) { - CHECK_GT(thr->in_rtl, 0); if (p == 0) return 0; MBlock *b = (MBlock*)allocator()->GetMetaData(p); @@ -182,24 +177,21 @@ MBlock *user_mblock(ThreadState *thr, void *p) { } void invoke_malloc_hook(void *ptr, uptr size) { - Context *ctx = CTX(); ThreadState *thr = cur_thread(); - if (ctx == 0 || !ctx->initialized || thr->in_rtl) + if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors) return; __tsan_malloc_hook(ptr, size); } void invoke_free_hook(void *ptr) { - Context *ctx = CTX(); ThreadState *thr = cur_thread(); - if (ctx == 0 || !ctx->initialized || thr->in_rtl) + if (ctx == 0 || !ctx->initialized || thr->ignore_interceptors) return; __tsan_free_hook(ptr); } void *internal_alloc(MBlockType typ, uptr sz) { ThreadState *thr = cur_thread(); - CHECK_GT(thr->in_rtl, 0); CHECK_LE(sz, InternalSizeClassMap::kMaxSize); if (thr->nomalloc) { thr->nomalloc = 0; // CHECK calls internal_malloc(). @@ -210,7 +202,6 @@ void *internal_alloc(MBlockType typ, uptr sz) { void internal_free(void *p) { ThreadState *thr = cur_thread(); - CHECK_GT(thr->in_rtl, 0); if (thr->nomalloc) { thr->nomalloc = 0; // CHECK calls internal_malloc(). CHECK(0); diff --git a/libsanitizer/tsan/tsan_mutex.cc b/libsanitizer/tsan/tsan_mutex.cc index e7846c5..0c3bb4a 100644 --- a/libsanitizer/tsan/tsan_mutex.cc +++ b/libsanitizer/tsan/tsan_mutex.cc @@ -31,13 +31,14 @@ static MutexType CanLockTab[MutexTypeCount][MutexTypeCount] = { /*2 MutexTypeThreads*/ {MutexTypeReport}, /*3 MutexTypeReport*/ {MutexTypeSyncTab, MutexTypeSyncVar, MutexTypeMBlock, MutexTypeJavaMBlock}, - /*4 MutexTypeSyncVar*/ {}, + /*4 MutexTypeSyncVar*/ {MutexTypeDDetector}, /*5 MutexTypeSyncTab*/ {MutexTypeSyncVar}, /*6 MutexTypeSlab*/ {MutexTypeLeaf}, /*7 MutexTypeAnnotations*/ {}, /*8 MutexTypeAtExit*/ {MutexTypeSyncTab}, /*9 MutexTypeMBlock*/ {MutexTypeSyncVar}, /*10 MutexTypeJavaMBlock*/ {MutexTypeSyncVar}, + /*11 MutexTypeDDetector*/ {}, }; static bool CanLockAdj[MutexTypeCount][MutexTypeCount]; @@ -121,12 +122,12 @@ void InitializeMutex() { #endif } -DeadlockDetector::DeadlockDetector() { +InternalDeadlockDetector::InternalDeadlockDetector() { // Rely on zero initialization because some mutexes can be locked before ctor. } #if TSAN_DEBUG && !TSAN_GO -void DeadlockDetector::Lock(MutexType t) { +void InternalDeadlockDetector::Lock(MutexType t) { // Printf("LOCK %d @%zu\n", t, seq_ + 1); CHECK_GT(t, MutexTypeInvalid); CHECK_LT(t, MutexTypeCount); @@ -153,7 +154,7 @@ void DeadlockDetector::Lock(MutexType t) { } } -void DeadlockDetector::Unlock(MutexType t) { +void InternalDeadlockDetector::Unlock(MutexType t) { // Printf("UNLO %d @%zu #%zu\n", t, seq_, locked_[t]); CHECK(locked_[t]); locked_[t] = 0; @@ -208,7 +209,7 @@ Mutex::~Mutex() { void Mutex::Lock() { #if TSAN_DEBUG && !TSAN_GO - cur_thread()->deadlock_detector.Lock(type_); + cur_thread()->internal_deadlock_detector.Lock(type_); #endif uptr cmp = kUnlocked; if (atomic_compare_exchange_strong(&state_, &cmp, kWriteLock, @@ -233,13 +234,13 @@ void Mutex::Unlock() { (void)prev; DCHECK_NE(prev & kWriteLock, 0); #if TSAN_DEBUG && !TSAN_GO - cur_thread()->deadlock_detector.Unlock(type_); + cur_thread()->internal_deadlock_detector.Unlock(type_); #endif } void Mutex::ReadLock() { #if TSAN_DEBUG && !TSAN_GO - cur_thread()->deadlock_detector.Lock(type_); + cur_thread()->internal_deadlock_detector.Lock(type_); #endif uptr prev = atomic_fetch_add(&state_, kReadLock, memory_order_acquire); if ((prev & kWriteLock) == 0) @@ -261,7 +262,7 @@ void Mutex::ReadUnlock() { DCHECK_EQ(prev & kWriteLock, 0); DCHECK_GT(prev & ~kWriteLock, 0); #if TSAN_DEBUG && !TSAN_GO - cur_thread()->deadlock_detector.Unlock(type_); + cur_thread()->internal_deadlock_detector.Unlock(type_); #endif } diff --git a/libsanitizer/tsan/tsan_mutex.h b/libsanitizer/tsan/tsan_mutex.h index 6d14505..f075ce8 100644 --- a/libsanitizer/tsan/tsan_mutex.h +++ b/libsanitizer/tsan/tsan_mutex.h @@ -29,6 +29,7 @@ enum MutexType { MutexTypeAtExit, MutexTypeMBlock, MutexTypeJavaMBlock, + MutexTypeDDetector, // This must be the last. MutexTypeCount @@ -63,9 +64,9 @@ class Mutex { typedef GenericScopedLock<Mutex> Lock; typedef GenericScopedReadLock<Mutex> ReadLock; -class DeadlockDetector { +class InternalDeadlockDetector { public: - DeadlockDetector(); + InternalDeadlockDetector(); void Lock(MutexType t); void Unlock(MutexType t); private: diff --git a/libsanitizer/tsan/tsan_mutexset.h b/libsanitizer/tsan/tsan_mutexset.h index df36b46..7fdc194 100644 --- a/libsanitizer/tsan/tsan_mutexset.h +++ b/libsanitizer/tsan/tsan_mutexset.h @@ -36,6 +36,10 @@ class MutexSet { uptr Size() const; Desc Get(uptr i) const; + void operator=(const MutexSet &other) { + internal_memcpy(this, &other, sizeof(*this)); + } + private: #ifndef TSAN_GO uptr size_; @@ -43,6 +47,7 @@ class MutexSet { #endif void RemovePos(uptr i); + MutexSet(const MutexSet&); }; // Go does not have mutexes, so do not spend memory and time. diff --git a/libsanitizer/tsan/tsan_platform.h b/libsanitizer/tsan/tsan_platform.h index 164ee45..60eb1a8 100644 --- a/libsanitizer/tsan/tsan_platform.h +++ b/libsanitizer/tsan/tsan_platform.h @@ -75,6 +75,8 @@ static const uptr kLinuxShadowMsk = 0x200000000000ULL; #elif defined(TSAN_COMPAT_SHADOW) && TSAN_COMPAT_SHADOW static const uptr kLinuxAppMemBeg = 0x290000000000ULL; static const uptr kLinuxAppMemEnd = 0x7fffffffffffULL; +static const uptr kAppMemGapBeg = 0x2c0000000000ULL; +static const uptr kAppMemGapEnd = 0x7d0000000000ULL; #else static const uptr kLinuxAppMemBeg = 0x7cf000000000ULL; static const uptr kLinuxAppMemEnd = 0x7fffffffffffULL; @@ -103,7 +105,12 @@ static const uptr kLinuxShadowEnd = MemToShadow(kLinuxAppMemEnd) | 0xff; static inline bool IsAppMem(uptr mem) { +#if defined(TSAN_COMPAT_SHADOW) && TSAN_COMPAT_SHADOW + return (mem >= kLinuxAppMemBeg && mem < kAppMemGapBeg) || + (mem >= kAppMemGapEnd && mem <= kLinuxAppMemEnd); +#else return mem >= kLinuxAppMemBeg && mem <= kLinuxAppMemEnd; +#endif } static inline bool IsShadowMem(uptr mem) { @@ -138,8 +145,9 @@ const char *InitializePlatform(); void FinalizePlatform(); // The additional page is to catch shadow stack overflow as paging fault. -const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace) + 4096 - + 4095) & ~4095; +// Windows wants 64K alignment for mmaps. +const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace) + + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1); uptr ALWAYS_INLINE GetThreadTrace(int tid) { uptr p = kTraceMemBegin + (uptr)tid * kTotalTraceSize; @@ -154,7 +162,8 @@ uptr ALWAYS_INLINE GetThreadTraceHeader(int tid) { return p; } -void internal_start_thread(void(*func)(void*), void *arg); +void *internal_start_thread(void(*func)(void*), void *arg); +void internal_join_thread(void *th); // Says whether the addr relates to a global var. // Guesses with high probability, may yield both false positives and negatives. @@ -162,6 +171,10 @@ bool IsGlobalVar(uptr addr); int ExtractResolvFDs(void *state, int *fds, int nfd); int ExtractRecvmsgFDs(void *msg, int *fds, int nfd); +int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, + void *abstime), void *c, void *m, void *abstime, + void(*cleanup)(void *arg), void *arg); + } // namespace __tsan #else // defined(__LP64__) || defined(_WIN64) diff --git a/libsanitizer/tsan/tsan_platform_linux.cc b/libsanitizer/tsan/tsan_platform_linux.cc index fe69430..062e846 100644 --- a/libsanitizer/tsan/tsan_platform_linux.cc +++ b/libsanitizer/tsan/tsan_platform_linux.cc @@ -59,27 +59,6 @@ namespace __tsan { const uptr kPageSize = 4096; -#ifndef TSAN_GO -ScopedInRtl::ScopedInRtl() - : thr_(cur_thread()) { - in_rtl_ = thr_->in_rtl; - thr_->in_rtl++; - errno_ = errno; -} - -ScopedInRtl::~ScopedInRtl() { - thr_->in_rtl--; - errno = errno_; - CHECK_EQ(in_rtl_, thr_->in_rtl); -} -#else -ScopedInRtl::ScopedInRtl() { -} - -ScopedInRtl::~ScopedInRtl() { -} -#endif - void FillProfileCallback(uptr start, uptr rss, bool file, uptr *mem, uptr stats_size) { CHECK_EQ(7, stats_size); @@ -133,7 +112,6 @@ void FlushShadowMemory() { #ifndef TSAN_GO static void ProtectRange(uptr beg, uptr end) { - ScopedInRtl in_rtl; CHECK_LE(beg, end); if (beg == end) return; @@ -159,17 +137,19 @@ static void MapRodata() { #endif if (tmpdir == 0) return; - char filename[256]; - internal_snprintf(filename, sizeof(filename), "%s/tsan.rodata.%d", + char name[256]; + internal_snprintf(name, sizeof(name), "%s/tsan.rodata.%d", tmpdir, (int)internal_getpid()); - uptr openrv = internal_open(filename, O_RDWR | O_CREAT | O_EXCL, 0600); + uptr openrv = internal_open(name, O_RDWR | O_CREAT | O_EXCL, 0600); if (internal_iserror(openrv)) return; + internal_unlink(name); // Unlink it now, so that we can reuse the buffer. fd_t fd = openrv; // Fill the file with kShadowRodata. const uptr kMarkerSize = 512 * 1024 / sizeof(u64); InternalScopedBuffer<u64> marker(kMarkerSize); - for (u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++) + // volatile to prevent insertion of memset + for (volatile u64 *p = marker.data(); p < marker.data() + kMarkerSize; p++) *p = kShadowRodata; internal_write(fd, marker.data(), marker.size()); // Map the file into memory. @@ -177,13 +157,12 @@ static void MapRodata() { MAP_PRIVATE | MAP_ANONYMOUS, fd, 0); if (internal_iserror(page)) { internal_close(fd); - internal_unlink(filename); return; } // Map the file into shadow of .rodata sections. MemoryMappingLayout proc_maps(/*cache_enabled*/true); uptr start, end, offset, prot; - char name[128]; + // Reusing the buffer 'name'. while (proc_maps.Next(&start, &end, &offset, name, ARRAY_SIZE(name), &prot)) { if (name[0] != 0 && name[0] != '[' && (prot & MemoryMappingLayout::kProtectionRead) @@ -200,7 +179,6 @@ static void MapRodata() { } } internal_close(fd); - internal_unlink(filename); } void InitializeShadowMemory() { @@ -314,10 +292,10 @@ const char *InitializePlatform() { // we re-exec the program with limited stack size as a best effort. if (getlim(RLIMIT_STACK) == (rlim_t)-1) { const uptr kMaxStackSize = 32 * 1024 * 1024; - Report("WARNING: Program is run with unlimited stack size, which " - "wouldn't work with ThreadSanitizer.\n"); - Report("Re-execing with stack size limited to %zd bytes.\n", - kMaxStackSize); + VReport(1, "Program is run with unlimited stack size, which wouldn't " + "work with ThreadSanitizer.\n" + "Re-execing with stack size limited to %zd bytes.\n", + kMaxStackSize); SetStackSizeLimitInBytes(kMaxStackSize); reexec = true; } @@ -378,8 +356,19 @@ int ExtractRecvmsgFDs(void *msgp, int *fds, int nfd) { } return res; } -#endif +int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, + void *abstime), void *c, void *m, void *abstime, + void(*cleanup)(void *arg), void *arg) { + // pthread_cleanup_push/pop are hardcore macros mess. + // We can't intercept nor call them w/o including pthread.h. + int res; + pthread_cleanup_push(cleanup, arg); + res = fn(c, m, abstime); + pthread_cleanup_pop(0); + return res; +} +#endif } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_platform_mac.cc b/libsanitizer/tsan/tsan_platform_mac.cc index 3dca611..c3d4d90 100644 --- a/libsanitizer/tsan/tsan_platform_mac.cc +++ b/libsanitizer/tsan/tsan_platform_mac.cc @@ -38,17 +38,18 @@ namespace __tsan { -ScopedInRtl::ScopedInRtl() { +uptr GetShadowMemoryConsumption() { + return 0; } -ScopedInRtl::~ScopedInRtl() { +void FlushShadowMemory() { } -uptr GetShadowMemoryConsumption() { - return 0; +void WriteMemoryProfile(char *buf, uptr buf_size) { } -void FlushShadowMemory() { +uptr GetRSS() { + return 0; } #ifndef TSAN_GO @@ -88,6 +89,20 @@ void FinalizePlatform() { fflush(0); } +#ifndef TSAN_GO +int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m, + void *abstime), void *c, void *m, void *abstime, + void(*cleanup)(void *arg), void *arg) { + // pthread_cleanup_push/pop are hardcore macros mess. + // We can't intercept nor call them w/o including pthread.h. + int res; + pthread_cleanup_push(cleanup, arg); + res = fn(c, m, abstime); + pthread_cleanup_pop(0); + return res; +} +#endif + } // namespace __tsan #endif // SANITIZER_MAC diff --git a/libsanitizer/tsan/tsan_platform_windows.cc b/libsanitizer/tsan/tsan_platform_windows.cc index 6e49ef4..f16bebf 100644 --- a/libsanitizer/tsan/tsan_platform_windows.cc +++ b/libsanitizer/tsan/tsan_platform_windows.cc @@ -19,17 +19,18 @@ namespace __tsan { -ScopedInRtl::ScopedInRtl() { +uptr GetShadowMemoryConsumption() { + return 0; } -ScopedInRtl::~ScopedInRtl() { +void FlushShadowMemory() { } -uptr GetShadowMemoryConsumption() { - return 0; +void WriteMemoryProfile(char *buf, uptr buf_size) { } -void FlushShadowMemory() { +uptr GetRSS() { + return 0; } const char *InitializePlatform() { diff --git a/libsanitizer/tsan/tsan_report.cc b/libsanitizer/tsan/tsan_report.cc index f248416..00a512e 100644 --- a/libsanitizer/tsan/tsan_report.cc +++ b/libsanitizer/tsan/tsan_report.cc @@ -38,6 +38,7 @@ ReportDesc::ReportDesc() , locs(MBlockReportLoc) , mutexes(MBlockReportMutex) , threads(MBlockReportThread) + , unique_tids(MBlockReportThread) , sleep() , count() { } @@ -71,10 +72,20 @@ static const char *ReportTypeString(ReportType typ) { return "thread leak"; if (typ == ReportTypeMutexDestroyLocked) return "destroy of a locked mutex"; + if (typ == ReportTypeMutexDoubleLock) + return "double lock of a mutex"; + if (typ == ReportTypeMutexBadUnlock) + return "unlock of an unlocked mutex (or by a wrong thread)"; + if (typ == ReportTypeMutexBadReadLock) + return "read lock of a write locked mutex"; + if (typ == ReportTypeMutexBadReadUnlock) + return "read unlock of a write locked mutex"; if (typ == ReportTypeSignalUnsafe) return "signal-unsafe call inside of a signal"; if (typ == ReportTypeErrnoInSignal) return "signal handler spoils errno"; + if (typ == ReportTypeDeadlock) + return "lock-order-inversion (potential deadlock)"; return ""; } @@ -153,6 +164,17 @@ static void PrintLocation(const ReportLocation *loc) { PrintStack(loc->stack); } +static void PrintMutexShort(const ReportMutex *rm, const char *after) { + Decorator d; + Printf("%sM%zd%s%s", d.Mutex(), rm->id, d.EndMutex(), after); +} + +static void PrintMutexShortWithAddress(const ReportMutex *rm, + const char *after) { + Decorator d; + Printf("%sM%zd (%p)%s%s", d.Mutex(), rm->id, rm->addr, d.EndMutex(), after); +} + static void PrintMutex(const ReportMutex *rm) { Decorator d; if (rm->destroyed) { @@ -161,7 +183,7 @@ static void PrintMutex(const ReportMutex *rm) { Printf("%s", d.EndMutex()); } else { Printf("%s", d.Mutex()); - Printf(" Mutex M%llu created at:\n", rm->id); + Printf(" Mutex M%llu (%p) created at:\n", rm->id, rm->addr); Printf("%s", d.EndMutex()); PrintStack(rm->stack); } @@ -221,10 +243,42 @@ void PrintReport(const ReportDesc *rep) { (int)internal_getpid()); Printf("%s", d.EndWarning()); - for (uptr i = 0; i < rep->stacks.Size(); i++) { - if (i) - Printf(" and:\n"); - PrintStack(rep->stacks[i]); + if (rep->typ == ReportTypeDeadlock) { + char thrbuf[kThreadBufSize]; + Printf(" Cycle in lock order graph: "); + for (uptr i = 0; i < rep->mutexes.Size(); i++) + PrintMutexShortWithAddress(rep->mutexes[i], " => "); + PrintMutexShort(rep->mutexes[0], "\n\n"); + CHECK_GT(rep->mutexes.Size(), 0U); + CHECK_EQ(rep->mutexes.Size() * (flags()->second_deadlock_stack ? 2 : 1), + rep->stacks.Size()); + for (uptr i = 0; i < rep->mutexes.Size(); i++) { + Printf(" Mutex "); + PrintMutexShort(rep->mutexes[(i + 1) % rep->mutexes.Size()], + " acquired here while holding mutex "); + PrintMutexShort(rep->mutexes[i], " in "); + Printf("%s", d.ThreadDescription()); + Printf("%s:\n", thread_name(thrbuf, rep->unique_tids[i])); + Printf("%s", d.EndThreadDescription()); + if (flags()->second_deadlock_stack) { + PrintStack(rep->stacks[2*i]); + Printf(" Mutex "); + PrintMutexShort(rep->mutexes[i], + " previously acquired by the same thread here:\n"); + PrintStack(rep->stacks[2*i+1]); + } else { + PrintStack(rep->stacks[i]); + if (i == 0) + Printf(" Hint: use TSAN_OPTIONS=second_deadlock_stack=1 " + "to get more informative warning message\n\n"); + } + } + } else { + for (uptr i = 0; i < rep->stacks.Size(); i++) { + if (i) + Printf(" and:\n"); + PrintStack(rep->stacks[i]); + } } for (uptr i = 0; i < rep->mops.Size(); i++) @@ -236,8 +290,10 @@ void PrintReport(const ReportDesc *rep) { for (uptr i = 0; i < rep->locs.Size(); i++) PrintLocation(rep->locs[i]); - for (uptr i = 0; i < rep->mutexes.Size(); i++) - PrintMutex(rep->mutexes[i]); + if (rep->typ != ReportTypeDeadlock) { + for (uptr i = 0; i < rep->mutexes.Size(); i++) + PrintMutex(rep->mutexes[i]); + } for (uptr i = 0; i < rep->threads.Size(); i++) PrintThread(rep->threads[i]); @@ -289,11 +345,26 @@ static void PrintThread(const ReportThread *rt) { void PrintReport(const ReportDesc *rep) { Printf("==================\n"); - Printf("WARNING: DATA RACE"); - for (uptr i = 0; i < rep->mops.Size(); i++) - PrintMop(rep->mops[i], i == 0); - for (uptr i = 0; i < rep->threads.Size(); i++) - PrintThread(rep->threads[i]); + if (rep->typ == ReportTypeRace) { + Printf("WARNING: DATA RACE"); + for (uptr i = 0; i < rep->mops.Size(); i++) + PrintMop(rep->mops[i], i == 0); + for (uptr i = 0; i < rep->threads.Size(); i++) + PrintThread(rep->threads[i]); + } else if (rep->typ == ReportTypeDeadlock) { + Printf("WARNING: DEADLOCK\n"); + for (uptr i = 0; i < rep->mutexes.Size(); i++) { + Printf("Goroutine %d lock mutex %d while holding mutex %d:\n", + 999, rep->mutexes[i]->id, + rep->mutexes[(i+1) % rep->mutexes.Size()]->id); + PrintStack(rep->stacks[2*i]); + Printf("\n"); + Printf("Mutex %d was previously locked here:\n", + rep->mutexes[(i+1) % rep->mutexes.Size()]->id); + PrintStack(rep->stacks[2*i + 1]); + Printf("\n"); + } + } Printf("==================\n"); } diff --git a/libsanitizer/tsan/tsan_report.h b/libsanitizer/tsan/tsan_report.h index c0eef9e..d773e05 100644 --- a/libsanitizer/tsan/tsan_report.h +++ b/libsanitizer/tsan/tsan_report.h @@ -22,8 +22,13 @@ enum ReportType { ReportTypeUseAfterFree, ReportTypeThreadLeak, ReportTypeMutexDestroyLocked, + ReportTypeMutexDoubleLock, + ReportTypeMutexBadUnlock, + ReportTypeMutexBadReadLock, + ReportTypeMutexBadReadUnlock, ReportTypeSignalUnsafe, - ReportTypeErrnoInSignal + ReportTypeErrnoInSignal, + ReportTypeDeadlock }; struct ReportStack { @@ -87,6 +92,7 @@ struct ReportThread { struct ReportMutex { u64 id; + uptr addr; bool destroyed; ReportStack *stack; }; @@ -99,6 +105,7 @@ class ReportDesc { Vector<ReportLocation*> locs; Vector<ReportMutex*> mutexes; Vector<ReportThread*> threads; + Vector<int> unique_tids; ReportStack *sleep; int count; diff --git a/libsanitizer/tsan/tsan_rtl.cc b/libsanitizer/tsan/tsan_rtl.cc index 573eeb8..7932a6d 100644 --- a/libsanitizer/tsan/tsan_rtl.cc +++ b/libsanitizer/tsan/tsan_rtl.cc @@ -35,22 +35,21 @@ namespace __tsan { THREADLOCAL char cur_thread_placeholder[sizeof(ThreadState)] ALIGNED(64); #endif static char ctx_placeholder[sizeof(Context)] ALIGNED(64); +Context *ctx; // Can be overriden by a front-end. #ifdef TSAN_EXTERNAL_HOOKS bool OnFinalize(bool failed); +void OnInitialize(); #else SANITIZER_INTERFACE_ATTRIBUTE bool WEAK OnFinalize(bool failed) { return failed; } +SANITIZER_INTERFACE_ATTRIBUTE +void WEAK OnInitialize() {} #endif -static Context *ctx; -Context *CTX() { - return ctx; -} - static char thread_registry_placeholder[sizeof(ThreadRegistry)]; static ThreadContextBase *CreateThreadContext(u32 tid) { @@ -74,7 +73,7 @@ Context::Context() , nreported() , nmissed_expected() , thread_registry(new(thread_registry_placeholder) ThreadRegistry( - CreateThreadContext, kMaxTid, kThreadQuarantineSize)) + CreateThreadContext, kMaxTid, kThreadQuarantineSize, kMaxTidReuse)) , racy_stacks(MBlockRacyStacks) , racy_addresses(MBlockRacyAddresses) , fired_suppressions(8) { @@ -82,13 +81,15 @@ Context::Context() // The objects are allocated in TLS, so one may rely on zero-initialization. ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, + unsigned reuse_count, uptr stk_addr, uptr stk_size, uptr tls_addr, uptr tls_size) : fast_state(tid, epoch) // Do not touch these, rely on zero initialization, // they may be accessed before the ctor. // , ignore_reads_and_writes() - // , in_rtl() + // , ignore_interceptors() + , clock(tid, reuse_count) #ifndef TSAN_GO , jmp_bufs(MBlockJmpBuf) #endif @@ -97,7 +98,11 @@ ThreadState::ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, , stk_addr(stk_addr) , stk_size(stk_size) , tls_addr(tls_addr) - , tls_size(tls_size) { + , tls_size(tls_size) +#ifndef TSAN_GO + , last_sleep_clock(tid) +#endif +{ } static void MemoryProfiler(Context *ctx, fd_t fd, int i) { @@ -113,8 +118,13 @@ static void MemoryProfiler(Context *ctx, fd_t fd, int i) { } static void BackgroundThread(void *arg) { - ScopedInRtl in_rtl; - Context *ctx = CTX(); +#ifndef TSAN_GO + // This is a non-initialized non-user thread, nothing to see here. + // We don't use ScopedIgnoreInterceptors, because we want ignores to be + // enabled even when the thread function exits (e.g. during pthread thread + // shutdown code). + cur_thread()->ignore_interceptors++; +#endif const u64 kMs2Ns = 1000 * 1000; fd_t mprof_fd = kInvalidFd; @@ -133,8 +143,10 @@ static void BackgroundThread(void *arg) { u64 last_flush = NanoTime(); uptr last_rss = 0; - for (int i = 0; ; i++) { - SleepForSeconds(1); + for (int i = 0; + atomic_load(&ctx->stop_background_thread, memory_order_relaxed) == 0; + i++) { + SleepForMillis(100); u64 now = NanoTime(); // Flush memory if requested. @@ -185,6 +197,16 @@ static void BackgroundThread(void *arg) { } } +static void StartBackgroundThread() { + ctx->background_thread = internal_start_thread(&BackgroundThread, 0); +} + +static void StopBackgroundThread() { + atomic_store(&ctx->stop_background_thread, 1, memory_order_relaxed); + internal_join_thread(ctx->background_thread); + ctx->background_thread = 0; +} + void DontNeedShadowFor(uptr addr, uptr size) { uptr shadow_beg = MemToShadow(addr); uptr shadow_end = MemToShadow(addr + size); @@ -192,6 +214,9 @@ void DontNeedShadowFor(uptr addr, uptr size) { } void MapShadow(uptr addr, uptr size) { + // Global data is not 64K aligned, but there are no adjacent mappings, + // so we can get away with unaligned mapping. + // CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment MmapFixedNoReserve(MemToShadow(addr), size * kShadowMultiplier); } @@ -199,6 +224,7 @@ void MapThreadTrace(uptr addr, uptr size) { DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr, addr + size, size); CHECK_GE(addr, kTraceMemBegin); CHECK_LE(addr + size, kTraceMemBegin + kTraceMemSize); + CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment uptr addr1 = (uptr)MmapFixedNoReserve(addr, size); if (addr1 != addr) { Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p->%p)\n", @@ -213,11 +239,12 @@ void Initialize(ThreadState *thr) { if (is_initialized) return; is_initialized = true; + // We are not ready to handle interceptors yet. + ScopedIgnoreInterceptors ignore; SanitizerToolName = "ThreadSanitizer"; // Install tool-specific callbacks in sanitizer_common. SetCheckFailedCallback(TsanCheckFailed); - ScopedInRtl in_rtl; #ifndef TSAN_GO InitializeAllocator(); #endif @@ -235,19 +262,13 @@ void Initialize(ThreadState *thr) { InitializeSuppressions(); #ifndef TSAN_GO InitializeLibIgnore(); - // Initialize external symbolizer before internal threads are started. - const char *external_symbolizer = flags()->external_symbolizer_path; - bool external_symbolizer_started = - Symbolizer::Init(external_symbolizer)->IsExternalAvailable(); - if (external_symbolizer != 0 && external_symbolizer[0] != '\0' && - !external_symbolizer_started) { - Printf("Failed to start external symbolizer: '%s'\n", - external_symbolizer); - Die(); - } + Symbolizer::Init(common_flags()->external_symbolizer_path); Symbolizer::Get()->AddHooks(EnterSymbolizer, ExitSymbolizer); #endif - internal_start_thread(&BackgroundThread, 0); + StartBackgroundThread(); + SetSandboxingCallback(StopBackgroundThread); + if (flags()->detect_deadlocks) + ctx->dd = DDetector::Create(flags()); if (ctx->flags.verbosity) Printf("***** Running under ThreadSanitizer v2 (pid %d) *****\n", @@ -257,7 +278,6 @@ void Initialize(ThreadState *thr) { int tid = ThreadCreate(thr, 0, 0, true); CHECK_EQ(tid, 0); ThreadStart(thr, tid, internal_getpid()); - CHECK_EQ(thr->in_rtl, 1); ctx->initialized = true; if (flags()->stop_on_start) { @@ -266,10 +286,11 @@ void Initialize(ThreadState *thr) { (int)internal_getpid()); while (__tsan_resumed == 0) {} } + + OnInitialize(); } int Finalize(ThreadState *thr) { - ScopedInRtl in_rtl; Context *ctx = __tsan::ctx; bool failed = false; @@ -319,6 +340,38 @@ int Finalize(ThreadState *thr) { } #ifndef TSAN_GO +void ForkBefore(ThreadState *thr, uptr pc) { + ctx->thread_registry->Lock(); + ctx->report_mtx.Lock(); +} + +void ForkParentAfter(ThreadState *thr, uptr pc) { + ctx->report_mtx.Unlock(); + ctx->thread_registry->Unlock(); +} + +void ForkChildAfter(ThreadState *thr, uptr pc) { + ctx->report_mtx.Unlock(); + ctx->thread_registry->Unlock(); + + uptr nthread = 0; + ctx->thread_registry->GetNumberOfThreads(0, 0, &nthread /* alive threads */); + VPrintf(1, "ThreadSanitizer: forked new process with pid %d," + " parent had %d threads\n", (int)internal_getpid(), (int)nthread); + if (nthread == 1) { + internal_start_thread(&BackgroundThread, 0); + } else { + // We've just forked a multi-threaded process. We cannot reasonably function + // after that (some mutexes may be locked before fork). So just enable + // ignores for everything in the hope that we will exec soon. + ctx->after_multithreaded_fork = true; + thr->ignore_interceptors++; + ThreadIgnoreBegin(thr, pc); + ThreadIgnoreSyncBegin(thr, pc); + } +} +#endif + u32 CurrentStackId(ThreadState *thr, uptr pc) { if (thr->shadow_stack_pos == 0) // May happen during bootstrap. return 0; @@ -332,11 +385,9 @@ u32 CurrentStackId(ThreadState *thr, uptr pc) { thr->shadow_stack_pos--; return id; } -#endif void TraceSwitch(ThreadState *thr) { thr->nomalloc++; - ScopedInRtl in_rtl; Trace *thr_trace = ThreadTrace(thr->tid); Lock l(&thr_trace->mtx); unsigned trace = (thr->fast_state.epoch() / kTracePartSize) % TraceParts(); @@ -541,17 +592,19 @@ void MemoryAccess(ThreadState *thr, uptr pc, uptr addr, FastState fast_state = thr->fast_state; if (fast_state.GetIgnoreBit()) return; - fast_state.IncrementEpoch(); - thr->fast_state = fast_state; + if (kCollectHistory) { + fast_state.IncrementEpoch(); + thr->fast_state = fast_state; + // We must not store to the trace if we do not store to the shadow. + // That is, this call must be moved somewhere below. + TraceAddEvent(thr, fast_state, EventTypeMop, pc); + } + Shadow cur(fast_state); cur.SetAddr0AndSizeLog(addr & 7, kAccessSizeLog); cur.SetWrite(kAccessIsWrite); cur.SetAtomic(kIsAtomic); - // We must not store to the trace if we do not store to the shadow. - // That is, this call must be moved somewhere below. - TraceAddEvent(thr, fast_state, EventTypeMop, pc); - MemoryAccessImpl(thr, addr, kAccessSizeLog, kAccessIsWrite, kIsAtomic, shadow_mem, cur); } @@ -581,7 +634,7 @@ static void MemoryRangeSet(ThreadState *thr, uptr pc, uptr addr, uptr size, size = (size + (kShadowCell - 1)) & ~(kShadowCell - 1); // UnmapOrDie/MmapFixedNoReserve does not work on Windows, // so we do it only for C/C++. - if (kGoMode || size < 64*1024) { + if (kGoMode || size < common_flags()->clear_shadow_mmap_threshold) { u64 *p = (u64*)MemToShadow(addr); CHECK(IsShadowMem((uptr)p)); CHECK(IsShadowMem((uptr)(p + size * kShadowCnt / kShadowCell - 1))); @@ -631,8 +684,10 @@ void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) { thr->is_freeing = true; MemoryAccessRange(thr, pc, addr, size, true); thr->is_freeing = false; - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc); + if (kCollectHistory) { + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc); + } Shadow s(thr->fast_state); s.ClearIgnoreBit(); s.MarkAsFreed(); @@ -642,8 +697,10 @@ void MemoryRangeFreed(ThreadState *thr, uptr pc, uptr addr, uptr size) { } void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) { - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc); + if (kCollectHistory) { + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeMop, pc); + } Shadow s(thr->fast_state); s.ClearIgnoreBit(); s.SetWrite(true); @@ -653,11 +710,12 @@ void MemoryRangeImitateWrite(ThreadState *thr, uptr pc, uptr addr, uptr size) { ALWAYS_INLINE USED void FuncEntry(ThreadState *thr, uptr pc) { - DCHECK_EQ(thr->in_rtl, 0); StatInc(thr, StatFuncEnter); DPrintf2("#%d: FuncEntry %p\n", (int)thr->fast_state.tid(), (void*)pc); - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc); + if (kCollectHistory) { + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeFuncEnter, pc); + } // Shadow stack maintenance can be replaced with // stack unwinding during trace switch (which presumably must be faster). @@ -683,11 +741,12 @@ void FuncEntry(ThreadState *thr, uptr pc) { ALWAYS_INLINE USED void FuncExit(ThreadState *thr) { - DCHECK_EQ(thr->in_rtl, 0); StatInc(thr, StatFuncExit); DPrintf2("#%d: FuncExit\n", (int)thr->fast_state.tid()); - thr->fast_state.IncrementEpoch(); - TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0); + if (kCollectHistory) { + thr->fast_state.IncrementEpoch(); + TraceAddEvent(thr, thr->fast_state, EventTypeFuncExit, 0); + } DCHECK_GT(thr->shadow_stack_pos, thr->shadow_stack); #ifndef TSAN_GO @@ -702,7 +761,8 @@ void ThreadIgnoreBegin(ThreadState *thr, uptr pc) { CHECK_GT(thr->ignore_reads_and_writes, 0); thr->fast_state.SetIgnoreBit(); #ifndef TSAN_GO - thr->mop_ignore_set.Add(CurrentStackId(thr, pc)); + if (!ctx->after_multithreaded_fork) + thr->mop_ignore_set.Add(CurrentStackId(thr, pc)); #endif } @@ -723,7 +783,8 @@ void ThreadIgnoreSyncBegin(ThreadState *thr, uptr pc) { thr->ignore_sync++; CHECK_GT(thr->ignore_sync, 0); #ifndef TSAN_GO - thr->sync_ignore_set.Add(CurrentStackId(thr, pc)); + if (!ctx->after_multithreaded_fork) + thr->sync_ignore_set.Add(CurrentStackId(thr, pc)); #endif } @@ -733,7 +794,7 @@ void ThreadIgnoreSyncEnd(ThreadState *thr, uptr pc) { CHECK_GE(thr->ignore_sync, 0); #ifndef TSAN_GO if (thr->ignore_sync == 0) - thr->mop_ignore_set.Reset(); + thr->sync_ignore_set.Reset(); #endif } diff --git a/libsanitizer/tsan/tsan_rtl.h b/libsanitizer/tsan/tsan_rtl.h index 20493ea..e6bc8b2 100644 --- a/libsanitizer/tsan/tsan_rtl.h +++ b/libsanitizer/tsan/tsan_rtl.h @@ -28,6 +28,7 @@ #include "sanitizer_common/sanitizer_allocator_internal.h" #include "sanitizer_common/sanitizer_asm.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_deadlock_detector_interface.h" #include "sanitizer_common/sanitizer_libignore.h" #include "sanitizer_common/sanitizer_suppressions.h" #include "sanitizer_common/sanitizer_thread_registry.h" @@ -431,11 +432,11 @@ struct ThreadState { AllocatorCache alloc_cache; InternalAllocatorCache internal_alloc_cache; Vector<JmpBuf> jmp_bufs; + int ignore_interceptors; #endif u64 stat[StatCnt]; const int tid; const int unique_id; - int in_rtl; bool in_symbolizer; bool in_ignored_lib; bool is_alive; @@ -447,7 +448,9 @@ struct ThreadState { const uptr tls_size; ThreadContext *tctx; - DeadlockDetector deadlock_detector; + InternalDeadlockDetector internal_deadlock_detector; + DDPhysicalThread *dd_pt; + DDLogicalThread *dd_lt; bool in_signal_handler; SignalContext *signal_ctx; @@ -462,13 +465,13 @@ struct ThreadState { int nomalloc; explicit ThreadState(Context *ctx, int tid, int unique_id, u64 epoch, + unsigned reuse_count, uptr stk_addr, uptr stk_size, uptr tls_addr, uptr tls_size); }; -Context *CTX(); - #ifndef TSAN_GO +__attribute__((tls_model("initial-exec"))) extern THREADLOCAL char cur_thread_placeholder[]; INLINE ThreadState *cur_thread() { return reinterpret_cast<ThreadState *>(&cur_thread_placeholder); @@ -480,11 +483,7 @@ class ThreadContext : public ThreadContextBase { explicit ThreadContext(int tid); ~ThreadContext(); ThreadState *thr; -#ifdef TSAN_GO - StackTrace creation_stack; -#else u32 creation_stack_id; -#endif SyncClock sync; // Epoch at which the thread had started. // If we see an event from the thread stamped by an older epoch, @@ -527,6 +526,7 @@ struct Context { Context(); bool initialized; + bool after_multithreaded_fork; SyncTab synctab; @@ -535,12 +535,16 @@ struct Context { int nmissed_expected; atomic_uint64_t last_symbolize_time_ns; + void *background_thread; + atomic_uint32_t stop_background_thread; + ThreadRegistry *thread_registry; Vector<RacyStacks> racy_stacks; Vector<RacyAddress> racy_addresses; // Number of fired suppressions may be large enough. InternalMmapVector<FiredSuppression> fired_suppressions; + DDetector *dd; Flags flags; @@ -549,14 +553,20 @@ struct Context { u64 int_alloc_siz[MBlockTypeCount]; }; -class ScopedInRtl { - public: - ScopedInRtl(); - ~ScopedInRtl(); - private: - ThreadState*thr_; - int in_rtl_; - int errno_; +extern Context *ctx; // The one and the only global runtime context. + +struct ScopedIgnoreInterceptors { + ScopedIgnoreInterceptors() { +#ifndef TSAN_GO + cur_thread()->ignore_interceptors++; +#endif + } + + ~ScopedIgnoreInterceptors() { +#ifndef TSAN_GO + cur_thread()->ignore_interceptors--; +#endif + } }; class ScopedReport { @@ -568,7 +578,10 @@ class ScopedReport { void AddMemoryAccess(uptr addr, Shadow s, const StackTrace *stack, const MutexSet *mset); void AddThread(const ThreadContext *tctx); + void AddThread(int unique_tid); + void AddUniqueTid(int unique_tid); void AddMutex(const SyncVar *s); + u64 AddMutex(u64 id); void AddLocation(uptr addr, uptr size); void AddSleep(u32 stack_id); void SetCount(int count); @@ -576,10 +589,12 @@ class ScopedReport { const ReportDesc *GetReport() const; private: - Context *ctx_; ReportDesc *rep_; + // Symbolizer makes lots of intercepted calls. If we try to process them, + // at best it will cause deadlocks on internal mutexes. + ScopedIgnoreInterceptors ignore_interceptors_; - void AddMutex(u64 id); + void AddDeadMutex(u64 id); ScopedReport(const ScopedReport&); void operator = (const ScopedReport&); @@ -606,10 +621,14 @@ void InitializeInterceptors(); void InitializeLibIgnore(); void InitializeDynamicAnnotations(); +void ForkBefore(ThreadState *thr, uptr pc); +void ForkParentAfter(ThreadState *thr, uptr pc); +void ForkChildAfter(ThreadState *thr, uptr pc); + void ReportRace(ThreadState *thr); bool OutputReport(Context *ctx, const ScopedReport &srep, - const ReportStack *suppress_stack1 = 0, + const ReportStack *suppress_stack1, const ReportStack *suppress_stack2 = 0, const ReportLocation *suppress_loc = 0); bool IsFiredSuppression(Context *ctx, @@ -707,9 +726,10 @@ void ProcessPendingSignals(ThreadState *thr); void MutexCreate(ThreadState *thr, uptr pc, uptr addr, bool rw, bool recursive, bool linker_init); void MutexDestroy(ThreadState *thr, uptr pc, uptr addr); -void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec = 1); +void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec = 1, + bool try_lock = false); int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all = false); -void MutexReadLock(ThreadState *thr, uptr pc, uptr addr); +void MutexReadLock(ThreadState *thr, uptr pc, uptr addr, bool try_lock = false); void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr); void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr); void MutexRepair(ThreadState *thr, uptr pc, uptr addr); // call on EOWNERDEAD @@ -754,6 +774,8 @@ Trace *ThreadTrace(int tid); extern "C" void __tsan_trace_switch(); void ALWAYS_INLINE TraceAddEvent(ThreadState *thr, FastState fs, EventType typ, u64 addr) { + if (!kCollectHistory) + return; DCHECK_GE((int)typ, 0); DCHECK_LE((int)typ, 7); DCHECK_EQ(GetLsb(addr, 61), addr); diff --git a/libsanitizer/tsan/tsan_rtl_mutex.cc b/libsanitizer/tsan/tsan_rtl_mutex.cc index d9a3a3b..3724571 100644 --- a/libsanitizer/tsan/tsan_rtl_mutex.cc +++ b/libsanitizer/tsan/tsan_rtl_mutex.cc @@ -9,6 +9,9 @@ // //===----------------------------------------------------------------------===// +#include <sanitizer_common/sanitizer_deadlock_detector_interface.h> +#include <sanitizer_common/sanitizer_stackdepot.h> + #include "tsan_rtl.h" #include "tsan_flags.h" #include "tsan_sync.h" @@ -18,10 +21,47 @@ namespace __tsan { +void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r); + +struct Callback : DDCallback { + ThreadState *thr; + uptr pc; + + Callback(ThreadState *thr, uptr pc) + : thr(thr) + , pc(pc) { + DDCallback::pt = thr->dd_pt; + DDCallback::lt = thr->dd_lt; + } + + virtual u32 Unwind() { + return CurrentStackId(thr, pc); + } + virtual int UniqueTid() { + return thr->unique_id; + } +}; + +void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s) { + Callback cb(thr, pc); + ctx->dd->MutexInit(&cb, &s->dd); + s->dd.ctx = s->GetId(); +} + +static void ReportMutexMisuse(ThreadState *thr, uptr pc, ReportType typ, + uptr addr, u64 mid) { + ThreadRegistryLock l(ctx->thread_registry); + ScopedReport rep(typ); + rep.AddMutex(mid); + StackTrace trace; + trace.ObtainCurrent(thr, pc); + rep.AddStack(&trace); + rep.AddLocation(addr, 1); + OutputReport(ctx, rep, rep.GetReport()->stacks[0]); +} + void MutexCreate(ThreadState *thr, uptr pc, uptr addr, bool rw, bool recursive, bool linker_init) { - Context *ctx = CTX(); - CHECK_GT(thr->in_rtl, 0); DPrintf("#%d: MutexCreate %zx\n", thr->tid, addr); StatInc(thr, StatMutexCreate); if (!linker_init && IsAppMem(addr)) { @@ -38,8 +78,6 @@ void MutexCreate(ThreadState *thr, uptr pc, uptr addr, } void MutexDestroy(ThreadState *thr, uptr pc, uptr addr) { - Context *ctx = CTX(); - CHECK_GT(thr->in_rtl, 0); DPrintf("#%d: MutexDestroy %zx\n", thr->tid, addr); StatInc(thr, StatMutexDestroy); #ifndef TSAN_GO @@ -51,6 +89,10 @@ void MutexDestroy(ThreadState *thr, uptr pc, uptr addr) { SyncVar *s = ctx->synctab.GetAndRemove(thr, pc, addr); if (s == 0) return; + if (flags()->detect_deadlocks) { + Callback cb(thr, pc); + ctx->dd->MutexDestroy(&cb, &s->dd); + } if (IsAppMem(addr)) { CHECK(!thr->is_freeing); thr->is_freeing = true; @@ -71,30 +113,30 @@ void MutexDestroy(ThreadState *thr, uptr pc, uptr addr) { RestoreStack(last.tid(), last.epoch(), &trace, 0); rep.AddStack(&trace); rep.AddLocation(s->addr, 1); - OutputReport(ctx, rep); + OutputReport(ctx, rep, rep.GetReport()->stacks[0]); } thr->mset.Remove(s->GetId()); DestroyAndFree(s); } -void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec) { - CHECK_GT(thr->in_rtl, 0); +void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec, bool try_lock) { DPrintf("#%d: MutexLock %zx rec=%d\n", thr->tid, addr, rec); CHECK_GT(rec, 0); if (IsAppMem(addr)) MemoryReadAtomic(thr, pc, addr, kSizeLog1); - SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, addr, true); + SyncVar *s = ctx->synctab.GetOrCreateAndLock(thr, pc, addr, true); thr->fast_state.IncrementEpoch(); TraceAddEvent(thr, thr->fast_state, EventTypeLock, s->GetId()); + bool report_double_lock = false; if (s->owner_tid == SyncVar::kInvalidTid) { CHECK_EQ(s->recursion, 0); s->owner_tid = thr->tid; s->last_lock = thr->fast_state.raw(); } else if (s->owner_tid == thr->tid) { CHECK_GT(s->recursion, 0); - } else { - Printf("ThreadSanitizer WARNING: double lock of mutex %p\n", addr); - PrintCurrentStack(thr, pc); + } else if (flags()->report_mutex_bugs && !s->is_broken) { + s->is_broken = true; + report_double_lock = true; } if (s->recursion == 0) { StatInc(thr, StatMutexLock); @@ -105,30 +147,36 @@ void MutexLock(ThreadState *thr, uptr pc, uptr addr, int rec) { } s->recursion += rec; thr->mset.Add(s->GetId(), true, thr->fast_state.epoch()); + if (flags()->detect_deadlocks && s->recursion == 1) { + Callback cb(thr, pc); + if (!try_lock) + ctx->dd->MutexBeforeLock(&cb, &s->dd, true); + ctx->dd->MutexAfterLock(&cb, &s->dd, true, try_lock); + } + u64 mid = s->GetId(); s->mtx.Unlock(); + // Can't touch s after this point. + if (report_double_lock) + ReportMutexMisuse(thr, pc, ReportTypeMutexDoubleLock, addr, mid); + if (flags()->detect_deadlocks) { + Callback cb(thr, pc); + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + } } int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all) { - CHECK_GT(thr->in_rtl, 0); DPrintf("#%d: MutexUnlock %zx all=%d\n", thr->tid, addr, all); if (IsAppMem(addr)) MemoryReadAtomic(thr, pc, addr, kSizeLog1); - SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, addr, true); + SyncVar *s = ctx->synctab.GetOrCreateAndLock(thr, pc, addr, true); thr->fast_state.IncrementEpoch(); TraceAddEvent(thr, thr->fast_state, EventTypeUnlock, s->GetId()); int rec = 0; - if (s->recursion == 0) { - if (!s->is_broken) { - s->is_broken = true; - Printf("ThreadSanitizer WARNING: unlock of unlocked mutex %p\n", addr); - PrintCurrentStack(thr, pc); - } - } else if (s->owner_tid != thr->tid) { - if (!s->is_broken) { + bool report_bad_unlock = false; + if (s->recursion == 0 || s->owner_tid != thr->tid) { + if (flags()->report_mutex_bugs && !s->is_broken) { s->is_broken = true; - Printf("ThreadSanitizer WARNING: mutex %p is unlocked by wrong thread\n", - addr); - PrintCurrentStack(thr, pc); + report_bad_unlock = true; } } else { rec = all ? s->recursion : 1; @@ -142,56 +190,96 @@ int MutexUnlock(ThreadState *thr, uptr pc, uptr addr, bool all) { } } thr->mset.Del(s->GetId(), true); + if (flags()->detect_deadlocks && s->recursion == 0) { + Callback cb(thr, pc); + ctx->dd->MutexBeforeUnlock(&cb, &s->dd, true); + } + u64 mid = s->GetId(); s->mtx.Unlock(); + // Can't touch s after this point. + if (report_bad_unlock) + ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid); + if (flags()->detect_deadlocks) { + Callback cb(thr, pc); + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + } return rec; } -void MutexReadLock(ThreadState *thr, uptr pc, uptr addr) { - CHECK_GT(thr->in_rtl, 0); +void MutexReadLock(ThreadState *thr, uptr pc, uptr addr, bool trylock) { DPrintf("#%d: MutexReadLock %zx\n", thr->tid, addr); StatInc(thr, StatMutexReadLock); if (IsAppMem(addr)) MemoryReadAtomic(thr, pc, addr, kSizeLog1); - SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, addr, false); + SyncVar *s = ctx->synctab.GetOrCreateAndLock(thr, pc, addr, false); thr->fast_state.IncrementEpoch(); TraceAddEvent(thr, thr->fast_state, EventTypeRLock, s->GetId()); + bool report_bad_lock = false; if (s->owner_tid != SyncVar::kInvalidTid) { - Printf("ThreadSanitizer WARNING: read lock of a write locked mutex %p\n", - addr); - PrintCurrentStack(thr, pc); + if (flags()->report_mutex_bugs && !s->is_broken) { + s->is_broken = true; + report_bad_lock = true; + } } AcquireImpl(thr, pc, &s->clock); s->last_lock = thr->fast_state.raw(); thr->mset.Add(s->GetId(), false, thr->fast_state.epoch()); + if (flags()->detect_deadlocks && s->recursion == 0) { + Callback cb(thr, pc); + if (!trylock) + ctx->dd->MutexBeforeLock(&cb, &s->dd, false); + ctx->dd->MutexAfterLock(&cb, &s->dd, false, trylock); + } + u64 mid = s->GetId(); s->mtx.ReadUnlock(); + // Can't touch s after this point. + if (report_bad_lock) + ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadLock, addr, mid); + if (flags()->detect_deadlocks) { + Callback cb(thr, pc); + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + } } void MutexReadUnlock(ThreadState *thr, uptr pc, uptr addr) { - CHECK_GT(thr->in_rtl, 0); DPrintf("#%d: MutexReadUnlock %zx\n", thr->tid, addr); StatInc(thr, StatMutexReadUnlock); if (IsAppMem(addr)) MemoryReadAtomic(thr, pc, addr, kSizeLog1); - SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, addr, true); + SyncVar *s = ctx->synctab.GetOrCreateAndLock(thr, pc, addr, true); thr->fast_state.IncrementEpoch(); TraceAddEvent(thr, thr->fast_state, EventTypeRUnlock, s->GetId()); + bool report_bad_unlock = false; if (s->owner_tid != SyncVar::kInvalidTid) { - Printf("ThreadSanitizer WARNING: read unlock of a write locked mutex %p\n", - addr); - PrintCurrentStack(thr, pc); + if (flags()->report_mutex_bugs && !s->is_broken) { + s->is_broken = true; + report_bad_unlock = true; + } } ReleaseImpl(thr, pc, &s->read_clock); + if (flags()->detect_deadlocks && s->recursion == 0) { + Callback cb(thr, pc); + ctx->dd->MutexBeforeUnlock(&cb, &s->dd, false); + } + u64 mid = s->GetId(); s->mtx.Unlock(); - thr->mset.Del(s->GetId(), false); + // Can't touch s after this point. + thr->mset.Del(mid, false); + if (report_bad_unlock) + ReportMutexMisuse(thr, pc, ReportTypeMutexBadReadUnlock, addr, mid); + if (flags()->detect_deadlocks) { + Callback cb(thr, pc); + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + } } void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) { - CHECK_GT(thr->in_rtl, 0); DPrintf("#%d: MutexReadOrWriteUnlock %zx\n", thr->tid, addr); if (IsAppMem(addr)) MemoryReadAtomic(thr, pc, addr, kSizeLog1); - SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, addr, true); + SyncVar *s = ctx->synctab.GetOrCreateAndLock(thr, pc, addr, true); bool write = true; + bool report_bad_unlock = false; if (s->owner_tid == SyncVar::kInvalidTid) { // Seems to be read unlock. write = false; @@ -214,17 +302,25 @@ void MutexReadOrWriteUnlock(ThreadState *thr, uptr pc, uptr addr) { } } else if (!s->is_broken) { s->is_broken = true; - Printf("ThreadSanitizer WARNING: mutex %p is unlock by wrong thread\n", - addr); - PrintCurrentStack(thr, pc); + report_bad_unlock = true; } thr->mset.Del(s->GetId(), write); + if (flags()->detect_deadlocks && s->recursion == 0) { + Callback cb(thr, pc); + ctx->dd->MutexBeforeUnlock(&cb, &s->dd, write); + } + u64 mid = s->GetId(); s->mtx.Unlock(); + // Can't touch s after this point. + if (report_bad_unlock) + ReportMutexMisuse(thr, pc, ReportTypeMutexBadUnlock, addr, mid); + if (flags()->detect_deadlocks) { + Callback cb(thr, pc); + ReportDeadlock(thr, pc, ctx->dd->GetReport(&cb)); + } } void MutexRepair(ThreadState *thr, uptr pc, uptr addr) { - Context *ctx = CTX(); - CHECK_GT(thr->in_rtl, 0); DPrintf("#%d: MutexRepair %zx\n", thr->tid, addr); SyncVar *s = ctx->synctab.GetOrCreateAndLock(thr, pc, addr, true); s->owner_tid = SyncVar::kInvalidTid; @@ -233,11 +329,10 @@ void MutexRepair(ThreadState *thr, uptr pc, uptr addr) { } void Acquire(ThreadState *thr, uptr pc, uptr addr) { - CHECK_GT(thr->in_rtl, 0); DPrintf("#%d: Acquire %zx\n", thr->tid, addr); if (thr->ignore_sync) return; - SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, addr, false); + SyncVar *s = ctx->synctab.GetOrCreateAndLock(thr, pc, addr, false); AcquireImpl(thr, pc, &s->clock); s->mtx.ReadUnlock(); } @@ -255,17 +350,16 @@ void AcquireGlobal(ThreadState *thr, uptr pc) { DPrintf("#%d: AcquireGlobal\n", thr->tid); if (thr->ignore_sync) return; - ThreadRegistryLock l(CTX()->thread_registry); - CTX()->thread_registry->RunCallbackForEachThreadLocked( + ThreadRegistryLock l(ctx->thread_registry); + ctx->thread_registry->RunCallbackForEachThreadLocked( UpdateClockCallback, thr); } void Release(ThreadState *thr, uptr pc, uptr addr) { - CHECK_GT(thr->in_rtl, 0); DPrintf("#%d: Release %zx\n", thr->tid, addr); if (thr->ignore_sync) return; - SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, addr, true); + SyncVar *s = ctx->synctab.GetOrCreateAndLock(thr, pc, addr, true); thr->fast_state.IncrementEpoch(); // Can't increment epoch w/o writing to the trace as well. TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); @@ -274,11 +368,10 @@ void Release(ThreadState *thr, uptr pc, uptr addr) { } void ReleaseStore(ThreadState *thr, uptr pc, uptr addr) { - CHECK_GT(thr->in_rtl, 0); DPrintf("#%d: ReleaseStore %zx\n", thr->tid, addr); if (thr->ignore_sync) return; - SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, addr, true); + SyncVar *s = ctx->synctab.GetOrCreateAndLock(thr, pc, addr, true); thr->fast_state.IncrementEpoch(); // Can't increment epoch w/o writing to the trace as well. TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0); @@ -301,8 +394,8 @@ void AfterSleep(ThreadState *thr, uptr pc) { if (thr->ignore_sync) return; thr->last_sleep_stack_id = CurrentStackId(thr, pc); - ThreadRegistryLock l(CTX()->thread_registry); - CTX()->thread_registry->RunCallbackForEachThreadLocked( + ThreadRegistryLock l(ctx->thread_registry); + ctx->thread_registry->RunCallbackForEachThreadLocked( UpdateSleepClockCallback, thr); } #endif @@ -310,7 +403,7 @@ void AfterSleep(ThreadState *thr, uptr pc) { void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) { if (thr->ignore_sync) return; - thr->clock.set(thr->tid, thr->fast_state.epoch()); + thr->clock.set(thr->fast_state.epoch()); thr->clock.acquire(c); StatInc(thr, StatSyncAcquire); } @@ -318,7 +411,7 @@ void AcquireImpl(ThreadState *thr, uptr pc, SyncClock *c) { void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { if (thr->ignore_sync) return; - thr->clock.set(thr->tid, thr->fast_state.epoch()); + thr->clock.set(thr->fast_state.epoch()); thr->fast_synch_epoch = thr->fast_state.epoch(); thr->clock.release(c); StatInc(thr, StatSyncRelease); @@ -327,7 +420,7 @@ void ReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) { if (thr->ignore_sync) return; - thr->clock.set(thr->tid, thr->fast_state.epoch()); + thr->clock.set(thr->fast_state.epoch()); thr->fast_synch_epoch = thr->fast_state.epoch(); thr->clock.ReleaseStore(c); StatInc(thr, StatSyncRelease); @@ -336,11 +429,43 @@ void ReleaseStoreImpl(ThreadState *thr, uptr pc, SyncClock *c) { void AcquireReleaseImpl(ThreadState *thr, uptr pc, SyncClock *c) { if (thr->ignore_sync) return; - thr->clock.set(thr->tid, thr->fast_state.epoch()); + thr->clock.set(thr->fast_state.epoch()); thr->fast_synch_epoch = thr->fast_state.epoch(); thr->clock.acq_rel(c); StatInc(thr, StatSyncAcquire); StatInc(thr, StatSyncRelease); } +void ReportDeadlock(ThreadState *thr, uptr pc, DDReport *r) { + if (r == 0) + return; + ThreadRegistryLock l(ctx->thread_registry); + ScopedReport rep(ReportTypeDeadlock); + for (int i = 0; i < r->n; i++) { + rep.AddMutex(r->loop[i].mtx_ctx0); + rep.AddUniqueTid((int)r->loop[i].thr_ctx); + rep.AddThread((int)r->loop[i].thr_ctx); + } + StackTrace stacks[2 * DDReport::kMaxLoopSize]; + uptr dummy_pc = 0x42; + for (int i = 0; i < r->n; i++) { + uptr size; + for (int j = 0; j < (flags()->second_deadlock_stack ? 2 : 1); j++) { + u32 stk = r->loop[i].stk[j]; + if (stk) { + const uptr *trace = StackDepotGet(stk, &size); + stacks[i].Init(const_cast<uptr *>(trace), size); + } else { + // Sometimes we fail to extract the stack trace (FIXME: investigate), + // but we should still produce some stack trace in the report. + stacks[i].Init(&dummy_pc, 1); + } + rep.AddStack(&stacks[i]); + } + } + // FIXME: use all stacks for suppressions, not just the second stack of the + // first edge. + OutputReport(ctx, rep, rep.GetReport()->stacks[0]); +} + } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_rtl_report.cc b/libsanitizer/tsan/tsan_rtl_report.cc index f2248af..d19deb0 100644 --- a/libsanitizer/tsan/tsan_rtl_report.cc +++ b/libsanitizer/tsan/tsan_rtl_report.cc @@ -32,7 +32,10 @@ static ReportStack *SymbolizeStack(const StackTrace& trace); void TsanCheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2) { - ScopedInRtl in_rtl; + // There is high probability that interceptors will check-fail as well, + // on the other hand there is no sense in processing interceptors + // since we are going to die soon. + ScopedIgnoreInterceptors ignore; Printf("FATAL: ThreadSanitizer CHECK failed: " "%s:%d \"%s\" (0x%zx, 0x%zx)\n", file, line, cond, (uptr)v1, (uptr)v2); @@ -99,8 +102,9 @@ static void StackStripMain(ReportStack *stack) { #endif } -#ifndef TSAN_GO ReportStack *SymbolizeStackId(u32 stack_id) { + if (stack_id == 0) + return 0; uptr ssz = 0; const uptr *stack = StackDepotGet(stack_id, &ssz); if (stack == 0) @@ -109,7 +113,6 @@ ReportStack *SymbolizeStackId(u32 stack_id) { trace.Init(stack, ssz); return SymbolizeStack(trace); } -#endif static ReportStack *SymbolizeStack(const StackTrace& trace) { if (trace.IsEmpty()) @@ -143,18 +146,17 @@ static ReportStack *SymbolizeStack(const StackTrace& trace) { } ScopedReport::ScopedReport(ReportType typ) { - ctx_ = CTX(); - ctx_->thread_registry->CheckLocked(); + ctx->thread_registry->CheckLocked(); void *mem = internal_alloc(MBlockReport, sizeof(ReportDesc)); rep_ = new(mem) ReportDesc; rep_->typ = typ; - ctx_->report_mtx.Lock(); + ctx->report_mtx.Lock(); CommonSanitizerReportMutex.Lock(); } ScopedReport::~ScopedReport() { CommonSanitizerReportMutex.Unlock(); - ctx_->report_mtx.Unlock(); + ctx->report_mtx.Unlock(); DestroyAndFree(rep_); } @@ -176,26 +178,16 @@ void ScopedReport::AddMemoryAccess(uptr addr, Shadow s, mop->stack = SymbolizeStack(*stack); for (uptr i = 0; i < mset->Size(); i++) { MutexSet::Desc d = mset->Get(i); - u64 uid = 0; - uptr addr = SyncVar::SplitId(d.id, &uid); - SyncVar *s = ctx_->synctab.GetIfExistsAndLock(addr, false); - // Check that the mutex is still alive. - // Another mutex can be created at the same address, - // so check uid as well. - if (s && s->CheckId(uid)) { - ReportMopMutex mtx = {s->uid, d.write}; - mop->mset.PushBack(mtx); - AddMutex(s); - } else { - ReportMopMutex mtx = {d.id, d.write}; - mop->mset.PushBack(mtx); - AddMutex(d.id); - } - if (s) - s->mtx.ReadUnlock(); + u64 mid = this->AddMutex(d.id); + ReportMopMutex mtx = {mid, d.write}; + mop->mset.PushBack(mtx); } } +void ScopedReport::AddUniqueTid(int unique_tid) { + rep_->unique_tids.PushBack(unique_tid); +} + void ScopedReport::AddThread(const ThreadContext *tctx) { for (uptr i = 0; i < rep_->threads.Size(); i++) { if ((u32)rep_->threads[i]->id == tctx->tid) @@ -207,19 +199,14 @@ void ScopedReport::AddThread(const ThreadContext *tctx) { rt->id = tctx->tid; rt->pid = tctx->os_id; rt->running = (tctx->status == ThreadStatusRunning); - rt->name = tctx->name ? internal_strdup(tctx->name) : 0; + rt->name = internal_strdup(tctx->name); rt->parent_tid = tctx->parent_tid; rt->stack = 0; -#ifdef TSAN_GO - rt->stack = SymbolizeStack(tctx->creation_stack); -#else rt->stack = SymbolizeStackId(tctx->creation_stack_id); -#endif } #ifndef TSAN_GO static ThreadContext *FindThreadByUidLocked(int unique_id) { - Context *ctx = CTX(); ctx->thread_registry->CheckLocked(); for (unsigned i = 0; i < kMaxTid; i++) { ThreadContext *tctx = static_cast<ThreadContext*>( @@ -232,7 +219,6 @@ static ThreadContext *FindThreadByUidLocked(int unique_id) { } static ThreadContext *FindThreadByTidLocked(int tid) { - Context *ctx = CTX(); ctx->thread_registry->CheckLocked(); return static_cast<ThreadContext*>( ctx->thread_registry->GetThreadLocked(tid)); @@ -250,7 +236,6 @@ static bool IsInStackOrTls(ThreadContextBase *tctx_base, void *arg) { } ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) { - Context *ctx = CTX(); ctx->thread_registry->CheckLocked(); ThreadContext *tctx = static_cast<ThreadContext*>( ctx->thread_registry->FindThreadContextLocked(IsInStackOrTls, @@ -264,6 +249,12 @@ ThreadContext *IsThreadStackOrTls(uptr addr, bool *is_stack) { } #endif +void ScopedReport::AddThread(int unique_tid) { +#ifndef TSAN_GO + AddThread(FindThreadByUidLocked(unique_tid)); +#endif +} + void ScopedReport::AddMutex(const SyncVar *s) { for (uptr i = 0; i < rep_->mutexes.Size(); i++) { if (rep_->mutexes[i]->id == s->uid) @@ -273,14 +264,31 @@ void ScopedReport::AddMutex(const SyncVar *s) { ReportMutex *rm = new(mem) ReportMutex(); rep_->mutexes.PushBack(rm); rm->id = s->uid; + rm->addr = s->addr; rm->destroyed = false; - rm->stack = 0; -#ifndef TSAN_GO rm->stack = SymbolizeStackId(s->creation_stack_id); -#endif } -void ScopedReport::AddMutex(u64 id) { +u64 ScopedReport::AddMutex(u64 id) { + u64 uid = 0; + u64 mid = id; + uptr addr = SyncVar::SplitId(id, &uid); + SyncVar *s = ctx->synctab.GetIfExistsAndLock(addr, false); + // Check that the mutex is still alive. + // Another mutex can be created at the same address, + // so check uid as well. + if (s && s->CheckId(uid)) { + mid = s->uid; + AddMutex(s); + } else { + AddDeadMutex(id); + } + if (s) + s->mtx.ReadUnlock(); + return mid; +} + +void ScopedReport::AddDeadMutex(u64 id) { for (uptr i = 0; i < rep_->mutexes.Size(); i++) { if (rep_->mutexes[i]->id == id) return; @@ -289,6 +297,7 @@ void ScopedReport::AddMutex(u64 id) { ReportMutex *rm = new(mem) ReportMutex(); rep_->mutexes.PushBack(rm); rm->id = id; + rm->addr = 0; rm->destroyed = true; rm->stack = 0; } @@ -369,7 +378,6 @@ void RestoreStack(int tid, const u64 epoch, StackTrace *stk, MutexSet *mset) { // This function restores stack trace and mutex set for the thread/epoch. // It does so by getting stack trace and mutex set at the beginning of // trace part, and then replaying the trace till the given epoch. - Context *ctx = CTX(); ctx->thread_registry->CheckLocked(); ThreadContext *tctx = static_cast<ThreadContext*>( ctx->thread_registry->GetThreadLocked(tid)); @@ -434,7 +442,6 @@ void RestoreStack(int tid, const u64 epoch, StackTrace *stk, MutexSet *mset) { static bool HandleRacyStacks(ThreadState *thr, const StackTrace (&traces)[2], uptr addr_min, uptr addr_max) { - Context *ctx = CTX(); bool equal_stack = false; RacyStacks hash; if (flags()->suppress_equal_stacks) { @@ -474,7 +481,6 @@ static bool HandleRacyStacks(ThreadState *thr, const StackTrace (&traces)[2], static void AddRacyStacks(ThreadState *thr, const StackTrace (&traces)[2], uptr addr_min, uptr addr_max) { - Context *ctx = CTX(); if (flags()->suppress_equal_stacks) { RacyStacks hash; hash.hash[0] = md5_hash(traces[0].Begin(), traces[0].Size() * sizeof(uptr)); @@ -579,7 +585,7 @@ static bool IsJavaNonsense(const ReportDesc *rep) { && frame->module == 0)) { if (frame) { FiredSuppression supp = {rep->typ, frame->pc, 0}; - CTX()->fired_suppressions.push_back(supp); + ctx->fired_suppressions.push_back(supp); } return true; } @@ -603,10 +609,12 @@ static bool RaceBetweenAtomicAndFree(ThreadState *thr) { } void ReportRace(ThreadState *thr) { + // Symbolizer makes lots of intercepted calls. If we try to process them, + // at best it will cause deadlocks on internal mutexes. + ScopedIgnoreInterceptors ignore; + if (!flags()->report_bugs) return; - ScopedInRtl in_rtl; - if (!flags()->report_atomic_races && !RaceBetweenAtomicAndFree(thr)) return; @@ -631,7 +639,6 @@ void ReportRace(ThreadState *thr) { return; } - Context *ctx = CTX(); ThreadRegistryLock l0(ctx->thread_registry); ReportType typ = ReportTypeRace; @@ -706,8 +713,8 @@ void PrintCurrentStackSlow() { #ifndef TSAN_GO __sanitizer::StackTrace *ptrace = new(internal_alloc(MBlockStackTrace, sizeof(__sanitizer::StackTrace))) __sanitizer::StackTrace; - ptrace->Unwind(kStackTraceMax, __sanitizer::StackTrace::GetCurrentPc(), - 0, 0, 0, false); + ptrace->Unwind(kStackTraceMax, __sanitizer::StackTrace::GetCurrentPc(), 0, 0, + 0, 0, false); for (uptr i = 0; i < ptrace->size / 2; i++) { uptr tmp = ptrace->trace[i]; ptrace->trace[i] = ptrace->trace[ptrace->size - i - 1]; diff --git a/libsanitizer/tsan/tsan_rtl_thread.cc b/libsanitizer/tsan/tsan_rtl_thread.cc index dea6698..385af7e 100644 --- a/libsanitizer/tsan/tsan_rtl_thread.cc +++ b/libsanitizer/tsan/tsan_rtl_thread.cc @@ -57,11 +57,7 @@ void ThreadContext::OnCreated(void *arg) { // Can't increment epoch w/o writing to the trace as well. TraceAddEvent(args->thr, args->thr->fast_state, EventTypeMop, 0); ReleaseImpl(args->thr, 0, &sync); -#ifdef TSAN_GO - creation_stack.ObtainCurrent(args->thr, args->pc); -#else creation_stack_id = CurrentStackId(args->thr, args->pc); -#endif if (reuse_count == 0) StatInc(args->thr, StatThreadMaxTid); } @@ -87,8 +83,8 @@ void ThreadContext::OnStarted(void *arg) { // from different threads. epoch0 = RoundUp(epoch1 + 1, kTracePartSize); epoch1 = (u64)-1; - new(thr) ThreadState(CTX(), tid, unique_id, - epoch0, args->stk_addr, args->stk_size, args->tls_addr, args->tls_size); + new(thr) ThreadState(ctx, tid, unique_id, epoch0, reuse_count, + args->stk_addr, args->stk_size, args->tls_addr, args->tls_size); #ifndef TSAN_GO thr->shadow_stack = &ThreadTrace(thr->tid)->shadow_stack[0]; thr->shadow_stack_pos = thr->shadow_stack; @@ -104,6 +100,10 @@ void ThreadContext::OnStarted(void *arg) { #ifndef TSAN_GO AllocatorThreadStart(thr); #endif + if (flags()->detect_deadlocks) { + thr->dd_pt = ctx->dd->CreatePhysicalThread(); + thr->dd_lt = ctx->dd->CreateLogicalThread(unique_id); + } thr->fast_synch_epoch = epoch0; AcquireImpl(thr, 0, &sync); thr->fast_state.SetHistorySize(flags()->history_size); @@ -128,11 +128,15 @@ void ThreadContext::OnFinished() { } epoch1 = thr->fast_state.epoch(); + if (flags()->detect_deadlocks) { + ctx->dd->DestroyPhysicalThread(thr->dd_pt); + ctx->dd->DestroyLogicalThread(thr->dd_lt); + } #ifndef TSAN_GO AllocatorThreadFinish(thr); #endif thr->~ThreadState(); - StatAggregate(CTX()->stat, thr->stat); + StatAggregate(ctx->stat, thr->stat); thr = 0; } @@ -177,6 +181,8 @@ static void ReportIgnoresEnabled(ThreadContext *tctx, IgnoreSet *set) { } static void ThreadCheckIgnore(ThreadState *thr) { + if (ctx->after_multithreaded_fork) + return; if (thr->ignore_reads_and_writes) ReportIgnoresEnabled(thr->tctx, &thr->mop_ignore_set); if (thr->ignore_sync) @@ -187,36 +193,31 @@ static void ThreadCheckIgnore(ThreadState *thr) {} #endif void ThreadFinalize(ThreadState *thr) { - CHECK_GT(thr->in_rtl, 0); ThreadCheckIgnore(thr); #ifndef TSAN_GO if (!flags()->report_thread_leaks) return; - ThreadRegistryLock l(CTX()->thread_registry); + ThreadRegistryLock l(ctx->thread_registry); Vector<ThreadLeak> leaks(MBlockScopedBuf); - CTX()->thread_registry->RunCallbackForEachThreadLocked( + ctx->thread_registry->RunCallbackForEachThreadLocked( MaybeReportThreadLeak, &leaks); for (uptr i = 0; i < leaks.Size(); i++) { ScopedReport rep(ReportTypeThreadLeak); rep.AddThread(leaks[i].tctx); rep.SetCount(leaks[i].count); - OutputReport(CTX(), rep); + OutputReport(ctx, rep, rep.GetReport()->threads[0]->stack); } #endif } int ThreadCount(ThreadState *thr) { - CHECK_GT(thr->in_rtl, 0); - Context *ctx = CTX(); uptr result; ctx->thread_registry->GetNumberOfThreads(0, 0, &result); return (int)result; } int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) { - CHECK_GT(thr->in_rtl, 0); StatInc(thr, StatThreadCreate); - Context *ctx = CTX(); OnCreatedArgs args = { thr, pc }; int tid = ctx->thread_registry->CreateThread(uid, detached, thr->tid, &args); DPrintf("#%d: ThreadCreate tid=%d uid=%zu\n", thr->tid, tid, uid); @@ -225,8 +226,6 @@ int ThreadCreate(ThreadState *thr, uptr pc, uptr uid, bool detached) { } void ThreadStart(ThreadState *thr, int tid, uptr os_id) { - Context *ctx = CTX(); - CHECK_GT(thr->in_rtl, 0); uptr stk_addr = 0; uptr stk_size = 0; uptr tls_addr = 0; @@ -259,10 +258,17 @@ void ThreadStart(ThreadState *thr, int tid, uptr os_id) { tr->Lock(); thr->tctx = (ThreadContext*)tr->GetThreadLocked(tid); tr->Unlock(); + +#ifndef TSAN_GO + if (ctx->after_multithreaded_fork) { + thr->ignore_interceptors++; + ThreadIgnoreBegin(thr, 0); + ThreadIgnoreSyncBegin(thr, 0); + } +#endif } void ThreadFinish(ThreadState *thr) { - CHECK_GT(thr->in_rtl, 0); ThreadCheckIgnore(thr); StatInc(thr, StatThreadFinish); if (thr->stk_addr && thr->stk_size) @@ -270,7 +276,6 @@ void ThreadFinish(ThreadState *thr) { if (thr->tls_addr && thr->tls_size) DontNeedShadowFor(thr->tls_addr, thr->tls_size); thr->is_alive = false; - Context *ctx = CTX(); ctx->thread_registry->FinishThread(thr->tid); } @@ -284,33 +289,26 @@ static bool FindThreadByUid(ThreadContextBase *tctx, void *arg) { } int ThreadTid(ThreadState *thr, uptr pc, uptr uid) { - CHECK_GT(thr->in_rtl, 0); - Context *ctx = CTX(); int res = ctx->thread_registry->FindThread(FindThreadByUid, (void*)uid); DPrintf("#%d: ThreadTid uid=%zu tid=%d\n", thr->tid, uid, res); return res; } void ThreadJoin(ThreadState *thr, uptr pc, int tid) { - CHECK_GT(thr->in_rtl, 0); CHECK_GT(tid, 0); CHECK_LT(tid, kMaxTid); DPrintf("#%d: ThreadJoin tid=%d\n", thr->tid, tid); - Context *ctx = CTX(); ctx->thread_registry->JoinThread(tid, thr); } void ThreadDetach(ThreadState *thr, uptr pc, int tid) { - CHECK_GT(thr->in_rtl, 0); CHECK_GT(tid, 0); CHECK_LT(tid, kMaxTid); - Context *ctx = CTX(); ctx->thread_registry->DetachThread(tid); } void ThreadSetName(ThreadState *thr, const char *name) { - CHECK_GT(thr->in_rtl, 0); - CTX()->thread_registry->SetThreadName(thr->tid, name); + ctx->thread_registry->SetThreadName(thr->tid, name); } void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr, diff --git a/libsanitizer/tsan/tsan_stat.cc b/libsanitizer/tsan/tsan_stat.cc index 6bc3453..e8d3a79 100644 --- a/libsanitizer/tsan/tsan_stat.cc +++ b/libsanitizer/tsan/tsan_stat.cc @@ -72,6 +72,28 @@ void StatOutput(u64 *stat) { name[StatSyncAcquire] = " acquired "; name[StatSyncRelease] = " released "; + name[StatClockAcquire] = "Clock acquire "; + name[StatClockAcquireEmpty] = " empty clock "; + name[StatClockAcquireFastRelease] = " fast from release-store "; + name[StatClockAcquireLarge] = " contains my tid "; + name[StatClockAcquireRepeat] = " repeated (fast) "; + name[StatClockAcquireFull] = " full (slow) "; + name[StatClockAcquiredSomething] = " acquired something "; + name[StatClockRelease] = "Clock release "; + name[StatClockReleaseResize] = " resize "; + name[StatClockReleaseFast1] = " fast1 "; + name[StatClockReleaseFast2] = " fast2 "; + name[StatClockReleaseSlow] = " dirty overflow (slow) "; + name[StatClockReleaseFull] = " full (slow) "; + name[StatClockReleaseAcquired] = " was acquired "; + name[StatClockReleaseClearTail] = " clear tail "; + name[StatClockStore] = "Clock release store "; + name[StatClockStoreResize] = " resize "; + name[StatClockStoreFast] = " fast "; + name[StatClockStoreFull] = " slow "; + name[StatClockStoreTail] = " clear tail "; + name[StatClockAcquireRelease] = "Clock acquire-release "; + name[StatAtomic] = "Atomic operations "; name[StatAtomicLoad] = " Including load "; name[StatAtomicStore] = " store "; @@ -96,338 +118,6 @@ void StatOutput(u64 *stat) { name[StatAtomic8] = " size 8 "; name[StatAtomic16] = " size 16 "; - name[StatInterceptor] = "Interceptors "; - name[StatInt_longjmp] = " longjmp "; - name[StatInt_siglongjmp] = " siglongjmp "; - name[StatInt_malloc] = " malloc "; - name[StatInt___libc_memalign] = " __libc_memalign "; - name[StatInt_calloc] = " calloc "; - name[StatInt_realloc] = " realloc "; - name[StatInt_free] = " free "; - name[StatInt_cfree] = " cfree "; - name[StatInt_malloc_usable_size] = " malloc_usable_size "; - name[StatInt_mmap] = " mmap "; - name[StatInt_mmap64] = " mmap64 "; - name[StatInt_munmap] = " munmap "; - name[StatInt_memalign] = " memalign "; - name[StatInt_valloc] = " valloc "; - name[StatInt_pvalloc] = " pvalloc "; - name[StatInt_posix_memalign] = " posix_memalign "; - name[StatInt__Znwm] = " _Znwm "; - name[StatInt__ZnwmRKSt9nothrow_t] = " _ZnwmRKSt9nothrow_t "; - name[StatInt__Znam] = " _Znam "; - name[StatInt__ZnamRKSt9nothrow_t] = " _ZnamRKSt9nothrow_t "; - name[StatInt__ZdlPv] = " _ZdlPv "; - name[StatInt__ZdlPvRKSt9nothrow_t] = " _ZdlPvRKSt9nothrow_t "; - name[StatInt__ZdaPv] = " _ZdaPv "; - name[StatInt__ZdaPvRKSt9nothrow_t] = " _ZdaPvRKSt9nothrow_t "; - name[StatInt_strlen] = " strlen "; - name[StatInt_memset] = " memset "; - name[StatInt_memcpy] = " memcpy "; - name[StatInt_textdomain] = " textdomain "; - name[StatInt_strcmp] = " strcmp "; - name[StatInt_memchr] = " memchr "; - name[StatInt_memrchr] = " memrchr "; - name[StatInt_memmove] = " memmove "; - name[StatInt_memcmp] = " memcmp "; - name[StatInt_strchr] = " strchr "; - name[StatInt_strchrnul] = " strchrnul "; - name[StatInt_strrchr] = " strrchr "; - name[StatInt_strncmp] = " strncmp "; - name[StatInt_strcpy] = " strcpy "; - name[StatInt_strncpy] = " strncpy "; - name[StatInt_strstr] = " strstr "; - name[StatInt_strdup] = " strdup "; - name[StatInt_strcasecmp] = " strcasecmp "; - name[StatInt_strncasecmp] = " strncasecmp "; - name[StatInt_atexit] = " atexit "; - name[StatInt__exit] = " _exit "; - name[StatInt___cxa_guard_acquire] = " __cxa_guard_acquire "; - name[StatInt___cxa_guard_release] = " __cxa_guard_release "; - name[StatInt___cxa_guard_abort] = " __cxa_guard_abort "; - name[StatInt_pthread_create] = " pthread_create "; - name[StatInt_pthread_join] = " pthread_join "; - name[StatInt_pthread_detach] = " pthread_detach "; - name[StatInt_pthread_mutex_init] = " pthread_mutex_init "; - name[StatInt_pthread_mutex_destroy] = " pthread_mutex_destroy "; - name[StatInt_pthread_mutex_lock] = " pthread_mutex_lock "; - name[StatInt_pthread_mutex_trylock] = " pthread_mutex_trylock "; - name[StatInt_pthread_mutex_timedlock] = " pthread_mutex_timedlock "; - name[StatInt_pthread_mutex_unlock] = " pthread_mutex_unlock "; - name[StatInt_pthread_spin_init] = " pthread_spin_init "; - name[StatInt_pthread_spin_destroy] = " pthread_spin_destroy "; - name[StatInt_pthread_spin_lock] = " pthread_spin_lock "; - name[StatInt_pthread_spin_trylock] = " pthread_spin_trylock "; - name[StatInt_pthread_spin_unlock] = " pthread_spin_unlock "; - name[StatInt_pthread_rwlock_init] = " pthread_rwlock_init "; - name[StatInt_pthread_rwlock_destroy] = " pthread_rwlock_destroy "; - name[StatInt_pthread_rwlock_rdlock] = " pthread_rwlock_rdlock "; - name[StatInt_pthread_rwlock_tryrdlock] = " pthread_rwlock_tryrdlock "; - name[StatInt_pthread_rwlock_timedrdlock] - = " pthread_rwlock_timedrdlock "; - name[StatInt_pthread_rwlock_wrlock] = " pthread_rwlock_wrlock "; - name[StatInt_pthread_rwlock_trywrlock] = " pthread_rwlock_trywrlock "; - name[StatInt_pthread_rwlock_timedwrlock] - = " pthread_rwlock_timedwrlock "; - name[StatInt_pthread_rwlock_unlock] = " pthread_rwlock_unlock "; - name[StatInt_pthread_cond_init] = " pthread_cond_init "; - name[StatInt_pthread_cond_destroy] = " pthread_cond_destroy "; - name[StatInt_pthread_cond_signal] = " pthread_cond_signal "; - name[StatInt_pthread_cond_broadcast] = " pthread_cond_broadcast "; - name[StatInt_pthread_cond_wait] = " pthread_cond_wait "; - name[StatInt_pthread_cond_timedwait] = " pthread_cond_timedwait "; - name[StatInt_pthread_barrier_init] = " pthread_barrier_init "; - name[StatInt_pthread_barrier_destroy] = " pthread_barrier_destroy "; - name[StatInt_pthread_barrier_wait] = " pthread_barrier_wait "; - name[StatInt_pthread_once] = " pthread_once "; - name[StatInt_pthread_getschedparam] = " pthread_getschedparam "; - name[StatInt_pthread_setname_np] = " pthread_setname_np "; - name[StatInt_sem_init] = " sem_init "; - name[StatInt_sem_destroy] = " sem_destroy "; - name[StatInt_sem_wait] = " sem_wait "; - name[StatInt_sem_trywait] = " sem_trywait "; - name[StatInt_sem_timedwait] = " sem_timedwait "; - name[StatInt_sem_post] = " sem_post "; - name[StatInt_sem_getvalue] = " sem_getvalue "; - name[StatInt_stat] = " stat "; - name[StatInt___xstat] = " __xstat "; - name[StatInt_stat64] = " stat64 "; - name[StatInt___xstat64] = " __xstat64 "; - name[StatInt_lstat] = " lstat "; - name[StatInt___lxstat] = " __lxstat "; - name[StatInt_lstat64] = " lstat64 "; - name[StatInt___lxstat64] = " __lxstat64 "; - name[StatInt_fstat] = " fstat "; - name[StatInt___fxstat] = " __fxstat "; - name[StatInt_fstat64] = " fstat64 "; - name[StatInt___fxstat64] = " __fxstat64 "; - name[StatInt_open] = " open "; - name[StatInt_open64] = " open64 "; - name[StatInt_creat] = " creat "; - name[StatInt_creat64] = " creat64 "; - name[StatInt_dup] = " dup "; - name[StatInt_dup2] = " dup2 "; - name[StatInt_dup3] = " dup3 "; - name[StatInt_eventfd] = " eventfd "; - name[StatInt_signalfd] = " signalfd "; - name[StatInt_inotify_init] = " inotify_init "; - name[StatInt_inotify_init1] = " inotify_init1 "; - name[StatInt_socket] = " socket "; - name[StatInt_socketpair] = " socketpair "; - name[StatInt_connect] = " connect "; - name[StatInt_bind] = " bind "; - name[StatInt_listen] = " listen "; - name[StatInt_accept] = " accept "; - name[StatInt_accept4] = " accept4 "; - name[StatInt_epoll_create] = " epoll_create "; - name[StatInt_epoll_create1] = " epoll_create1 "; - name[StatInt_close] = " close "; - name[StatInt___close] = " __close "; - name[StatInt___res_iclose] = " __res_iclose "; - name[StatInt_pipe] = " pipe "; - name[StatInt_pipe2] = " pipe2 "; - name[StatInt_read] = " read "; - name[StatInt_prctl] = " prctl "; - name[StatInt_pread] = " pread "; - name[StatInt_pread64] = " pread64 "; - name[StatInt_readv] = " readv "; - name[StatInt_preadv] = " preadv "; - name[StatInt_preadv64] = " preadv64 "; - name[StatInt_write] = " write "; - name[StatInt_pwrite] = " pwrite "; - name[StatInt_pwrite64] = " pwrite64 "; - name[StatInt_writev] = " writev "; - name[StatInt_pwritev] = " pwritev "; - name[StatInt_pwritev64] = " pwritev64 "; - name[StatInt_send] = " send "; - name[StatInt_sendmsg] = " sendmsg "; - name[StatInt_recv] = " recv "; - name[StatInt_recvmsg] = " recvmsg "; - name[StatInt_unlink] = " unlink "; - name[StatInt_fopen] = " fopen "; - name[StatInt_freopen] = " freopen "; - name[StatInt_fclose] = " fclose "; - name[StatInt_fread] = " fread "; - name[StatInt_fwrite] = " fwrite "; - name[StatInt_fflush] = " fflush "; - name[StatInt_abort] = " abort "; - name[StatInt_puts] = " puts "; - name[StatInt_rmdir] = " rmdir "; - name[StatInt_opendir] = " opendir "; - name[StatInt_epoll_ctl] = " epoll_ctl "; - name[StatInt_epoll_wait] = " epoll_wait "; - name[StatInt_poll] = " poll "; - name[StatInt_ppoll] = " ppoll "; - name[StatInt_sigaction] = " sigaction "; - name[StatInt_signal] = " signal "; - name[StatInt_sigsuspend] = " sigsuspend "; - name[StatInt_raise] = " raise "; - name[StatInt_kill] = " kill "; - name[StatInt_pthread_kill] = " pthread_kill "; - name[StatInt_sleep] = " sleep "; - name[StatInt_usleep] = " usleep "; - name[StatInt_nanosleep] = " nanosleep "; - name[StatInt_gettimeofday] = " gettimeofday "; - name[StatInt_fork] = " fork "; - name[StatInt_vscanf] = " vscanf "; - name[StatInt_vsscanf] = " vsscanf "; - name[StatInt_vfscanf] = " vfscanf "; - name[StatInt_scanf] = " scanf "; - name[StatInt_sscanf] = " sscanf "; - name[StatInt_fscanf] = " fscanf "; - name[StatInt___isoc99_vscanf] = " vscanf "; - name[StatInt___isoc99_vsscanf] = " vsscanf "; - name[StatInt___isoc99_vfscanf] = " vfscanf "; - name[StatInt___isoc99_scanf] = " scanf "; - name[StatInt___isoc99_sscanf] = " sscanf "; - name[StatInt___isoc99_fscanf] = " fscanf "; - name[StatInt_on_exit] = " on_exit "; - name[StatInt___cxa_atexit] = " __cxa_atexit "; - name[StatInt_localtime] = " localtime "; - name[StatInt_localtime_r] = " localtime_r "; - name[StatInt_gmtime] = " gmtime "; - name[StatInt_gmtime_r] = " gmtime_r "; - name[StatInt_ctime] = " ctime "; - name[StatInt_ctime_r] = " ctime_r "; - name[StatInt_asctime] = " asctime "; - name[StatInt_asctime_r] = " asctime_r "; - name[StatInt_strptime] = " strptime "; - name[StatInt_frexp] = " frexp "; - name[StatInt_frexpf] = " frexpf "; - name[StatInt_frexpl] = " frexpl "; - name[StatInt_getpwnam] = " getpwnam "; - name[StatInt_getpwuid] = " getpwuid "; - name[StatInt_getgrnam] = " getgrnam "; - name[StatInt_getgrgid] = " getgrgid "; - name[StatInt_getpwnam_r] = " getpwnam_r "; - name[StatInt_getpwuid_r] = " getpwuid_r "; - name[StatInt_getgrnam_r] = " getgrnam_r "; - name[StatInt_getgrgid_r] = " getgrgid_r "; - name[StatInt_clock_getres] = " clock_getres "; - name[StatInt_clock_gettime] = " clock_gettime "; - name[StatInt_clock_settime] = " clock_settime "; - name[StatInt_getitimer] = " getitimer "; - name[StatInt_setitimer] = " setitimer "; - name[StatInt_time] = " time "; - name[StatInt_glob] = " glob "; - name[StatInt_glob64] = " glob64 "; - name[StatInt_wait] = " wait "; - name[StatInt_waitid] = " waitid "; - name[StatInt_waitpid] = " waitpid "; - name[StatInt_wait3] = " wait3 "; - name[StatInt_wait4] = " wait4 "; - name[StatInt_inet_ntop] = " inet_ntop "; - name[StatInt_inet_pton] = " inet_pton "; - name[StatInt_inet_aton] = " inet_aton "; - name[StatInt_getaddrinfo] = " getaddrinfo "; - name[StatInt_getnameinfo] = " getnameinfo "; - name[StatInt_getsockname] = " getsockname "; - name[StatInt_gethostent] = " gethostent "; - name[StatInt_gethostbyname] = " gethostbyname "; - name[StatInt_gethostbyname2] = " gethostbyname2 "; - name[StatInt_gethostbyaddr] = " gethostbyaddr "; - name[StatInt_gethostent_r] = " gethostent_r "; - name[StatInt_gethostbyname_r] = " gethostbyname_r "; - name[StatInt_gethostbyname2_r] = " gethostbyname2_r "; - name[StatInt_gethostbyaddr_r] = " gethostbyaddr_r "; - name[StatInt_getsockopt] = " getsockopt "; - name[StatInt_modf] = " modf "; - name[StatInt_modff] = " modff "; - name[StatInt_modfl] = " modfl "; - name[StatInt_getpeername] = " getpeername "; - name[StatInt_ioctl] = " ioctl "; - name[StatInt_sysinfo] = " sysinfo "; - name[StatInt_readdir] = " readdir "; - name[StatInt_readdir64] = " readdir64 "; - name[StatInt_readdir_r] = " readdir_r "; - name[StatInt_readdir64_r] = " readdir64_r "; - name[StatInt_ptrace] = " ptrace "; - name[StatInt_setlocale] = " setlocale "; - name[StatInt_getcwd] = " getcwd "; - name[StatInt_get_current_dir_name] = " get_current_dir_name "; - name[StatInt_strtoimax] = " strtoimax "; - name[StatInt_strtoumax] = " strtoumax "; - name[StatInt_mbstowcs] = " mbstowcs "; - name[StatInt_mbsrtowcs] = " mbsrtowcs "; - name[StatInt_mbsnrtowcs] = " mbsnrtowcs "; - name[StatInt_wcstombs] = " wcstombs "; - name[StatInt_wcsrtombs] = " wcsrtombs "; - name[StatInt_wcsnrtombs] = " wcsnrtombs "; - name[StatInt_tcgetattr] = " tcgetattr "; - name[StatInt_realpath] = " realpath "; - name[StatInt_canonicalize_file_name] = " canonicalize_file_name "; - name[StatInt_confstr] = " confstr "; - name[StatInt_sched_getaffinity] = " sched_getaffinity "; - name[StatInt_strerror] = " strerror "; - name[StatInt_strerror_r] = " strerror_r "; - name[StatInt___xpg_strerror_r] = " __xpg_strerror_r "; - name[StatInt_scandir] = " scandir "; - name[StatInt_scandir64] = " scandir64 "; - name[StatInt_getgroups] = " getgroups "; - name[StatInt_wordexp] = " wordexp "; - name[StatInt_sigwait] = " sigwait "; - name[StatInt_sigwaitinfo] = " sigwaitinfo "; - name[StatInt_sigtimedwait] = " sigtimedwait "; - name[StatInt_sigemptyset] = " sigemptyset "; - name[StatInt_sigfillset] = " sigfillset "; - name[StatInt_sigpending] = " sigpending "; - name[StatInt_sigprocmask] = " sigprocmask "; - name[StatInt_backtrace] = " backtrace "; - name[StatInt_backtrace_symbols] = " backtrace_symbols "; - name[StatInt_dlopen] = " dlopen "; - name[StatInt_dlclose] = " dlclose "; - name[StatInt_getmntent] = " getmntent "; - name[StatInt_getmntent_r] = " getmntent_r "; - name[StatInt_statfs] = " statfs "; - name[StatInt_statfs64] = " statfs64 "; - name[StatInt_fstatfs] = " fstatfs "; - name[StatInt_fstatfs64] = " fstatfs64 "; - name[StatInt_statvfs] = " statvfs "; - name[StatInt_statvfs64] = " statvfs64 "; - name[StatInt_fstatvfs] = " fstatvfs "; - name[StatInt_fstatvfs64] = " fstatvfs64 "; - name[StatInt_initgroups] = " initgroups "; - name[StatInt_ether_ntoa] = " ether_ntoa "; - name[StatInt_ether_aton] = " ether_aton "; - name[StatInt_ether_ntoa_r] = " ether_ntoa_r "; - name[StatInt_ether_aton_r] = " ether_aton_r "; - name[StatInt_ether_ntohost] = " ether_ntohost "; - name[StatInt_ether_hostton] = " ether_hostton "; - name[StatInt_ether_line] = " ether_line "; - name[StatInt_shmctl] = " shmctl "; - name[StatInt_random_r] = " random_r "; - name[StatInt_tmpnam] = " tmpnam "; - name[StatInt_tmpnam_r] = " tmpnam_r "; - name[StatInt_tempnam] = " tempnam "; - name[StatInt_sincos] = " sincos "; - name[StatInt_sincosf] = " sincosf "; - name[StatInt_sincosl] = " sincosl "; - name[StatInt_remquo] = " remquo "; - name[StatInt_remquof] = " remquof "; - name[StatInt_remquol] = " remquol "; - name[StatInt_lgamma] = " lgamma "; - name[StatInt_lgammaf] = " lgammaf "; - name[StatInt_lgammal] = " lgammal "; - name[StatInt_lgamma_r] = " lgamma_r "; - name[StatInt_lgammaf_r] = " lgammaf_r "; - name[StatInt_lgammal_r] = " lgammal_r "; - name[StatInt_drand48_r] = " drand48_r "; - name[StatInt_lrand48_r] = " lrand48_r "; - name[StatInt_getline] = " getline "; - name[StatInt_getdelim] = " getdelim "; - name[StatInt_iconv] = " iconv "; - name[StatInt_times] = " times "; - - name[StatInt_pthread_attr_getdetachstate] = " pthread_addr_getdetachstate "; // NOLINT - name[StatInt_pthread_attr_getguardsize] = " pthread_addr_getguardsize "; // NOLINT - name[StatInt_pthread_attr_getschedparam] = " pthread_addr_getschedparam "; // NOLINT - name[StatInt_pthread_attr_getschedpolicy] = " pthread_addr_getschedpolicy "; // NOLINT - name[StatInt_pthread_attr_getinheritsched] = " pthread_addr_getinheritsched "; // NOLINT - name[StatInt_pthread_attr_getscope] = " pthread_addr_getscope "; // NOLINT - name[StatInt_pthread_attr_getstacksize] = " pthread_addr_getstacksize "; // NOLINT - name[StatInt_pthread_attr_getstack] = " pthread_addr_getstack "; // NOLINT - name[StatInt_pthread_attr_getaffinity_np] = " pthread_addr_getaffinity_np "; // NOLINT - name[StatAnnotation] = "Dynamic annotations "; name[StatAnnotateHappensBefore] = " HappensBefore "; name[StatAnnotateHappensAfter] = " HappensAfter "; @@ -475,11 +165,12 @@ void StatOutput(u64 *stat) { name[StatMtxAnnotations] = " Annotations "; name[StatMtxMBlock] = " MBlock "; name[StatMtxJavaMBlock] = " JavaMBlock "; + name[StatMtxDeadlockDetector] = " DeadlockDetector "; name[StatMtxFD] = " FD "; Printf("Statistics:\n"); for (int i = 0; i < StatCnt; i++) - Printf("%s: %zu\n", name[i], (uptr)stat[i]); + Printf("%s: %16zu\n", name[i], (uptr)stat[i]); } } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_stat.h b/libsanitizer/tsan/tsan_stat.h index 3e08313..5bdd9de 100644 --- a/libsanitizer/tsan/tsan_stat.h +++ b/libsanitizer/tsan/tsan_stat.h @@ -67,6 +67,32 @@ enum StatType { StatSyncAcquire, StatSyncRelease, + // Clocks - acquire. + StatClockAcquire, + StatClockAcquireEmpty, + StatClockAcquireFastRelease, + StatClockAcquireLarge, + StatClockAcquireRepeat, + StatClockAcquireFull, + StatClockAcquiredSomething, + // Clocks - release. + StatClockRelease, + StatClockReleaseResize, + StatClockReleaseFast1, + StatClockReleaseFast2, + StatClockReleaseSlow, + StatClockReleaseFull, + StatClockReleaseAcquired, + StatClockReleaseClearTail, + // Clocks - release store. + StatClockStore, + StatClockStoreResize, + StatClockStoreFast, + StatClockStoreFull, + StatClockStoreTail, + // Clocks - acquire-release. + StatClockAcquireRelease, + // Atomics. StatAtomic, StatAtomicLoad, @@ -92,337 +118,6 @@ enum StatType { StatAtomic8, StatAtomic16, - // Interceptors. - StatInterceptor, - StatInt_longjmp, - StatInt_siglongjmp, - StatInt_malloc, - StatInt___libc_memalign, - StatInt_calloc, - StatInt_realloc, - StatInt_free, - StatInt_cfree, - StatInt_malloc_usable_size, - StatInt_mmap, - StatInt_mmap64, - StatInt_munmap, - StatInt_memalign, - StatInt_valloc, - StatInt_pvalloc, - StatInt_posix_memalign, - StatInt__Znwm, - StatInt__ZnwmRKSt9nothrow_t, - StatInt__Znam, - StatInt__ZnamRKSt9nothrow_t, - StatInt__ZdlPv, - StatInt__ZdlPvRKSt9nothrow_t, - StatInt__ZdaPv, - StatInt__ZdaPvRKSt9nothrow_t, - StatInt_strlen, - StatInt_memset, - StatInt_memcpy, - StatInt_textdomain, - StatInt_strcmp, - StatInt_memchr, - StatInt_memrchr, - StatInt_memmove, - StatInt_memcmp, - StatInt_strchr, - StatInt_strchrnul, - StatInt_strrchr, - StatInt_strncmp, - StatInt_strcpy, - StatInt_strncpy, - StatInt_strcasecmp, - StatInt_strncasecmp, - StatInt_strstr, - StatInt_strdup, - StatInt_atexit, - StatInt__exit, - StatInt___cxa_guard_acquire, - StatInt___cxa_guard_release, - StatInt___cxa_guard_abort, - StatInt_pthread_create, - StatInt_pthread_join, - StatInt_pthread_detach, - StatInt_pthread_mutex_init, - StatInt_pthread_mutex_destroy, - StatInt_pthread_mutex_lock, - StatInt_pthread_mutex_trylock, - StatInt_pthread_mutex_timedlock, - StatInt_pthread_mutex_unlock, - StatInt_pthread_spin_init, - StatInt_pthread_spin_destroy, - StatInt_pthread_spin_lock, - StatInt_pthread_spin_trylock, - StatInt_pthread_spin_unlock, - StatInt_pthread_rwlock_init, - StatInt_pthread_rwlock_destroy, - StatInt_pthread_rwlock_rdlock, - StatInt_pthread_rwlock_tryrdlock, - StatInt_pthread_rwlock_timedrdlock, - StatInt_pthread_rwlock_wrlock, - StatInt_pthread_rwlock_trywrlock, - StatInt_pthread_rwlock_timedwrlock, - StatInt_pthread_rwlock_unlock, - StatInt_pthread_cond_init, - StatInt_pthread_cond_destroy, - StatInt_pthread_cond_signal, - StatInt_pthread_cond_broadcast, - StatInt_pthread_cond_wait, - StatInt_pthread_cond_timedwait, - StatInt_pthread_barrier_init, - StatInt_pthread_barrier_destroy, - StatInt_pthread_barrier_wait, - StatInt_pthread_once, - StatInt_pthread_getschedparam, - StatInt_pthread_setname_np, - StatInt_sem_init, - StatInt_sem_destroy, - StatInt_sem_wait, - StatInt_sem_trywait, - StatInt_sem_timedwait, - StatInt_sem_post, - StatInt_sem_getvalue, - StatInt_stat, - StatInt___xstat, - StatInt_stat64, - StatInt___xstat64, - StatInt_lstat, - StatInt___lxstat, - StatInt_lstat64, - StatInt___lxstat64, - StatInt_fstat, - StatInt___fxstat, - StatInt_fstat64, - StatInt___fxstat64, - StatInt_open, - StatInt_open64, - StatInt_creat, - StatInt_creat64, - StatInt_dup, - StatInt_dup2, - StatInt_dup3, - StatInt_eventfd, - StatInt_signalfd, - StatInt_inotify_init, - StatInt_inotify_init1, - StatInt_socket, - StatInt_socketpair, - StatInt_connect, - StatInt_bind, - StatInt_listen, - StatInt_accept, - StatInt_accept4, - StatInt_epoll_create, - StatInt_epoll_create1, - StatInt_close, - StatInt___close, - StatInt___res_iclose, - StatInt_pipe, - StatInt_pipe2, - StatInt_read, - StatInt_prctl, - StatInt_pread, - StatInt_pread64, - StatInt_readv, - StatInt_preadv, - StatInt_preadv64, - StatInt_write, - StatInt_pwrite, - StatInt_pwrite64, - StatInt_writev, - StatInt_pwritev, - StatInt_pwritev64, - StatInt_send, - StatInt_sendmsg, - StatInt_recv, - StatInt_recvmsg, - StatInt_unlink, - StatInt_fopen, - StatInt_freopen, - StatInt_fclose, - StatInt_fread, - StatInt_fwrite, - StatInt_fflush, - StatInt_abort, - StatInt_puts, - StatInt_rmdir, - StatInt_opendir, - StatInt_epoll_ctl, - StatInt_epoll_wait, - StatInt_poll, - StatInt_ppoll, - StatInt_sigaction, - StatInt_signal, - StatInt_sigsuspend, - StatInt_raise, - StatInt_kill, - StatInt_pthread_kill, - StatInt_sleep, - StatInt_usleep, - StatInt_nanosleep, - StatInt_gettimeofday, - StatInt_fork, - StatInt_vscanf, - StatInt_vsscanf, - StatInt_vfscanf, - StatInt_scanf, - StatInt_sscanf, - StatInt_fscanf, - StatInt___isoc99_vscanf, - StatInt___isoc99_vsscanf, - StatInt___isoc99_vfscanf, - StatInt___isoc99_scanf, - StatInt___isoc99_sscanf, - StatInt___isoc99_fscanf, - StatInt_on_exit, - StatInt___cxa_atexit, - StatInt_localtime, - StatInt_localtime_r, - StatInt_gmtime, - StatInt_gmtime_r, - StatInt_ctime, - StatInt_ctime_r, - StatInt_asctime, - StatInt_asctime_r, - StatInt_strptime, - StatInt_frexp, - StatInt_frexpf, - StatInt_frexpl, - StatInt_getpwnam, - StatInt_getpwuid, - StatInt_getgrnam, - StatInt_getgrgid, - StatInt_getpwnam_r, - StatInt_getpwuid_r, - StatInt_getgrnam_r, - StatInt_getgrgid_r, - StatInt_clock_getres, - StatInt_clock_gettime, - StatInt_clock_settime, - StatInt_getitimer, - StatInt_setitimer, - StatInt_time, - StatInt_glob, - StatInt_glob64, - StatInt_wait, - StatInt_waitid, - StatInt_waitpid, - StatInt_wait3, - StatInt_wait4, - StatInt_inet_ntop, - StatInt_inet_pton, - StatInt_inet_aton, - StatInt_getaddrinfo, - StatInt_getnameinfo, - StatInt_getsockname, - StatInt_gethostent, - StatInt_gethostbyname, - StatInt_gethostbyname2, - StatInt_gethostbyaddr, - StatInt_gethostent_r, - StatInt_gethostbyname_r, - StatInt_gethostbyname2_r, - StatInt_gethostbyaddr_r, - StatInt_getsockopt, - StatInt_modf, - StatInt_modff, - StatInt_modfl, - StatInt_getpeername, - StatInt_ioctl, - StatInt_sysinfo, - StatInt_readdir, - StatInt_readdir64, - StatInt_readdir_r, - StatInt_readdir64_r, - StatInt_ptrace, - StatInt_setlocale, - StatInt_getcwd, - StatInt_get_current_dir_name, - StatInt_strtoimax, - StatInt_strtoumax, - StatInt_mbstowcs, - StatInt_mbsrtowcs, - StatInt_mbsnrtowcs, - StatInt_wcstombs, - StatInt_wcsrtombs, - StatInt_wcsnrtombs, - StatInt_tcgetattr, - StatInt_realpath, - StatInt_canonicalize_file_name, - StatInt_confstr, - StatInt_sched_getaffinity, - StatInt_strerror, - StatInt_strerror_r, - StatInt___xpg_strerror_r, - StatInt_scandir, - StatInt_scandir64, - StatInt_getgroups, - StatInt_wordexp, - StatInt_sigwait, - StatInt_sigwaitinfo, - StatInt_sigtimedwait, - StatInt_sigemptyset, - StatInt_sigfillset, - StatInt_sigpending, - StatInt_sigprocmask, - StatInt_backtrace, - StatInt_backtrace_symbols, - StatInt_dlopen, - StatInt_dlclose, - StatInt_getmntent, - StatInt_getmntent_r, - StatInt_statfs, - StatInt_statfs64, - StatInt_fstatfs, - StatInt_fstatfs64, - StatInt_statvfs, - StatInt_statvfs64, - StatInt_fstatvfs, - StatInt_fstatvfs64, - StatInt_initgroups, - StatInt_ether_ntoa, - StatInt_ether_aton, - StatInt_ether_ntoa_r, - StatInt_ether_aton_r, - StatInt_ether_ntohost, - StatInt_ether_hostton, - StatInt_ether_line, - StatInt_shmctl, - StatInt_random_r, - StatInt_tmpnam, - StatInt_tmpnam_r, - StatInt_tempnam, - StatInt_sincos, - StatInt_sincosf, - StatInt_sincosl, - StatInt_remquo, - StatInt_remquof, - StatInt_remquol, - StatInt_lgamma, - StatInt_lgammaf, - StatInt_lgammal, - StatInt_lgamma_r, - StatInt_lgammaf_r, - StatInt_lgammal_r, - StatInt_drand48_r, - StatInt_lrand48_r, - StatInt_getline, - StatInt_getdelim, - StatInt_iconv, - StatInt_times, - - StatInt_pthread_attr_getdetachstate, - StatInt_pthread_attr_getguardsize, - StatInt_pthread_attr_getschedparam, - StatInt_pthread_attr_getschedpolicy, - StatInt_pthread_attr_getinheritsched, - StatInt_pthread_attr_getscope, - StatInt_pthread_attr_getstacksize, - StatInt_pthread_attr_getstack, - StatInt_pthread_attr_getaffinity_np, - // Dynamic annotations. StatAnnotation, StatAnnotateHappensBefore, @@ -472,6 +167,7 @@ enum StatType { StatMtxAtExit, StatMtxMBlock, StatMtxJavaMBlock, + StatMtxDeadlockDetector, StatMtxFD, // This must be the last. diff --git a/libsanitizer/tsan/tsan_suppressions.cc b/libsanitizer/tsan/tsan_suppressions.cc index fa0c30d..ce8d5fe 100644 --- a/libsanitizer/tsan/tsan_suppressions.cc +++ b/libsanitizer/tsan/tsan_suppressions.cc @@ -101,10 +101,20 @@ SuppressionType conv(ReportType typ) { return SuppressionThread; else if (typ == ReportTypeMutexDestroyLocked) return SuppressionMutex; + else if (typ == ReportTypeMutexDoubleLock) + return SuppressionMutex; + else if (typ == ReportTypeMutexBadUnlock) + return SuppressionMutex; + else if (typ == ReportTypeMutexBadReadLock) + return SuppressionMutex; + else if (typ == ReportTypeMutexBadReadUnlock) + return SuppressionMutex; else if (typ == ReportTypeSignalUnsafe) return SuppressionSignal; else if (typ == ReportTypeErrnoInSignal) return SuppressionNone; + else if (typ == ReportTypeDeadlock) + return SuppressionDeadlock; Printf("ThreadSanitizer: unknown report type %d\n", typ), Die(); } diff --git a/libsanitizer/tsan/tsan_symbolize.cc b/libsanitizer/tsan/tsan_symbolize.cc index c0e794b..fa36017 100644 --- a/libsanitizer/tsan/tsan_symbolize.cc +++ b/libsanitizer/tsan/tsan_symbolize.cc @@ -24,12 +24,14 @@ void EnterSymbolizer() { ThreadState *thr = cur_thread(); CHECK(!thr->in_symbolizer); thr->in_symbolizer = true; + thr->ignore_interceptors++; } void ExitSymbolizer() { ThreadState *thr = cur_thread(); CHECK(thr->in_symbolizer); thr->in_symbolizer = false; + thr->ignore_interceptors--; } ReportStack *NewReportStackEntry(uptr addr) { @@ -103,13 +105,11 @@ ReportStack *SymbolizeCode(uptr addr) { ent->col = col; return ent; } - if (!Symbolizer::Get()->IsAvailable()) - return SymbolizeCodeAddr2Line(addr); static const uptr kMaxAddrFrames = 16; InternalScopedBuffer<AddressInfo> addr_frames(kMaxAddrFrames); for (uptr i = 0; i < kMaxAddrFrames; i++) new(&addr_frames[i]) AddressInfo(); - uptr addr_frames_num = Symbolizer::Get()->SymbolizeCode( + uptr addr_frames_num = Symbolizer::Get()->SymbolizePC( addr, addr_frames.data(), kMaxAddrFrames); if (addr_frames_num == 0) return NewReportStackEntry(addr); @@ -129,8 +129,6 @@ ReportStack *SymbolizeCode(uptr addr) { } ReportLocation *SymbolizeData(uptr addr) { - if (!Symbolizer::Get()->IsAvailable()) - return 0; DataInfo info; if (!Symbolizer::Get()->SymbolizeData(addr, &info)) return 0; @@ -148,8 +146,6 @@ ReportLocation *SymbolizeData(uptr addr) { } void SymbolizeFlush() { - if (!Symbolizer::Get()->IsAvailable()) - return; Symbolizer::Get()->Flush(); } diff --git a/libsanitizer/tsan/tsan_symbolize.h b/libsanitizer/tsan/tsan_symbolize.h index 892c11c..0d9077e 100644 --- a/libsanitizer/tsan/tsan_symbolize.h +++ b/libsanitizer/tsan/tsan_symbolize.h @@ -22,8 +22,6 @@ ReportStack *SymbolizeCode(uptr addr); ReportLocation *SymbolizeData(uptr addr); void SymbolizeFlush(); -ReportStack *SymbolizeCodeAddr2Line(uptr addr); - ReportStack *NewReportStackEntry(uptr addr); } // namespace __tsan diff --git a/libsanitizer/tsan/tsan_symbolize_addr2line_linux.cc b/libsanitizer/tsan/tsan_symbolize_addr2line_linux.cc deleted file mode 100644 index c278a42..0000000 --- a/libsanitizer/tsan/tsan_symbolize_addr2line_linux.cc +++ /dev/null @@ -1,191 +0,0 @@ -//===-- tsan_symbolize_addr2line.cc ---------------------------------------===// -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file is a part of ThreadSanitizer (TSan), a race detector. -// -//===----------------------------------------------------------------------===// -#include "sanitizer_common/sanitizer_common.h" -#include "sanitizer_common/sanitizer_libc.h" -#include "tsan_symbolize.h" -#include "tsan_mman.h" -#include "tsan_rtl.h" -#include "tsan_platform.h" - -#include <unistd.h> -#include <stdlib.h> -#include <stdio.h> -#include <errno.h> -#include <link.h> -#include <linux/limits.h> -#include <sys/types.h> - -namespace __tsan { - -struct ModuleDesc { - const char *fullname; - const char *name; - uptr base; - int inp_fd; - int out_fd; -}; - -struct SectionDesc { - SectionDesc *next; - ModuleDesc *module; - uptr base; - uptr end; -}; - -struct DlIteratePhdrCtx { - SectionDesc *sections; - bool is_first; -}; - -static void NOINLINE InitModule(ModuleDesc *m) { - int outfd[2] = {}; - if (pipe(&outfd[0])) { - Printf("ThreadSanitizer: outfd pipe() failed (%d)\n", errno); - Die(); - } - int infd[2] = {}; - if (pipe(&infd[0])) { - Printf("ThreadSanitizer: infd pipe() failed (%d)\n", errno); - Die(); - } - int pid = fork(); - if (pid == 0) { - internal_close(STDOUT_FILENO); - internal_close(STDIN_FILENO); - internal_dup2(outfd[0], STDIN_FILENO); - internal_dup2(infd[1], STDOUT_FILENO); - internal_close(outfd[0]); - internal_close(outfd[1]); - internal_close(infd[0]); - internal_close(infd[1]); - for (int fd = getdtablesize(); fd > 2; fd--) - internal_close(fd); - execl("/usr/bin/addr2line", "/usr/bin/addr2line", "-Cfe", m->fullname, 0); - _exit(0); - } else if (pid < 0) { - Printf("ThreadSanitizer: failed to fork symbolizer\n"); - Die(); - } - internal_close(outfd[0]); - internal_close(infd[1]); - m->inp_fd = infd[0]; - m->out_fd = outfd[1]; -} - -static int dl_iterate_phdr_cb(dl_phdr_info *info, size_t size, void *arg) { - DlIteratePhdrCtx *ctx = (DlIteratePhdrCtx*)arg; - InternalScopedBuffer<char> tmp(128); - if (ctx->is_first) { - internal_snprintf(tmp.data(), tmp.size(), "/proc/%d/exe", - (int)internal_getpid()); - info->dlpi_name = tmp.data(); - } - ctx->is_first = false; - if (info->dlpi_name == 0 || info->dlpi_name[0] == 0) - return 0; - ModuleDesc *m = (ModuleDesc*)internal_alloc(MBlockReportStack, - sizeof(ModuleDesc)); - m->fullname = internal_strdup(info->dlpi_name); - m->name = internal_strrchr(m->fullname, '/'); - if (m->name) - m->name += 1; - else - m->name = m->fullname; - m->base = (uptr)info->dlpi_addr; - m->inp_fd = -1; - m->out_fd = -1; - DPrintf2("Module %s %zx\n", m->name, m->base); - for (int i = 0; i < info->dlpi_phnum; i++) { - const Elf64_Phdr *s = &info->dlpi_phdr[i]; - DPrintf2(" Section p_type=%zx p_offset=%zx p_vaddr=%zx p_paddr=%zx" - " p_filesz=%zx p_memsz=%zx p_flags=%zx p_align=%zx\n", - (uptr)s->p_type, (uptr)s->p_offset, (uptr)s->p_vaddr, - (uptr)s->p_paddr, (uptr)s->p_filesz, (uptr)s->p_memsz, - (uptr)s->p_flags, (uptr)s->p_align); - if (s->p_type != PT_LOAD) - continue; - SectionDesc *sec = (SectionDesc*)internal_alloc(MBlockReportStack, - sizeof(SectionDesc)); - sec->module = m; - sec->base = info->dlpi_addr + s->p_vaddr; - sec->end = sec->base + s->p_memsz; - sec->next = ctx->sections; - ctx->sections = sec; - DPrintf2(" Section %zx-%zx\n", sec->base, sec->end); - } - return 0; -} - -static SectionDesc *InitSections() { - DlIteratePhdrCtx ctx = {0, true}; - dl_iterate_phdr(dl_iterate_phdr_cb, &ctx); - return ctx.sections; -} - -static SectionDesc *GetSectionDesc(uptr addr) { - static SectionDesc *sections = 0; - if (sections == 0) - sections = InitSections(); - for (SectionDesc *s = sections; s; s = s->next) { - if (addr >= s->base && addr < s->end) { - if (s->module->inp_fd == -1) - InitModule(s->module); - return s; - } - } - return 0; -} - -ReportStack *SymbolizeCodeAddr2Line(uptr addr) { - SectionDesc *s = GetSectionDesc(addr); - if (s == 0) - return NewReportStackEntry(addr); - ModuleDesc *m = s->module; - uptr offset = addr - m->base; - char addrstr[32]; - internal_snprintf(addrstr, sizeof(addrstr), "%p\n", (void*)offset); - if (0 >= internal_write(m->out_fd, addrstr, internal_strlen(addrstr))) { - Printf("ThreadSanitizer: can't write from symbolizer (%d, %d)\n", - m->out_fd, errno); - Die(); - } - InternalScopedBuffer<char> func(1024); - ssize_t len = internal_read(m->inp_fd, func.data(), func.size() - 1); - if (len <= 0) { - Printf("ThreadSanitizer: can't read from symbolizer (%d, %d)\n", - m->inp_fd, errno); - Die(); - } - func.data()[len] = 0; - ReportStack *res = NewReportStackEntry(addr); - res->module = internal_strdup(m->name); - res->offset = offset; - char *pos = (char*)internal_strchr(func.data(), '\n'); - if (pos && func[0] != '?') { - res->func = (char*)internal_alloc(MBlockReportStack, pos - func.data() + 1); - internal_memcpy(res->func, func.data(), pos - func.data()); - res->func[pos - func.data()] = 0; - char *pos2 = (char*)internal_strchr(pos, ':'); - if (pos2) { - res->file = (char*)internal_alloc(MBlockReportStack, pos2 - pos - 1 + 1); - internal_memcpy(res->file, pos + 1, pos2 - pos - 1); - res->file[pos2 - pos - 1] = 0; - res->line = atoi(pos2 + 1); - } - } - return res; -} - -ReportStack *SymbolizeDataAddr2Line(uptr addr) { - return 0; -} - -} // namespace __tsan diff --git a/libsanitizer/tsan/tsan_sync.cc b/libsanitizer/tsan/tsan_sync.cc index 0c5be10..f6f2cb7 100644 --- a/libsanitizer/tsan/tsan_sync.cc +++ b/libsanitizer/tsan/tsan_sync.cc @@ -15,10 +15,13 @@ namespace __tsan { +void DDMutexInit(ThreadState *thr, uptr pc, SyncVar *s); + SyncVar::SyncVar(uptr addr, u64 uid) : mtx(MutexTypeSyncVar, StatMtxSyncVar) , addr(addr) , uid(uid) + , creation_stack_id() , owner_tid(kInvalidTid) , last_lock() , recursion() @@ -60,9 +63,11 @@ SyncVar* SyncTab::Create(ThreadState *thr, uptr pc, uptr addr) { void *mem = internal_alloc(MBlockSync, sizeof(SyncVar)); const u64 uid = atomic_fetch_add(&uid_gen_, 1, memory_order_relaxed); SyncVar *res = new(mem) SyncVar(addr, uid); -#ifndef TSAN_GO - res->creation_stack_id = CurrentStackId(thr, pc); -#endif + res->creation_stack_id = 0; + if (!kGoMode) // Go does not use them + res->creation_stack_id = CurrentStackId(thr, pc); + if (flags()->detect_deadlocks) + DDMutexInit(thr, pc, res); return res; } diff --git a/libsanitizer/tsan/tsan_sync.h b/libsanitizer/tsan/tsan_sync.h index 2867a8a..3838df9 100644 --- a/libsanitizer/tsan/tsan_sync.h +++ b/libsanitizer/tsan/tsan_sync.h @@ -13,14 +13,13 @@ #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_common.h" +#include "sanitizer_common/sanitizer_deadlock_detector_interface.h" #include "tsan_clock.h" #include "tsan_defs.h" #include "tsan_mutex.h" namespace __tsan { -class SlabCache; - class StackTrace { public: StackTrace(); @@ -55,8 +54,6 @@ struct SyncVar { Mutex mtx; uptr addr; const u64 uid; // Globally unique id. - SyncClock clock; - SyncClock read_clock; // Used for rw mutexes only. u32 creation_stack_id; int owner_tid; // Set only by exclusive owners. u64 last_lock; @@ -66,8 +63,12 @@ struct SyncVar { bool is_broken; bool is_linker_init; SyncVar *next; // In SyncTab hashtable. + DDMutex dd; + SyncClock read_clock; // Used for rw mutexes only. + // The clock is placed last, so that it is situated on a different cache line + // with the mtx. This reduces contention for hot sync objects. + SyncClock clock; - uptr GetMemoryConsumption(); u64 GetId() const { // 47 lsb is addr, then 14 bits is low part of uid, then 3 zero bits. return GetLsb((u64)addr | (uid << 47), 61); @@ -96,8 +97,6 @@ class SyncTab { SyncVar* Create(ThreadState *thr, uptr pc, uptr addr); - uptr GetMemoryConsumption(uptr *nsync); - private: struct Part { Mutex mtx; diff --git a/libsanitizer/tsan/tsan_vector.h b/libsanitizer/tsan/tsan_vector.h index 4da8b83..f65ad2b 100644 --- a/libsanitizer/tsan/tsan_vector.h +++ b/libsanitizer/tsan/tsan_vector.h @@ -56,10 +56,18 @@ class Vector { return begin_[i]; } - T *PushBack(T v = T()) { + T *PushBack() { EnsureSize(Size() + 1); - end_[-1] = v; - return &end_[-1]; + T *p = &end_[-1]; + internal_memset(p, 0, sizeof(*p)); + return p; + } + + T *PushBack(const T& v) { + EnsureSize(Size() + 1); + T *p = &end_[-1]; + internal_memcpy(p, &v, sizeof(*p)); + return p; } void PopBack() { @@ -72,7 +80,7 @@ class Vector { EnsureSize(size); if (old_size < size) { for (uptr i = old_size; i < size; i++) - begin_[i] = T(); + internal_memset(&begin_[i], 0, sizeof(begin_[i])); } } |