diff options
26 files changed, 404 insertions, 118 deletions
diff --git a/libc/config/config.json b/libc/config/config.json index d6ef891..d3d1ff1 100644 --- a/libc/config/config.json +++ b/libc/config/config.json @@ -40,5 +40,15 @@ "value": true, "doc": "Enable -fstack-protector-strong to defend against stack smashing attack." } + }, + "pthread": { + "LIBC_CONF_TIMEOUT_ENSURE_MONOTONICITY": { + "value": true, + "doc": "Automatically adjust timeout to CLOCK_MONOTONIC (default to true). POSIX API may require CLOCK_REALTIME, which can be unstable and leading to unexpected behavior. This option will convert the real-time timestamp to monotonic timestamp relative to the time of call." + }, + "LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT": { + "value": 100, + "doc": "Default number of spins before blocking if a mutex is in contention (default to 100)." + } } } diff --git a/libc/docs/configure.rst b/libc/docs/configure.rst index 8f8c44c..77ade07 100644 --- a/libc/docs/configure.rst +++ b/libc/docs/configure.rst @@ -34,6 +34,9 @@ to learn about the defaults for your platform and target. - ``LIBC_CONF_PRINTF_DISABLE_INDEX_MODE``: Disable index mode in the printf format string. - ``LIBC_CONF_PRINTF_DISABLE_WRITE_INT``: Disable handling of %n in printf format string. - ``LIBC_CONF_PRINTF_FLOAT_TO_STR_USE_MEGA_LONG_DOUBLE_TABLE``: Use large table for better printf long double performance. +* **"pthread" options** + - ``LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT``: Default number of spins before blocking if a mutex is in contention (default to 100). + - ``LIBC_CONF_TIMEOUT_ENSURE_MONOTONICITY``: Automatically adjust timeout to CLOCK_MONOTONIC (default to true). POSIX API may require CLOCK_REALTIME, which can be unstable and leading to unexpected behavior. This option will convert the real-time timestamp to monotonic timestamp relative to the time of call. * **"string" options** - ``LIBC_CONF_MEMSET_X86_USE_SOFTWARE_PREFETCHING``: Inserts prefetch for write instructions (PREFETCHW) for memset on x86 to recover performance when hardware prefetcher is disabled. - ``LIBC_CONF_STRING_UNSAFE_WIDE_READ``: Read more than a byte at a time to perform byte-string operations like strlen. diff --git a/libc/hdr/types/CMakeLists.txt b/libc/hdr/types/CMakeLists.txt index 3a1bb2f..8e87642 100644 --- a/libc/hdr/types/CMakeLists.txt +++ b/libc/hdr/types/CMakeLists.txt @@ -108,3 +108,12 @@ add_proxy_header_library( libc.include.llvm-libc-types.struct_timeval libc.include.sys_time ) + +add_proxy_header_library( + pid_t + HDRS + pid_t.h + FULL_BUILD_DEPENDS + libc.include.llvm-libc-types.pid_t + libc.include.sys_types +) diff --git a/libc/hdr/types/pid_t.h b/libc/hdr/types/pid_t.h new file mode 100644 index 0000000..34306fc --- /dev/null +++ b/libc/hdr/types/pid_t.h @@ -0,0 +1,22 @@ +//===-- Proxy for pid_t ---------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_HDR_TYPES_PID_T_H +#define LLVM_LIBC_HDR_TYPES_PID_T_H + +#ifdef LIBC_FULL_BUILD + +#include "include/llvm-libc-types/pid_t.h" + +#else // Overlay mode + +#include <sys/types.h> + +#endif // LLVM_LIBC_FULL_BUILD + +#endif // LLVM_LIBC_HDR_TYPES_PID_T_H diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt index 018b6c5..ee2c910 100644 --- a/libc/include/llvm-libc-types/CMakeLists.txt +++ b/libc/include/llvm-libc-types/CMakeLists.txt @@ -7,7 +7,8 @@ add_header(__call_once_func_t HDR __call_once_func_t.h) add_header(__exec_argv_t HDR __exec_argv_t.h) add_header(__exec_envp_t HDR __exec_envp_t.h) add_header(__futex_word HDR __futex_word.h) -add_header(__mutex_type HDR __mutex_type.h DEPENDS .__futex_word) +add_header(pid_t HDR pid_t.h) +add_header(__mutex_type HDR __mutex_type.h DEPENDS .__futex_word .pid_t) add_header(__pthread_once_func_t HDR __pthread_once_func_t.h) add_header(__pthread_start_t HDR __pthread_start_t.h) add_header(__pthread_tss_dtor_t HDR __pthread_tss_dtor_t.h) @@ -20,7 +21,7 @@ add_header(blksize_t HDR blksize_t.h) add_header(cc_t HDR cc_t.h) add_header(clock_t HDR clock_t.h) add_header(clockid_t HDR clockid_t.h) -add_header(cnd_t HDR cnd_t.h) +add_header(cnd_t HDR cnd_t.h DEPENDS .__futex_word) add_header(cookie_io_functions_t HDR cookie_io_functions_t.h DEPENDS .off64_t .ssize_t) add_header(cpu_set_t HDR cpu_set_t.h) add_header(double_t HDR double_t.h) @@ -45,7 +46,6 @@ add_header(mtx_t HDR mtx_t.h DEPENDS .__futex_word .__mutex_type) add_header(nlink_t HDR nlink_t.h) add_header(off_t HDR off_t.h) add_header(once_flag HDR once_flag.h DEPENDS .__futex_word) -add_header(pid_t HDR pid_t.h) add_header(posix_spawn_file_actions_t HDR posix_spawn_file_actions_t.h) add_header(posix_spawnattr_t HDR posix_spawnattr_t.h) add_header(pthread_attr_t HDR pthread_attr_t.h DEPENDS .size_t) diff --git a/libc/include/llvm-libc-types/cnd_t.h b/libc/include/llvm-libc-types/cnd_t.h index 1159ac4..266dfbb 100644 --- a/libc/include/llvm-libc-types/cnd_t.h +++ b/libc/include/llvm-libc-types/cnd_t.h @@ -9,12 +9,12 @@ #ifndef LLVM_LIBC_TYPES_CND_T_H #define LLVM_LIBC_TYPES_CND_T_H -#include "mtx_t.h" +#include "llvm-libc-types/__futex_word.h" typedef struct { void *__qfront; void *__qback; - mtx_t __qmtx; + __futex_word __qmtx; } cnd_t; #endif // LLVM_LIBC_TYPES_CND_T_H diff --git a/libc/src/__support/File/dir.h b/libc/src/__support/File/dir.h index a2f50f8..9f8c8db 100644 --- a/libc/src/__support/File/dir.h +++ b/libc/src/__support/File/dir.h @@ -50,14 +50,16 @@ class Dir { // A directory is to be opened by the static method open and closed // by the close method. So, all constructors and destructor are declared // as private. Inappropriate constructors are declared as deleted. - Dir() = delete; - Dir(const Dir &) = delete; + LIBC_INLINE Dir() = delete; + LIBC_INLINE Dir(const Dir &) = delete; - explicit Dir(int fdesc) - : fd(fdesc), readptr(0), fillsize(0), mutex(false, false, false) {} - ~Dir() = default; + LIBC_INLINE explicit Dir(int fdesc) + : fd(fdesc), readptr(0), fillsize(0), + mutex(/*timed=*/false, /*recursive=*/false, /*robust=*/false, + /*pshared=*/false) {} + LIBC_INLINE ~Dir() = default; - Dir &operator=(const Dir &) = delete; + LIBC_INLINE Dir &operator=(const Dir &) = delete; public: static ErrorOr<Dir *> open(const char *path); @@ -69,7 +71,7 @@ public: // cleaned up. int close(); - int getfd() { return fd; } + LIBC_INLINE int getfd() { return fd; } }; } // namespace LIBC_NAMESPACE diff --git a/libc/src/__support/File/file.h b/libc/src/__support/File/file.h index eafd3ab..0615487 100644 --- a/libc/src/__support/File/file.h +++ b/libc/src/__support/File/file.h @@ -154,10 +154,11 @@ public: uint8_t *buffer, size_t buffer_size, int buffer_mode, bool owned, ModeFlags modeflags) : platform_write(wf), platform_read(rf), platform_seek(sf), - platform_close(cf), mutex(false, false, false), ungetc_buf(0), - buf(buffer), bufsize(buffer_size), bufmode(buffer_mode), own_buf(owned), - mode(modeflags), pos(0), prev_op(FileOp::NONE), read_limit(0), - eof(false), err(false) { + platform_close(cf), mutex(/*timed=*/false, /*recursive=*/false, + /*robust=*/false, /*pshared=*/false), + ungetc_buf(0), buf(buffer), bufsize(buffer_size), bufmode(buffer_mode), + own_buf(owned), mode(modeflags), pos(0), prev_op(FileOp::NONE), + read_limit(0), eof(false), err(false) { adjust_buf(); } diff --git a/libc/src/__support/threads/CndVar.h b/libc/src/__support/threads/CndVar.h index baa2a68..55f6e02 100644 --- a/libc/src/__support/threads/CndVar.h +++ b/libc/src/__support/threads/CndVar.h @@ -10,13 +10,14 @@ #define LLVM_LIBC___SUPPORT_SRC_THREADS_LINUX_CNDVAR_H #include "src/__support/threads/linux/futex_utils.h" // Futex +#include "src/__support/threads/linux/raw_mutex.h" // RawMutex #include "src/__support/threads/mutex.h" // Mutex #include <stdint.h> // uint32_t namespace LIBC_NAMESPACE { -struct CndVar { +class CndVar { enum CndWaiterStatus : uint32_t { WS_Waiting = 0xE, WS_Signalled = 0x5, @@ -29,15 +30,16 @@ struct CndVar { CndWaiter *waitq_front; CndWaiter *waitq_back; - Mutex qmtx; + RawMutex qmtx; - static int init(CndVar *cv) { +public: + LIBC_INLINE static int init(CndVar *cv) { cv->waitq_front = cv->waitq_back = nullptr; - auto err = Mutex::init(&cv->qmtx, false, false, false); - return err == MutexError::NONE ? 0 : -1; + RawMutex::init(&cv->qmtx); + return 0; } - static void destroy(CndVar *cv) { + LIBC_INLINE static void destroy(CndVar *cv) { cv->waitq_front = cv->waitq_back = nullptr; } diff --git a/libc/src/__support/threads/fork_callbacks.cpp b/libc/src/__support/threads/fork_callbacks.cpp index 6efaf62..385a2bc 100644 --- a/libc/src/__support/threads/fork_callbacks.cpp +++ b/libc/src/__support/threads/fork_callbacks.cpp @@ -33,7 +33,10 @@ class AtForkCallbackManager { size_t next_index; public: - constexpr AtForkCallbackManager() : mtx(false, false, false), next_index(0) {} + constexpr AtForkCallbackManager() + : mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false, + /*pshared=*/false), + next_index(0) {} bool register_triple(const ForkCallbackTriple &triple) { cpp::lock_guard lock(mtx); diff --git a/libc/src/__support/threads/gpu/mutex.h b/libc/src/__support/threads/gpu/mutex.h index 71d0ef0..9554083 100644 --- a/libc/src/__support/threads/gpu/mutex.h +++ b/libc/src/__support/threads/gpu/mutex.h @@ -19,7 +19,7 @@ namespace LIBC_NAMESPACE { /// define the Mutex interface and require that only a single thread executes /// code requiring a mutex lock. struct Mutex { - LIBC_INLINE constexpr Mutex(bool, bool, bool) {} + LIBC_INLINE constexpr Mutex(bool, bool, bool, bool) {} LIBC_INLINE MutexError lock() { return MutexError::NONE; } LIBC_INLINE MutexError unlock() { return MutexError::NONE; } diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt index f6913ef..9bf88cc 100644 --- a/libc/src/__support/threads/linux/CMakeLists.txt +++ b/libc/src/__support/threads/linux/CMakeLists.txt @@ -22,12 +22,37 @@ add_header_library( libc.src.__support.time.linux.abs_timeout ) +set(raw_mutex_additional_flags) +if (LIBC_CONF_TIMEOUT_ENSURE_MONOTONICITY) + set(raw_mutex_additional_flags -DLIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY=1) +else() + set(raw_mutex_additional_flags -DLIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY=0) +endif() + +add_header_library( + raw_mutex + HDRS + mutex.h + DEPENDS + .futex_utils + libc.src.__support.threads.sleep + libc.src.__support.time.linux.abs_timeout + libc.src.__support.time.linux.monotonicity + libc.src.__support.CPP.optional + libc.hdr.types.pid_t + COMPILE_OPTIONS + -DLIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT=${LIBC_CONF_RAW_MUTEX_DEFAULT_SPIN_COUNT} + ${raw_mutex_additional_flags} + +) + add_header_library( mutex HDRS mutex.h DEPENDS .futex_utils + .raw_mutex libc.src.__support.threads.mutex_common ) @@ -75,5 +100,6 @@ add_object_library( libc.src.__support.OSUtil.osutil libc.src.__support.threads.linux.futex_word_type libc.src.__support.threads.mutex + libc.src.__support.threads.linux.raw_mutex libc.src.__support.CPP.mutex ) diff --git a/libc/src/__support/threads/linux/CndVar.cpp b/libc/src/__support/threads/linux/CndVar.cpp index b3a0fdb..0312532 100644 --- a/libc/src/__support/threads/linux/CndVar.cpp +++ b/libc/src/__support/threads/linux/CndVar.cpp @@ -10,6 +10,7 @@ #include "src/__support/CPP/mutex.h" #include "src/__support/OSUtil/syscall.h" // syscall_impl #include "src/__support/threads/linux/futex_word.h" // FutexWordType +#include "src/__support/threads/linux/raw_mutex.h" // RawMutex #include "src/__support/threads/mutex.h" // Mutex #include <sys/syscall.h> // For syscall numbers. @@ -74,11 +75,11 @@ void CndVar::notify_one() { if (waitq_front == nullptr) waitq_back = nullptr; - qmtx.futex_word = FutexWordType(Mutex::LockState::Free); + qmtx.reset(); // this is a special WAKE_OP, so we use syscall directly LIBC_NAMESPACE::syscall_impl<long>( - FUTEX_SYSCALL_ID, &qmtx.futex_word.val, FUTEX_WAKE_OP, 1, 1, + FUTEX_SYSCALL_ID, &qmtx.get_raw_futex(), FUTEX_WAKE_OP, 1, 1, &first->futex_word.val, FUTEX_OP(FUTEX_OP_SET, WS_Signalled, FUTEX_OP_CMP_EQ, WS_Waiting)); } diff --git a/libc/src/__support/threads/linux/mutex.h b/libc/src/__support/threads/linux/mutex.h index 6702de4..79506d7 100644 --- a/libc/src/__support/threads/linux/mutex.h +++ b/libc/src/__support/threads/linux/mutex.h @@ -9,114 +9,79 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H #define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_MUTEX_H +#include "hdr/types/pid_t.h" +#include "src/__support/CPP/optional.h" +#include "src/__support/libc_assert.h" #include "src/__support/threads/linux/futex_utils.h" +#include "src/__support/threads/linux/raw_mutex.h" #include "src/__support/threads/mutex_common.h" namespace LIBC_NAMESPACE { -struct Mutex { + +// TODO: support shared/recursive/robust mutexes. +class Mutex final : private RawMutex { + // reserved timed, may be useful when combined with other flags. unsigned char timed; unsigned char recursive; unsigned char robust; + unsigned char pshared; - void *owner; + // TLS address may not work across forked processes. Use thread id instead. + pid_t owner; unsigned long long lock_count; - Futex futex_word; - - enum class LockState : FutexWordType { - Free, - Locked, - Waiting, - }; - public: - constexpr Mutex(bool istimed, bool isrecursive, bool isrobust) - : timed(istimed), recursive(isrecursive), robust(isrobust), - owner(nullptr), lock_count(0), - futex_word(FutexWordType(LockState::Free)) {} - - static MutexError init(Mutex *mutex, bool istimed, bool isrecur, - bool isrobust) { - mutex->timed = istimed; + LIBC_INLINE constexpr Mutex(bool is_timed, bool is_recursive, bool is_robust, + bool is_pshared) + : RawMutex(), timed(is_timed), recursive(is_recursive), robust(is_robust), + pshared(is_pshared), owner(0), lock_count(0) {} + + LIBC_INLINE static MutexError init(Mutex *mutex, bool is_timed, bool isrecur, + bool isrobust, bool is_pshared) { + RawMutex::init(mutex); + mutex->timed = is_timed; mutex->recursive = isrecur; mutex->robust = isrobust; - mutex->owner = nullptr; + mutex->pshared = is_pshared; + mutex->owner = 0; mutex->lock_count = 0; - mutex->futex_word.set(FutexWordType(LockState::Free)); return MutexError::NONE; } - static MutexError destroy(Mutex *) { return MutexError::NONE; } - - MutexError reset(); - - MutexError lock() { - bool was_waiting = false; - while (true) { - FutexWordType mutex_status = FutexWordType(LockState::Free); - FutexWordType locked_status = FutexWordType(LockState::Locked); - - if (futex_word.compare_exchange_strong( - mutex_status, FutexWordType(LockState::Locked))) { - if (was_waiting) - futex_word = FutexWordType(LockState::Waiting); - return MutexError::NONE; - } + LIBC_INLINE static MutexError destroy(Mutex *lock) { + LIBC_ASSERT(lock->owner == 0 && lock->lock_count == 0 && + "Mutex destroyed while being locked."); + RawMutex::destroy(lock); + return MutexError::NONE; + } - switch (LockState(mutex_status)) { - case LockState::Waiting: - // If other threads are waiting already, then join them. Note that the - // futex syscall will block if the futex data is still - // `LockState::Waiting` (the 4th argument to the syscall function - // below.) - futex_word.wait(FutexWordType(LockState::Waiting)); - was_waiting = true; - // Once woken up/unblocked, try everything all over. - continue; - case LockState::Locked: - // Mutex has been locked by another thread so set the status to - // LockState::Waiting. - if (futex_word.compare_exchange_strong( - locked_status, FutexWordType(LockState::Waiting))) { - // If we are able to set the futex data to `LockState::Waiting`, then - // we will wait for the futex to be woken up. Note again that the - // following syscall will block only if the futex data is still - // `LockState::Waiting`. - futex_word.wait(FutexWordType(LockState::Waiting)); - was_waiting = true; - } - continue; - case LockState::Free: - // If it was LockState::Free, we shouldn't be here at all. - return MutexError::BAD_LOCK_STATE; - } - } + // TODO: record owner and lock count. + LIBC_INLINE MutexError lock() { + // Since timeout is not specified, we do not need to check the return value. + this->RawMutex::lock( + /* timeout=*/cpp::nullopt, this->pshared); + return MutexError::NONE; } - MutexError unlock() { - while (true) { - FutexWordType mutex_status = FutexWordType(LockState::Waiting); - if (futex_word.compare_exchange_strong(mutex_status, - FutexWordType(LockState::Free))) { - // If any thread is waiting to be woken up, then do it. - futex_word.notify_one(); - return MutexError::NONE; - } + // TODO: record owner and lock count. + LIBC_INLINE MutexError timed_lock(internal::AbsTimeout abs_time) { + if (this->RawMutex::lock(abs_time, this->pshared)) + return MutexError::NONE; + return MutexError::TIMEOUT; + } - if (mutex_status == FutexWordType(LockState::Locked)) { - // If nobody was waiting at this point, just free it. - if (futex_word.compare_exchange_strong(mutex_status, - FutexWordType(LockState::Free))) - return MutexError::NONE; - } else { - // This can happen, for example if some thread tries to unlock an - // already free mutex. - return MutexError::UNLOCK_WITHOUT_LOCK; - } - } + LIBC_INLINE MutexError unlock() { + if (this->RawMutex::unlock(this->pshared)) + return MutexError::NONE; + return MutexError::UNLOCK_WITHOUT_LOCK; } - MutexError trylock(); + // TODO: record owner and lock count. + LIBC_INLINE MutexError try_lock() { + if (this->RawMutex::try_lock()) + return MutexError::NONE; + return MutexError::BUSY; + } }; } // namespace LIBC_NAMESPACE diff --git a/libc/src/__support/threads/linux/raw_mutex.h b/libc/src/__support/threads/linux/raw_mutex.h new file mode 100644 index 0000000..75bd4bd --- /dev/null +++ b/libc/src/__support/threads/linux/raw_mutex.h @@ -0,0 +1,130 @@ +//===--- Implementation of a Linux RawMutex class ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H +#define LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H + +#include "src/__support/CPP/optional.h" +#include "src/__support/common.h" +#include "src/__support/libc_assert.h" +#include "src/__support/macros/attributes.h" +#include "src/__support/threads/linux/futex_utils.h" +#include "src/__support/threads/linux/futex_word.h" +#include "src/__support/threads/sleep.h" +#include "src/__support/time/linux/abs_timeout.h" + +#ifndef LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY +#define LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY 1 +#warning "LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY is not defined, defaulting to 1" +#endif + +#if LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY +#include "src/__support/time/linux/monotonicity.h" +#endif + +#ifndef LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT +#define LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT 100 +#warning \ + "LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT is not defined, defaulting to 100" +#endif + +namespace LIBC_NAMESPACE { +// Lock is a simple timable lock for internal usage. +// This is separated from Mutex because this one does not need to consider +// robustness and reentrancy. Also, this one has spin optimization for shorter +// critical sections. +class RawMutex { +protected: + Futex futex; + LIBC_INLINE_VAR static constexpr FutexWordType UNLOCKED = 0b00; + LIBC_INLINE_VAR static constexpr FutexWordType LOCKED = 0b01; + LIBC_INLINE_VAR static constexpr FutexWordType IN_CONTENTION = 0b10; + +private: + LIBC_INLINE FutexWordType spin(unsigned spin_count) { + FutexWordType result; + for (;;) { + result = futex.load(cpp::MemoryOrder::RELAXED); + // spin until one of the following conditions is met: + // - the mutex is unlocked + // - the mutex is in contention + // - the spin count reaches 0 + if (result != LOCKED || spin_count == 0u) + return result; + // Pause the pipeline to avoid extraneous memory operations due to + // speculation. + sleep_briefly(); + spin_count--; + }; + } + + // Return true if the lock is acquired. Return false if timeout happens before + // the lock is acquired. + LIBC_INLINE bool lock_slow(cpp::optional<Futex::Timeout> timeout, + bool is_pshared, unsigned spin_count) { + FutexWordType state = spin(spin_count); + // Before go into contention state, try to grab the lock. + if (state == UNLOCKED && + futex.compare_exchange_strong(state, LOCKED, cpp::MemoryOrder::ACQUIRE, + cpp::MemoryOrder::RELAXED)) + return true; +#if LIBC_COPT_TIMEOUT_ENSURE_MONOTONICITY + /* ADL should kick in */ + if (timeout) + ensure_monotonicity(*timeout); +#endif + for (;;) { + // Try to grab the lock if it is unlocked. Mark the contention flag if it + // is locked. + if (state != IN_CONTENTION && + futex.exchange(IN_CONTENTION, cpp::MemoryOrder::ACQUIRE) == UNLOCKED) + return true; + // Contention persists. Park the thread and wait for further notification. + if (ETIMEDOUT == -futex.wait(IN_CONTENTION, timeout, is_pshared)) + return false; + // Continue to spin after waking up. + state = spin(spin_count); + } + } + + LIBC_INLINE void wake(bool is_pshared) { futex.notify_one(is_pshared); } + +public: + LIBC_INLINE static void init(RawMutex *mutex) { mutex->futex = UNLOCKED; } + LIBC_INLINE constexpr RawMutex() : futex(UNLOCKED) {} + [[nodiscard]] LIBC_INLINE bool try_lock() { + FutexWordType expected = UNLOCKED; + // Use strong version since this is a one-time operation. + return futex.compare_exchange_strong( + expected, LOCKED, cpp::MemoryOrder::ACQUIRE, cpp::MemoryOrder::RELAXED); + } + LIBC_INLINE bool + lock(cpp::optional<Futex::Timeout> timeout = cpp::nullopt, + bool is_shared = false, + unsigned spin_count = LIBC_COPT_RAW_MUTEX_DEFAULT_SPIN_COUNT) { + // Timeout will not be checked if immediate lock is possible. + if (LIBC_LIKELY(try_lock())) + return true; + return lock_slow(timeout, is_shared, spin_count); + } + LIBC_INLINE bool unlock(bool is_pshared = false) { + FutexWordType prev = futex.exchange(UNLOCKED, cpp::MemoryOrder::RELEASE); + // if there is someone waiting, wake them up + if (LIBC_UNLIKELY(prev == IN_CONTENTION)) + wake(is_pshared); + // Detect invalid unlock operation. + return prev != UNLOCKED; + } + LIBC_INLINE void static destroy([[maybe_unused]] RawMutex *lock) { + LIBC_ASSERT(lock->futex == UNLOCKED && "Mutex destroyed while used."); + } + LIBC_INLINE Futex &get_raw_futex() { return futex; } + LIBC_INLINE void reset() { futex = UNLOCKED; } +}; +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC___SUPPORT_THREADS_LINUX_RAW_MUTEX_H diff --git a/libc/src/__support/threads/thread.cpp b/libc/src/__support/threads/thread.cpp index 7b02f82..d7bedb8 100644 --- a/libc/src/__support/threads/thread.cpp +++ b/libc/src/__support/threads/thread.cpp @@ -54,7 +54,9 @@ class TSSKeyMgr { cpp::array<TSSKeyUnit, TSS_KEY_COUNT> units; public: - constexpr TSSKeyMgr() : mtx(false, false, false) {} + constexpr TSSKeyMgr() + : mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false, + /*pshared=*/false) {} cpp::optional<unsigned int> new_key(TSSDtor *dtor) { cpp::lock_guard lock(mtx); @@ -111,7 +113,9 @@ class ThreadAtExitCallbackMgr { FixedVector<AtExitUnit, 1024> callback_list; public: - constexpr ThreadAtExitCallbackMgr() : mtx(false, false, false) {} + constexpr ThreadAtExitCallbackMgr() + : mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false, + /*pshared=*/false) {} int add_callback(AtExitCallback *callback, void *obj) { cpp::lock_guard lock(mtx); diff --git a/libc/src/pthread/pthread_mutex_init.cpp b/libc/src/pthread/pthread_mutex_init.cpp index f64d1fe..3914669 100644 --- a/libc/src/pthread/pthread_mutex_init.cpp +++ b/libc/src/pthread/pthread_mutex_init.cpp @@ -26,9 +26,10 @@ LLVM_LIBC_FUNCTION(int, pthread_mutex_init, const pthread_mutexattr_t *__restrict attr)) { auto mutexattr = attr == nullptr ? DEFAULT_MUTEXATTR : *attr; auto err = - Mutex::init(reinterpret_cast<Mutex *>(m), false, + Mutex::init(reinterpret_cast<Mutex *>(m), /*is_timed=*/true, get_mutexattr_type(mutexattr) & PTHREAD_MUTEX_RECURSIVE, - get_mutexattr_robust(mutexattr) & PTHREAD_MUTEX_ROBUST); + get_mutexattr_robust(mutexattr) & PTHREAD_MUTEX_ROBUST, + get_mutexattr_pshared(mutexattr) & PTHREAD_PROCESS_SHARED); return err == MutexError::NONE ? 0 : EAGAIN; } diff --git a/libc/src/pthread/pthread_mutexattr.h b/libc/src/pthread/pthread_mutexattr.h index 8b43585..292ceeb 100644 --- a/libc/src/pthread/pthread_mutexattr.h +++ b/libc/src/pthread/pthread_mutexattr.h @@ -43,6 +43,11 @@ LIBC_INLINE int get_mutexattr_robust(pthread_mutexattr_t attr) { unsigned(PThreadMutexAttrPos::ROBUST_SHIFT); } +LIBC_INLINE int get_mutexattr_pshared(pthread_mutexattr_t attr) { + return (attr & unsigned(PThreadMutexAttrPos::PSHARED_MASK)) >> + unsigned(PThreadMutexAttrPos::PSHARED_SHIFT); +} + } // namespace LIBC_NAMESPACE #endif // LLVM_LIBC_SRC_PTHREAD_PTHREAD_MUTEXATTR_H diff --git a/libc/src/stdlib/atexit.cpp b/libc/src/stdlib/atexit.cpp index 4f04974..9e37c4c 100644 --- a/libc/src/stdlib/atexit.cpp +++ b/libc/src/stdlib/atexit.cpp @@ -17,7 +17,8 @@ namespace LIBC_NAMESPACE { namespace { -Mutex handler_list_mtx(false, false, false); +Mutex handler_list_mtx(/*timed=*/false, /*recursive=*/false, /*robust=*/false, + /*pshared=*/false); using AtExitCallback = void(void *); using StdCAtExitCallback = void(void); diff --git a/libc/src/threads/linux/CMakeLists.txt b/libc/src/threads/linux/CMakeLists.txt index a5a02e4..6c8e084 100644 --- a/libc/src/threads/linux/CMakeLists.txt +++ b/libc/src/threads/linux/CMakeLists.txt @@ -9,6 +9,7 @@ add_header_library( libc.src.__support.CPP.mutex libc.src.__support.OSUtil.osutil libc.src.__support.threads.mutex + libc.src.__support.threads.linux.raw_mutex libc.src.__support.threads.linux.futex_utils ) diff --git a/libc/src/threads/mtx_init.cpp b/libc/src/threads/mtx_init.cpp index 74d08d3..7cd848d 100644 --- a/libc/src/threads/mtx_init.cpp +++ b/libc/src/threads/mtx_init.cpp @@ -20,7 +20,8 @@ static_assert(sizeof(Mutex) <= sizeof(mtx_t), LLVM_LIBC_FUNCTION(int, mtx_init, (mtx_t * m, int type)) { auto err = Mutex::init(reinterpret_cast<Mutex *>(m), type & mtx_timed, - type & mtx_recursive, 0); + type & mtx_recursive, /* is_robust */ false, + /* is_pshared */ false); return err == MutexError::NONE ? thrd_success : thrd_error; } diff --git a/libc/test/integration/src/__support/threads/thread_detach_test.cpp b/libc/test/integration/src/__support/threads/thread_detach_test.cpp index 697d991..995f8b0 100644 --- a/libc/test/integration/src/__support/threads/thread_detach_test.cpp +++ b/libc/test/integration/src/__support/threads/thread_detach_test.cpp @@ -10,7 +10,8 @@ #include "src/__support/threads/thread.h" #include "test/IntegrationTest/test.h" -LIBC_NAMESPACE::Mutex mutex(false, false, false); +LIBC_NAMESPACE::Mutex mutex(/*timed=*/false, /*recursive=*/false, + /*robust=*/false, /*pshared=*/false); int func(void *) { mutex.lock(); diff --git a/libc/test/src/__support/CMakeLists.txt b/libc/test/src/__support/CMakeLists.txt index 8bdc56e..663aa2b 100644 --- a/libc/test/src/__support/CMakeLists.txt +++ b/libc/test/src/__support/CMakeLists.txt @@ -207,3 +207,4 @@ add_subdirectory(FPUtil) add_subdirectory(fixed_point) add_subdirectory(HashTable) add_subdirectory(time) +add_subdirectory(threads) diff --git a/libc/test/src/__support/threads/CMakeLists.txt b/libc/test/src/__support/threads/CMakeLists.txt new file mode 100644 index 0000000..70d68ab --- /dev/null +++ b/libc/test/src/__support/threads/CMakeLists.txt @@ -0,0 +1,4 @@ +add_custom_target(libc-support-threads-tests) +if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${LIBC_TARGET_OS}) + add_subdirectory(${LIBC_TARGET_OS}) +endif() diff --git a/libc/test/src/__support/threads/linux/CMakeLists.txt b/libc/test/src/__support/threads/linux/CMakeLists.txt new file mode 100644 index 0000000..ce9becb --- /dev/null +++ b/libc/test/src/__support/threads/linux/CMakeLists.txt @@ -0,0 +1,12 @@ +add_libc_test( + raw_mutex_test + SUITE + libc-support-threads-tests + SRCS + raw_mutex_test.cpp + DEPENDS + libc.src.__support.threads.linux.raw_mutex + libc.src.sys.mman.mmap + libc.src.sys.mman.munmap + libc.src.stdlib.exit +) diff --git a/libc/test/src/__support/threads/linux/raw_mutex_test.cpp b/libc/test/src/__support/threads/linux/raw_mutex_test.cpp new file mode 100644 index 0000000..814f6f1 --- /dev/null +++ b/libc/test/src/__support/threads/linux/raw_mutex_test.cpp @@ -0,0 +1,81 @@ +//===-- Unittests for Linux's RawMutex ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "include/llvm-libc-macros/linux/time-macros.h" +#include "src/__support/CPP/atomic.h" +#include "src/__support/OSUtil/syscall.h" +#include "src/__support/threads/linux/raw_mutex.h" +#include "src/__support/threads/sleep.h" +#include "src/__support/time/linux/clock_gettime.h" +#include "src/stdlib/exit.h" +#include "src/sys/mman/mmap.h" +#include "src/sys/mman/munmap.h" +#include "test/UnitTest/Test.h" +#include <sys/syscall.h> + +TEST(LlvmLibcSupportThreadsRawMutexTest, SmokeTest) { + LIBC_NAMESPACE::RawMutex mutex; + ASSERT_TRUE(mutex.lock()); + ASSERT_TRUE(mutex.unlock()); + ASSERT_TRUE(mutex.try_lock()); + ASSERT_FALSE(mutex.try_lock()); + ASSERT_TRUE(mutex.unlock()); + ASSERT_FALSE(mutex.unlock()); +} + +TEST(LlvmLibcSupportThreadsRawMutexTest, Timeout) { + LIBC_NAMESPACE::RawMutex mutex; + ASSERT_TRUE(mutex.lock()); + timespec ts; + LIBC_NAMESPACE::internal::clock_gettime(CLOCK_MONOTONIC, &ts); + ts.tv_sec += 1; + // Timeout will be respected when deadlock happens. + auto timeout = LIBC_NAMESPACE::internal::AbsTimeout::from_timespec(ts, false); + ASSERT_TRUE(timeout.has_value()); + // The following will timeout + ASSERT_FALSE(mutex.lock(*timeout)); + ASSERT_TRUE(mutex.unlock()); + // Test that the mutex works after the timeout. + ASSERT_TRUE(mutex.lock()); + ASSERT_TRUE(mutex.unlock()); + // If a lock can be acquired directly, expired timeout will not count. + // Notice that the timeout is already reached during preivous deadlock. + ASSERT_TRUE(mutex.lock(*timeout)); + ASSERT_TRUE(mutex.unlock()); +} + +TEST(LlvmLibcSupportThreadsRawMutexTest, PSharedLock) { + struct SharedData { + LIBC_NAMESPACE::RawMutex mutex; + LIBC_NAMESPACE::cpp::Atomic<size_t> finished; + int data; + }; + void *addr = + LIBC_NAMESPACE::mmap(nullptr, sizeof(SharedData), PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_SHARED, -1, 0); + ASSERT_NE(addr, MAP_FAILED); + auto *shared = reinterpret_cast<SharedData *>(addr); + shared->data = 0; + LIBC_NAMESPACE::RawMutex::init(&shared->mutex); + // Avoid pull in our own implementation of pthread_t. + long pid = LIBC_NAMESPACE::syscall_impl<long>(SYS_fork); + for (int i = 0; i < 10000; ++i) { + shared->mutex.lock(LIBC_NAMESPACE::cpp::nullopt, true); + shared->data++; + shared->mutex.unlock(true); + } + // Mark the thread as finished. + shared->finished.fetch_add(1); + // let the child exit early to avoid output pollution + if (pid == 0) + LIBC_NAMESPACE::exit(0); + while (shared->finished.load() != 2) + LIBC_NAMESPACE::sleep_briefly(); + ASSERT_EQ(shared->data, 20000); + LIBC_NAMESPACE::munmap(addr, sizeof(SharedData)); +} |