diff options
Diffstat (limited to 'libc/src')
24 files changed, 539 insertions, 321 deletions
diff --git a/libc/src/__support/File/linux/lseekImpl.h b/libc/src/__support/File/linux/lseekImpl.h index c22a6c5..47df99a 100644 --- a/libc/src/__support/File/linux/lseekImpl.h +++ b/libc/src/__support/File/linux/lseekImpl.h @@ -25,8 +25,9 @@ namespace internal { LIBC_INLINE ErrorOr<off_t> lseekimpl(int fd, off_t offset, int whence) { off_t result; #ifdef SYS_lseek - int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_lseek, fd, offset, whence); - result = ret; + result = LIBC_NAMESPACE::syscall_impl<off_t>(SYS_lseek, fd, offset, whence); + if (result < 0) + return Error(-static_cast<int>(result)); #elif defined(SYS_llseek) || defined(SYS__llseek) static_assert(sizeof(size_t) == 4, "size_t must be 32 bits."); #ifdef SYS_llseek @@ -37,11 +38,11 @@ LIBC_INLINE ErrorOr<off_t> lseekimpl(int fd, off_t offset, int whence) { off_t offset_64 = offset; int ret = LIBC_NAMESPACE::syscall_impl<int>( LLSEEK_SYSCALL_NO, fd, offset_64 >> 32, offset_64, &result, whence); + if (ret < 0) + return Error(-ret); #else #error "lseek, llseek and _llseek syscalls not available." #endif - if (ret < 0) - return Error(-ret); return result; } diff --git a/libc/src/__support/OSUtil/linux/CMakeLists.txt b/libc/src/__support/OSUtil/linux/CMakeLists.txt index f303e54..0a60377 100644 --- a/libc/src/__support/OSUtil/linux/CMakeLists.txt +++ b/libc/src/__support/OSUtil/linux/CMakeLists.txt @@ -25,6 +25,18 @@ add_object_library( ) add_header_library( + auxv + HDRS + auxv.h + DEPENDS + libc.hdr.fcntl_macros + libc.src.__support.OSUtil.osutil + libc.src.__support.common + libc.src.__support.CPP.optional + libc.src.__support.threads.callonce +) + +add_header_library( getrandom HDRS getrandom.h @@ -58,11 +70,10 @@ add_object_library( libc.src.__support.CPP.string_view libc.src.__support.threads.callonce libc.src.__support.threads.linux.futex_word_type + libc.src.__support.OSUtil.linux.auxv libc.hdr.types.struct_timeval libc.hdr.types.struct_timespec libc.hdr.types.clockid_t libc.hdr.types.time_t libc.hdr.link_macros - libc.src.errno.errno - libc.src.sys.auxv.getauxval ) diff --git a/libc/src/__support/OSUtil/linux/auxv.h b/libc/src/__support/OSUtil/linux/auxv.h new file mode 100644 index 0000000..894868a --- /dev/null +++ b/libc/src/__support/OSUtil/linux/auxv.h @@ -0,0 +1,158 @@ +//===------------- Linux AUXV Header --------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AUXV_H +#define LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AUXV_H + +#include "hdr/fcntl_macros.h" // For open flags +#include "src/__support/OSUtil/syscall.h" +#include "src/__support/common.h" +#include "src/__support/threads/callonce.h" + +#include <linux/auxvec.h> // For AT_ macros +#include <linux/mman.h> // For mmap flags +#include <linux/prctl.h> // For prctl +#include <sys/syscall.h> // For syscall numbers + +namespace LIBC_NAMESPACE_DECL { + +namespace auxv { +struct Entry { + unsigned long type; // Entry type + unsigned long val; // Integer value +}; + +class Vector { + LIBC_INLINE_VAR static constexpr Entry END = {AT_NULL, AT_NULL}; + LIBC_INLINE_VAR static const Entry *entries = &END; + LIBC_INLINE_VAR static CallOnceFlag init_flag = callonce_impl::NOT_CALLED; + LIBC_INLINE_VAR constexpr static size_t FALLBACK_AUXV_ENTRIES = 64; + + LIBC_INLINE static void fallback_initialize_unsync(); + LIBC_INLINE static const Entry *get_entries() { + if (LIBC_LIKELY(entries != &END)) + return entries; + callonce(&init_flag, fallback_initialize_unsync); + return entries; + } + +public: + class Iterator { + const Entry *current; + + public: + LIBC_INLINE explicit Iterator(const Entry *entry) : current(entry) {} + LIBC_INLINE Iterator &operator++() { + ++current; + return *this; + } + LIBC_INLINE const Entry &operator*() const { return *current; } + LIBC_INLINE bool operator!=(const Iterator &other) const { + return current->type != other.current->type; + } + LIBC_INLINE bool operator==(const Iterator &other) const { + return current->type == other.current->type; + } + }; + using iterator = Iterator; + LIBC_INLINE static Iterator begin() { return Iterator(get_entries()); } + LIBC_INLINE static Iterator end() { return Iterator(&END); } + LIBC_INLINE static void initialize_unsafe(const Entry *auxv); +}; + +// Initializes the auxv entries. +// This function is intended to be called once inside crt0. +LIBC_INLINE void Vector::initialize_unsafe(const Entry *auxv) { + init_flag = callonce_impl::FINISH; + entries = auxv; +} + +// When CRT0 does not setup the global array, this function is called. +// As its name suggests, this function is not thread-safe and should be +// backed by a callonce guard. +// This initialize routine will do a mmap to allocate a memory region. +// Since auxv tends to live throughout the program lifetime, we do not +// munmap it. +[[gnu::cold]] +LIBC_INLINE void Vector::fallback_initialize_unsync() { + constexpr size_t AUXV_MMAP_SIZE = FALLBACK_AUXV_ENTRIES * sizeof(Entry); +#ifdef SYS_mmap2 + constexpr int MMAP_SYSNO = SYS_mmap2; +#else + constexpr int MMAP_SYSNO = SYS_mmap; +#endif + long mmap_ret = syscall_impl<long>(MMAP_SYSNO, nullptr, AUXV_MMAP_SIZE, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + // We do not proceed if mmap fails. + if (mmap_ret <= 0) + return; + + // Initialize the auxv array with AT_NULL entries. + Entry *vector = reinterpret_cast<Entry *>(mmap_ret); + for (size_t i = 0; i < FALLBACK_AUXV_ENTRIES; ++i) { + vector[i].type = AT_NULL; + vector[i].val = AT_NULL; + } + size_t avaiable_size = AUXV_MMAP_SIZE - sizeof(Entry); + +// Attempt 1: use PRCTL to get the auxv. +// We guarantee that the vector is always padded with AT_NULL entries. +#ifdef PR_GET_AUXV + long prctl_ret = syscall_impl<long>(SYS_prctl, PR_GET_AUXV, + reinterpret_cast<unsigned long>(vector), + avaiable_size, 0, 0); + if (prctl_ret >= 0) { + entries = vector; + return; + } +#endif + + // Attempt 2: read /proc/self/auxv. +#ifdef SYS_openat + int fd = syscall_impl<int>(SYS_openat, AT_FDCWD, "/proc/self/auxv", + O_RDONLY | O_CLOEXEC); +#else + int fd = syscall_impl<int>(SYS_open, "/proc/self/auxv", O_RDONLY | O_CLOEXEC); +#endif + if (fd < 0) { + syscall_impl<long>(SYS_munmap, vector, AUXV_MMAP_SIZE); + return; + } + uint8_t *cursor = reinterpret_cast<uint8_t *>(vector); + bool has_error = false; + while (avaiable_size != 0) { + long bytes_read = syscall_impl<long>(SYS_read, fd, cursor, avaiable_size); + if (bytes_read <= 0) { + if (bytes_read == -EINTR) + continue; + has_error = bytes_read < 0; + break; + } + avaiable_size -= bytes_read; + cursor += bytes_read; + } + syscall_impl<long>(SYS_close, fd); + if (has_error) { + syscall_impl<long>(SYS_munmap, vector, AUXV_MMAP_SIZE); + return; + } + entries = vector; +} + +LIBC_INLINE cpp::optional<unsigned long> get(unsigned long type) { + Vector auxvec; + for (const auto &entry : auxvec) + if (entry.type == type) + return entry.val; + return cpp::nullopt; +} +} // namespace auxv +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_OSUTIL_LINUX_AUXV_H diff --git a/libc/src/__support/OSUtil/linux/vdso.cpp b/libc/src/__support/OSUtil/linux/vdso.cpp index e4e53c3..d6fd3f3 100644 --- a/libc/src/__support/OSUtil/linux/vdso.cpp +++ b/libc/src/__support/OSUtil/linux/vdso.cpp @@ -11,10 +11,9 @@ #include "src/__support/CPP/array.h" #include "src/__support/CPP/optional.h" #include "src/__support/CPP/string_view.h" -#include "src/__support/libc_errno.h" +#include "src/__support/OSUtil/linux/auxv.h" #include "src/__support/threads/callonce.h" #include "src/__support/threads/linux/futex_word.h" -#include "src/sys/auxv/getauxval.h" #include <linux/auxvec.h> // TODO: This is a temporary workaround to avoid including elf.h @@ -189,17 +188,13 @@ void Symbol::initialize_vdso_global_cache() { for (auto &i : global_cache) i = nullptr; - // get the address of the VDSO, protect errno since getauxval may change - // it - int errno_backup = libc_errno; - uintptr_t vdso_ehdr_addr = getauxval(AT_SYSINFO_EHDR); + cpp::optional<unsigned long> auxv_res = auxv::get(AT_SYSINFO_EHDR); + uintptr_t vdso_ehdr_addr = auxv_res ? static_cast<uintptr_t>(*auxv_res) : 0; // Get the memory address of the vDSO ELF header. auto vdso_ehdr = reinterpret_cast<ElfW(Ehdr) *>(vdso_ehdr_addr); // leave the table unpopulated if we don't have vDSO - if (vdso_ehdr == nullptr) { - libc_errno = errno_backup; + if (vdso_ehdr == nullptr) return; - } // locate the section header inside the elf using the section header // offset diff --git a/libc/src/__support/fixed_point/fx_bits.h b/libc/src/__support/fixed_point/fx_bits.h index 00c6119b..25221d9 100644 --- a/libc/src/__support/fixed_point/fx_bits.h +++ b/libc/src/__support/fixed_point/fx_bits.h @@ -10,9 +10,11 @@ #define LLVM_LIBC_SRC___SUPPORT_FIXED_POINT_FX_BITS_H #include "include/llvm-libc-macros/stdfix-macros.h" +#include "src/__support/CPP/algorithm.h" #include "src/__support/CPP/bit.h" #include "src/__support/CPP/limits.h" // numeric_limits #include "src/__support/CPP/type_traits.h" +#include "src/__support/libc_assert.h" #include "src/__support/macros/attributes.h" // LIBC_INLINE #include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL #include "src/__support/macros/null_check.h" // LIBC_CRASH_ON_VALUE @@ -21,6 +23,8 @@ #include "fx_rep.h" +#include <stdio.h> + #ifdef LIBC_COMPILER_HAS_FIXED_POINT namespace LIBC_NAMESPACE_DECL { @@ -224,6 +228,113 @@ idiv(T x, T y) { return static_cast<XType>(result); } +LIBC_INLINE long accum nrstep(long accum d, long accum x0) { + auto v = x0 * (2.lk - (d * x0)); + return v; +} + +// Divide the two integers and return a fixed_point value +// +// For reference, see: +// https://en.wikipedia.org/wiki/Division_algorithm#Newton%E2%80%93Raphson_division +// https://stackoverflow.com/a/9231996 + +template <typename XType> LIBC_INLINE constexpr XType divi(int n, int d) { + // If the value of the second operand of the / operator is zero, the + // behavior is undefined. Ref: ISO/IEC TR 18037:2008(E) p.g. 16 + LIBC_CRASH_ON_VALUE(d, 0); + + if (LIBC_UNLIKELY(n == 0)) { + return FXRep<XType>::ZERO(); + } + auto is_power_of_two = [](int n) { return (n > 0) && ((n & (n - 1)) == 0); }; + long accum max_val = static_cast<long accum>(FXRep<XType>::MAX()); + long accum min_val = static_cast<long accum>(FXRep<XType>::MIN()); + + if (is_power_of_two(cpp::abs(d))) { + int k = cpp::countr_zero<uint32_t>(static_cast<uint32_t>(cpp::abs(d))); + constexpr int F = FXRep<XType>::FRACTION_LEN; + int64_t scaled_n = static_cast<int64_t>(n) << F; + int64_t res64 = scaled_n >> k; + constexpr int TOTAL_BITS = sizeof(XType) * 8; + const int64_t max_limit = (1LL << (TOTAL_BITS - 1)) - 1; + const int64_t min_limit = -(1LL << (TOTAL_BITS - 1)); + if (res64 > max_limit) { + return FXRep<XType>::MAX(); + } else if (res64 < min_limit) { + return FXRep<XType>::MIN(); + } + long accum res_accum = + static_cast<long accum>(res64) / static_cast<long accum>(1 << F); + res_accum = (d < 0) ? static_cast<long accum>(-1) * res_accum : res_accum; + if (res_accum > max_val) { + return FXRep<XType>::MAX(); + } else if (res_accum < min_val) { + return FXRep<XType>::MIN(); + } + return static_cast<XType>(res_accum); + } + + bool result_is_negative = ((n < 0) != (d < 0)); + int64_t n64 = static_cast<int64_t>(n); + int64_t d64 = static_cast<int64_t>(d); + + uint64_t nv = static_cast<uint64_t>(n64 < 0 ? -n64 : n64); + uint64_t dv = static_cast<uint64_t>(d64 < 0 ? -d64 : d64); + + if (d == INT_MIN) { + nv <<= 1; + dv >>= 1; + } + + uint32_t clz = cpp::countl_zero<uint32_t>(static_cast<uint32_t>(dv)) - 1; + uint64_t scaled_val = dv << clz; + // Scale denominator to be in the range of [0.5,1] + FXBits<long accum> d_scaled{scaled_val}; + uint64_t scaled_val_n = nv << clz; + // Scale the numerator as much as the denominator to maintain correctness of + // the original equation + FXBits<long accum> n_scaled{scaled_val_n}; + long accum n_scaled_val = n_scaled.get_val(); + long accum d_scaled_val = d_scaled.get_val(); + // x0 = (48/17) - (32/17) * d_n + long accum a = 0x2.d89d89d8p0lk; // 48/17 = 2.8235294... + long accum b = 0x1.e1e1e1e1p0lk; // 32/17 = 1.8823529... + // Error of the initial approximation, as derived + // from the wikipedia article is + // E0 = 1/17 = 0.059 (5.9%) + long accum initial_approx = a - (b * d_scaled_val); + // Since, 0.5 <= d_scaled_val <= 1.0, 0.9412 <= initial_approx <= 1.88235 + LIBC_ASSERT((initial_approx >= 0x0.78793dd9p0lk) && + (initial_approx <= 0x1.f0f0d845p0lk)); + // Each newton-raphson iteration will square the error, due + // to quadratic convergence. So, + // E1 = (0.059)^2 = 0.0034 + long accum val = nrstep(d_scaled_val, initial_approx); + if constexpr (FXRep<XType>::FRACTION_LEN > 8) { + // E2 = 0.0000121 + val = nrstep(d_scaled_val, val); + if constexpr (FXRep<XType>::FRACTION_LEN > 16) { + // E3 = 1.468e−10 + val = nrstep(d_scaled_val, val); + } + } + long accum res = n_scaled_val * val; + + if (result_is_negative) { + res *= static_cast<long accum>(-1); + } + + // Per clause 7.18a.6.1, saturate values on overflow + if (res > max_val) { + return FXRep<XType>::MAX(); + } else if (res < min_val) { + return FXRep<XType>::MIN(); + } else { + return static_cast<XType>(res); + } +} + } // namespace fixed_point } // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/__support/math/CMakeLists.txt b/libc/src/__support/math/CMakeLists.txt index 203ebb4..61253de 100644 --- a/libc/src/__support/math/CMakeLists.txt +++ b/libc/src/__support/math/CMakeLists.txt @@ -109,22 +109,6 @@ add_header_library( libc.src.__support.macros.properties.types ) - -add_header_library( - rsqrtf16 - HDRS - rsqrtf16.h - DEPENDS - libc.src.__support.FPUtil.cast - libc.src.__support.FPUtil.fenv_impl - libc.src.__support.FPUtil.fp_bits - libc.src.__support.FPUtil.multiply_add - libc.src.__support.FPUtil.polyeval - libc.src.__support.FPUtil.manipulation_functions - libc.src.__support.macros.optimization - libc.src.__support.macros.properties.types -) - add_header_library( asin_utils HDRS @@ -867,6 +851,34 @@ add_header_library( ) add_header_library( + rsqrtf + HDRS + rsqrtf.h + DEPENDS + libc.hdr.errno_macros + libc.hdr.fenv_macros + libc.src.__support.FPUtil.cast + libc.src.__support.FPUtil.fenv_impl + libc.src.__support.FPUtil.fp_bits + libc.src.__support.FPUtil.sqrt + libc.src.__support.macros.optimization +) + +add_header_library( + rsqrtf16 + HDRS + rsqrtf16.h + DEPENDS + libc.hdr.errno_macros + libc.hdr.fenv_macros + libc.src.__support.FPUtil.cast + libc.src.__support.FPUtil.fenv_impl + libc.src.__support.FPUtil.fp_bits + libc.src.__support.FPUtil.sqrt + libc.src.__support.macros.optimization +) + +add_header_library( sincos_eval HDRS sincos_eval.h diff --git a/libc/src/__support/math/rsqrtf.h b/libc/src/__support/math/rsqrtf.h new file mode 100644 index 0000000..5da1e73 --- /dev/null +++ b/libc/src/__support/math/rsqrtf.h @@ -0,0 +1,71 @@ +//===-- Implementation header for rsqrtf ------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC___SUPPORT_MATH_RSQRTF_H +#define LLVM_LIBC_SRC___SUPPORT_MATH_RSQRTF_H + +#include "hdr/errno_macros.h" +#include "hdr/fenv_macros.h" +#include "src/__support/FPUtil/FEnvImpl.h" +#include "src/__support/FPUtil/FPBits.h" +#include "src/__support/FPUtil/cast.h" +#include "src/__support/FPUtil/sqrt.h" +#include "src/__support/macros/optimization.h" + +namespace LIBC_NAMESPACE_DECL { +namespace math { + +LIBC_INLINE static constexpr float rsqrtf(float x) { + using FPBits = fputil::FPBits<float>; + FPBits xbits(x); + + uint32_t x_u = xbits.uintval(); + uint32_t x_abs = x_u & 0x7fff'ffffU; + + constexpr uint32_t INF_BITS = FPBits::inf().uintval(); + + // x is 0, inf/nan, or negative. + if (LIBC_UNLIKELY(x_u == 0 || x_u >= INF_BITS)) { + // x is NaN + if (x_abs > INF_BITS) { + if (xbits.is_signaling_nan()) { + fputil::raise_except_if_required(FE_INVALID); + return FPBits::quiet_nan().get_val(); + } + return x; + } + + // |x| = 0 + if (x_abs == 0) { + fputil::raise_except_if_required(FE_DIVBYZERO); + fputil::set_errno_if_required(ERANGE); + return FPBits::inf(xbits.sign()).get_val(); + } + + // -inf <= x < 0 + if (x_u > 0x7fff'ffffU) { + fputil::raise_except_if_required(FE_INVALID); + fputil::set_errno_if_required(EDOM); + return FPBits::quiet_nan().get_val(); + } + + // x = +inf => rsqrt(x) = 0 + return FPBits::zero(xbits.sign()).get_val(); + } + + // TODO: add float based approximation when + // LIBC_TARGET_CPU_HAS_FPU_DOUBLE is not defined + double result = 1.0 / fputil::sqrt<double>(fputil::cast<double>(x)); + + return fputil::cast<float>(result); +} + +} // namespace math +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC___SUPPORT_MATH_RSQRTF_H diff --git a/libc/src/__support/threads/callonce.h b/libc/src/__support/threads/callonce.h index 5392722..0ccbcf9 100644 --- a/libc/src/__support/threads/callonce.h +++ b/libc/src/__support/threads/callonce.h @@ -23,14 +23,8 @@ #endif namespace LIBC_NAMESPACE_DECL { - -// Common definitions -using CallOnceCallback = void(void); -namespace callonce_impl { -int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *callback); -} // namespace callonce_impl - -LIBC_INLINE int callonce(CallOnceFlag *flag, CallOnceCallback *callback) { +template <class CallOnceCallback> +LIBC_INLINE int callonce(CallOnceFlag *flag, CallOnceCallback callback) { if (LIBC_LIKELY(callonce_impl::callonce_fastpath(flag))) return 0; diff --git a/libc/src/__support/threads/linux/CMakeLists.txt b/libc/src/__support/threads/linux/CMakeLists.txt index 14aaad2..39d2c6f 100644 --- a/libc/src/__support/threads/linux/CMakeLists.txt +++ b/libc/src/__support/threads/linux/CMakeLists.txt @@ -97,10 +97,8 @@ add_object_library( # value other than 0 is dangerous. We know. ) -add_object_library( +add_header_library( callonce - SRCS - callonce.cpp HDRS ../callonce.h callonce.h diff --git a/libc/src/__support/threads/linux/callonce.cpp b/libc/src/__support/threads/linux/callonce.cpp deleted file mode 100644 index c6e5f2a..0000000 --- a/libc/src/__support/threads/linux/callonce.cpp +++ /dev/null @@ -1,40 +0,0 @@ -//===-- Linux implementation of the callonce function ---------------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "src/__support/threads/callonce.h" -#include "src/__support/macros/config.h" -#include "src/__support/threads/linux/callonce.h" -#include "src/__support/threads/linux/futex_utils.h" - -namespace LIBC_NAMESPACE_DECL { -namespace callonce_impl { -int callonce_slowpath(CallOnceFlag *flag, CallOnceCallback *func) { - auto *futex_word = reinterpret_cast<Futex *>(flag); - - FutexWordType not_called = NOT_CALLED; - - // The call_once call can return only after the called function |func| - // returns. So, we use futexes to synchronize calls with the same flag value. - if (futex_word->compare_exchange_strong(not_called, START)) { - func(); - auto status = futex_word->exchange(FINISH); - if (status == WAITING) - futex_word->notify_all(); - return 0; - } - - FutexWordType status = START; - if (futex_word->compare_exchange_strong(status, WAITING) || - status == WAITING) { - futex_word->wait(WAITING); - } - - return 0; -} -} // namespace callonce_impl -} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/__support/threads/linux/callonce.h b/libc/src/__support/threads/linux/callonce.h index b3ea3a5..70872b6 100644 --- a/libc/src/__support/threads/linux/callonce.h +++ b/libc/src/__support/threads/linux/callonce.h @@ -26,6 +26,31 @@ static constexpr FutexWordType FINISH = 0x33; LIBC_INLINE bool callonce_fastpath(CallOnceFlag *flag) { return flag->load(cpp::MemoryOrder::RELAXED) == FINISH; } + +template <class CallOnceCallback> +[[gnu::noinline, gnu::cold]] int callonce_slowpath(CallOnceFlag *flag, + CallOnceCallback callback) { + + auto *futex_word = reinterpret_cast<Futex *>(flag); + + FutexWordType not_called = NOT_CALLED; + + // The call_once call can return only after the called function |func| + // returns. So, we use futexes to synchronize calls with the same flag value. + if (futex_word->compare_exchange_strong(not_called, START)) { + callback(); + auto status = futex_word->exchange(FINISH); + if (status == WAITING) + futex_word->notify_all(); + return 0; + } + + FutexWordType status = START; + if (futex_word->compare_exchange_strong(status, WAITING) || status == WAITING) + futex_word->wait(WAITING); + + return 0; +} } // namespace callonce_impl } // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/math/CMakeLists.txt b/libc/src/math/CMakeLists.txt index 3c7e99f..e37e22f 100644 --- a/libc/src/math/CMakeLists.txt +++ b/libc/src/math/CMakeLists.txt @@ -517,6 +517,7 @@ add_math_entrypoint_object(roundevenf16) add_math_entrypoint_object(roundevenf128) add_math_entrypoint_object(roundevenbf16) +add_math_entrypoint_object(rsqrtf) add_math_entrypoint_object(rsqrtf16) add_math_entrypoint_object(scalbln) diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt index 5738fe8..55f4aaf 100644 --- a/libc/src/math/generic/CMakeLists.txt +++ b/libc/src/math/generic/CMakeLists.txt @@ -989,6 +989,16 @@ add_entrypoint_object( ) add_entrypoint_object( + rsqrtf + SRCS + rsqrtf.cpp + HDRS + ../rsqrtf.h + DEPENDS + libc.src.__support.math.rsqrtf +) + +add_entrypoint_object( rsqrtf16 SRCS rsqrtf16.cpp diff --git a/libc/src/math/generic/rsqrtf.cpp b/libc/src/math/generic/rsqrtf.cpp new file mode 100644 index 0000000..6a38fa7 --- /dev/null +++ b/libc/src/math/generic/rsqrtf.cpp @@ -0,0 +1,16 @@ +//===-- Single-precision rsqrt function -----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception. +// +//===----------------------------------------------------------------------===// + +#include "src/math/rsqrtf.h" +#include "src/__support/math/rsqrtf.h" + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(float, rsqrtf, (float x)) { return math::rsqrtf(x); } + +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/math/rsqrtf.h b/libc/src/math/rsqrtf.h new file mode 100644 index 0000000..bc55904 --- /dev/null +++ b/libc/src/math/rsqrtf.h @@ -0,0 +1,20 @@ +//===-- Implementation header for rsqrtf ------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_MATH_RSQRTF_H +#define LLVM_LIBC_SRC_MATH_RSQRTF_H + +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +float rsqrtf(float x); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_MATH_RSQRTF_H diff --git a/libc/src/pthread/pthread_once.cpp b/libc/src/pthread/pthread_once.cpp index d78644a..109f70c 100644 --- a/libc/src/pthread/pthread_once.cpp +++ b/libc/src/pthread/pthread_once.cpp @@ -17,8 +17,7 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(int, pthread_once, (pthread_once_t * flag, __pthread_once_func_t func)) { - return callonce(reinterpret_cast<CallOnceFlag *>(flag), - reinterpret_cast<CallOnceCallback *>(func)); + return callonce(reinterpret_cast<CallOnceFlag *>(flag), func); } } // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/stdfix/CMakeLists.txt b/libc/src/stdfix/CMakeLists.txt index 843111e..3cbabd1 100644 --- a/libc/src/stdfix/CMakeLists.txt +++ b/libc/src/stdfix/CMakeLists.txt @@ -89,6 +89,20 @@ foreach(suffix IN ITEMS r lr k lk ur ulr uk ulk) ) endforeach() +foreach(suffix IN ITEMS r) + add_entrypoint_object( + ${suffix}divi + HDRS + ${suffix}divi.h + SRCS + ${suffix}divi.cpp + COMPILE_OPTIONS + ${libc_opt_high_flag} + DEPENDS + libc.src.__support.fixed_point.fx_bits + ) +endforeach() + add_entrypoint_object( uhksqrtus HDRS diff --git a/libc/src/stdfix/rdivi.cpp b/libc/src/stdfix/rdivi.cpp new file mode 100644 index 0000000..7021a8b --- /dev/null +++ b/libc/src/stdfix/rdivi.cpp @@ -0,0 +1,21 @@ +//===-- Implementation of rdivi function ---------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "rdivi.h" +#include "include/llvm-libc-macros/stdfix-macros.h" // fract +#include "src/__support/common.h" // LLVM_LIBC_FUNCTION +#include "src/__support/fixed_point/fx_bits.h" // fixed_point +#include "src/__support/macros/config.h" // LIBC_NAMESPACE_DECL + +namespace LIBC_NAMESPACE_DECL { + +LLVM_LIBC_FUNCTION(fract, rdivi, (int a, int b)) { + return fixed_point::divi<fract>(a, b); +} + +} // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/stdfix/rdivi.h b/libc/src/stdfix/rdivi.h new file mode 100644 index 0000000..aeda1ee --- /dev/null +++ b/libc/src/stdfix/rdivi.h @@ -0,0 +1,21 @@ +//===-- Implementation header for rdivi ------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STDFIX_RDIVI_H +#define LLVM_LIBC_SRC_STDFIX_RDIVI_H + +#include "include/llvm-libc-macros/stdfix-macros.h" +#include "src/__support/macros/config.h" + +namespace LIBC_NAMESPACE_DECL { + +fract rdivi(int a, int b); + +} // namespace LIBC_NAMESPACE_DECL + +#endif // LLVM_LIBC_SRC_STDFIX_RDIVI_H diff --git a/libc/src/sys/auxv/linux/CMakeLists.txt b/libc/src/sys/auxv/linux/CMakeLists.txt index 4884184c..39cfb7b 100644 --- a/libc/src/sys/auxv/linux/CMakeLists.txt +++ b/libc/src/sys/auxv/linux/CMakeLists.txt @@ -5,15 +5,6 @@ add_entrypoint_object( HDRS ../getauxval.h DEPENDS - libc.src.sys.prctl.prctl - libc.src.sys.mman.mmap - libc.src.sys.mman.munmap - libc.src.__support.threads.callonce - libc.src.__support.common - libc.src.errno.errno - libc.config.app_h - libc.src.fcntl.open - libc.src.unistd.read - libc.src.unistd.close - libc.include.sys_auxv + libc.src.__support.OSUtil.linux.auxv + libc.src.__support.libc_errno ) diff --git a/libc/src/sys/auxv/linux/getauxval.cpp b/libc/src/sys/auxv/linux/getauxval.cpp index b50c584..52d5a98 100644 --- a/libc/src/sys/auxv/linux/getauxval.cpp +++ b/libc/src/sys/auxv/linux/getauxval.cpp @@ -7,227 +7,14 @@ //===----------------------------------------------------------------------===// #include "src/sys/auxv/getauxval.h" -#include "config/app.h" -#include "hdr/fcntl_macros.h" -#include "src/__support/OSUtil/fcntl.h" -#include "src/__support/common.h" +#include "src/__support/OSUtil/linux/auxv.h" #include "src/__support/libc_errno.h" -#include "src/__support/macros/config.h" -#include <linux/auxvec.h> - -// for guarded initialization -#include "src/__support/threads/callonce.h" -#include "src/__support/threads/linux/futex_word.h" - -// ----------------------------------------------------------------------------- -// TODO: This file should not include other public libc functions. Calling other -// public libc functions is an antipattern within LLVM-libc. This needs to be -// cleaned up. DO NOT COPY THIS. -// ----------------------------------------------------------------------------- - -// for mallocing the global auxv -#include "src/sys/mman/mmap.h" -#include "src/sys/mman/munmap.h" - -// for reading /proc/self/auxv -#include "src/sys/prctl/prctl.h" -#include "src/unistd/read.h" - -// getauxval will work either with or without __cxa_atexit support. -// In order to detect if __cxa_atexit is supported, we define a weak symbol. -// We prefer __cxa_atexit as it is always defined as a C symbol whileas atexit -// may not be created via objcopy yet. Also, for glibc, atexit is provided via -// libc_nonshared.a rather than libc.so. So, it is may not be made ready for -// overlay builds. -extern "C" [[gnu::weak]] int __cxa_atexit(void (*callback)(void *), - void *payload, void *); namespace LIBC_NAMESPACE_DECL { - -constexpr static size_t MAX_AUXV_ENTRIES = 64; - -// Helper to recover or set errno -class AuxvErrnoGuard { -public: - AuxvErrnoGuard() : saved(libc_errno), failure(false) {} - ~AuxvErrnoGuard() { libc_errno = failure ? ENOENT : saved; } - void mark_failure() { failure = true; } - -private: - int saved; - bool failure; -}; - -// Helper to manage the memory -static AuxEntry *auxv = nullptr; - -class AuxvMMapGuard { -public: - constexpr static size_t AUXV_MMAP_SIZE = sizeof(AuxEntry) * MAX_AUXV_ENTRIES; - - AuxvMMapGuard() - : ptr(LIBC_NAMESPACE::mmap(nullptr, AUXV_MMAP_SIZE, - PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)) {} - ~AuxvMMapGuard() { - if (ptr != MAP_FAILED) - LIBC_NAMESPACE::munmap(ptr, AUXV_MMAP_SIZE); - } - void submit_to_global() { - // atexit may fail, we do not set it to global in that case. - int ret = __cxa_atexit( - [](void *) { - LIBC_NAMESPACE::munmap(auxv, AUXV_MMAP_SIZE); - auxv = nullptr; - }, - nullptr, nullptr); - - if (ret != 0) - return; - - auxv = reinterpret_cast<AuxEntry *>(ptr); - ptr = MAP_FAILED; - } - bool allocated() const { return ptr != MAP_FAILED; } - void *get() const { return ptr; } - -private: - void *ptr; -}; - -class AuxvFdGuard { -public: - AuxvFdGuard() { - auto result = internal::open("/proc/self/auxv", O_RDONLY | O_CLOEXEC); - if (!result.has_value()) - fd = -1; - - fd = result.value(); - } - ~AuxvFdGuard() { - if (fd != -1) - internal::close(fd); - } - bool valid() const { return fd != -1; } - int get() const { return fd; } - -private: - int fd; -}; - -static void initialize_auxv_once(void) { - // If we cannot get atexit, we cannot register the cleanup function. - if (&__cxa_atexit == nullptr) - return; - - AuxvMMapGuard mmap_guard; - if (!mmap_guard.allocated()) - return; - auto *ptr = reinterpret_cast<AuxEntry *>(mmap_guard.get()); - - // We get one less than the max size to make sure the search always - // terminates. MMAP private pages are zeroed out already. - size_t available_size = AuxvMMapGuard::AUXV_MMAP_SIZE - sizeof(AuxEntryType); - // PR_GET_AUXV is only available on Linux kernel 6.1 and above. If this is not - // defined, we direcly fall back to reading /proc/self/auxv. In case the libc - // is compiled and run on separate kernels, we also check the return value of - // prctl. -#ifdef PR_GET_AUXV - int ret = prctl(PR_GET_AUXV, reinterpret_cast<unsigned long>(ptr), - available_size, 0, 0); - if (ret >= 0) { - mmap_guard.submit_to_global(); - return; - } -#endif - AuxvFdGuard fd_guard; - if (!fd_guard.valid()) - return; - auto *buf = reinterpret_cast<char *>(ptr); - libc_errno = 0; - bool error_detected = false; - // Read until we use up all the available space or we finish reading the file. - while (available_size != 0) { - ssize_t bytes_read = - LIBC_NAMESPACE::read(fd_guard.get(), buf, available_size); - if (bytes_read <= 0) { - if (libc_errno == EINTR) - continue; - // Now, we either have an non-recoverable error or we have reached the end - // of the file. Mark `error_detected` accordingly. - if (bytes_read == -1) - error_detected = true; - break; - } - buf += bytes_read; - available_size -= bytes_read; - } - // If we get out of the loop without an error, the auxv is ready. - if (!error_detected) - mmap_guard.submit_to_global(); -} - -static AuxEntry read_entry(int fd) { - AuxEntry buf; - size_t size = sizeof(AuxEntry); - char *ptr = reinterpret_cast<char *>(&buf); - while (size > 0) { - ssize_t ret = LIBC_NAMESPACE::read(fd, ptr, size); - if (ret < 0) { - if (libc_errno == EINTR) - continue; - // Error detected, return AT_NULL - buf.id = AT_NULL; - buf.value = AT_NULL; - break; - } - ptr += ret; - size -= ret; - } - return buf; -} - LLVM_LIBC_FUNCTION(unsigned long, getauxval, (unsigned long id)) { - // Fast path when libc is loaded by its own initialization code. In this case, - // app.auxv_ptr is already set to the auxv passed on the initial stack of the - // process. - AuxvErrnoGuard errno_guard; - - auto search_auxv = [&errno_guard](AuxEntry *auxv, - unsigned long id) -> AuxEntryType { - for (auto *ptr = auxv; ptr->id != AT_NULL; ptr++) - if (ptr->id == id) - return ptr->value; - - errno_guard.mark_failure(); - return AT_NULL; - }; - - // App is a weak symbol that is only defined if libc is linked to its own - // initialization routine. We need to check if it is null. - if (&app != nullptr) - return search_auxv(app.auxv_ptr, id); - - static FutexWordType once_flag; - LIBC_NAMESPACE::callonce(reinterpret_cast<CallOnceFlag *>(&once_flag), - initialize_auxv_once); - if (auxv != nullptr) - return search_auxv(auxv, id); - - // Fallback to use read without mmap - AuxvFdGuard fd_guard; - if (fd_guard.valid()) { - while (true) { - AuxEntry buf = read_entry(fd_guard.get()); - if (buf.id == AT_NULL) - break; - if (buf.id == id) - return buf.value; - } - } - - // cannot find the entry after all methods, mark failure and return 0 - errno_guard.mark_failure(); - return AT_NULL; + if (cpp::optional<unsigned long> val = auxv::get(id)) + return *val; + libc_errno = ENOENT; + return 0; } } // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/threads/call_once.cpp b/libc/src/threads/call_once.cpp index 8466cd62..b60a20d 100644 --- a/libc/src/threads/call_once.cpp +++ b/libc/src/threads/call_once.cpp @@ -17,8 +17,7 @@ namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(void, call_once, (once_flag * flag, __call_once_func_t func)) { - callonce(reinterpret_cast<CallOnceFlag *>(flag), - reinterpret_cast<CallOnceCallback *>(func)); + callonce(reinterpret_cast<CallOnceFlag *>(flag), func); } } // namespace LIBC_NAMESPACE_DECL diff --git a/libc/src/unistd/linux/CMakeLists.txt b/libc/src/unistd/linux/CMakeLists.txt index dff6ba2..4eb3c7d 100644 --- a/libc/src/unistd/linux/CMakeLists.txt +++ b/libc/src/unistd/linux/CMakeLists.txt @@ -563,8 +563,8 @@ add_entrypoint_object( DEPENDS libc.include.unistd libc.include.sys_auxv - libc.src.errno.errno - libc.src.sys.auxv.getauxval + libc.src.__support.libc_errno + libc.src.__support.OSUtil.linux.auxv ) add_entrypoint_object( diff --git a/libc/src/unistd/linux/sysconf.cpp b/libc/src/unistd/linux/sysconf.cpp index 03f224b..979e178 100644 --- a/libc/src/unistd/linux/sysconf.cpp +++ b/libc/src/unistd/linux/sysconf.cpp @@ -11,17 +11,20 @@ #include "src/__support/common.h" #include "hdr/unistd_macros.h" +#include "src/__support/OSUtil/linux/auxv.h" #include "src/__support/libc_errno.h" #include "src/__support/macros/config.h" -#include "src/sys/auxv/getauxval.h" -#include <sys/auxv.h> namespace LIBC_NAMESPACE_DECL { LLVM_LIBC_FUNCTION(long, sysconf, (int name)) { long ret = 0; - if (name == _SC_PAGESIZE) - return static_cast<long>(getauxval(AT_PAGESZ)); + if (name == _SC_PAGESIZE) { + cpp::optional<unsigned long> page_size = auxv::get(AT_PAGESZ); + if (page_size) + return static_cast<long>(*page_size); + ret = -1; + } // TODO: Complete the rest of the sysconf options. if (ret < 0) { |